/// <summary> /// Compare two object o1 and o2. Both objects o1 and o2 are /// CSpellScore. The compare algorithm: /// </summary> /// <param name="o1"> first object to be compared </param> /// <param name="o2"> second object to be compared /// </param> /// <returns> a negative integer, 0, or positive integer to represent the /// object o1 is less, equals, or greater than object 02. </returns> public virtual int Compare(CSpellScore o1, CSpellScore o2) { int @out = 0; if (compareMode_ == COMPARE_BY_COMBO) { @out = compareByCombo(o1, o2); } else if (compareMode_ == COMPARE_BY_ORTHOGRAPHICS) { @out = compareByOrthographics(o1, o2); } else if (compareMode_ == COMPARE_BY_FREQUENCY) { @out = compareByFrequency(o1, o2); } else if (compareMode_ == COMPARE_BY_CONTEXT) { @out = compareByContext(o1, o2); } else if (compareMode_ == COMPARE_BY_ENSEMBLE) { @out = compareByEnsemble(o1, o2); } return(@out); }
private int compareByEnsemble(CSpellScore o1, CSpellScore o2) { int @out = 0; OrthographicScore oScore1 = ((CSpellScore)o1).GetOScore(); OrthographicScore oScore2 = ((CSpellScore)o2).GetOScore(); FrequencyScore fScore1 = ((CSpellScore)o1).GetFScore(); FrequencyScore fScore2 = ((CSpellScore)o2).GetFScore(); ContextScore cScore1 = ((CSpellScore)o1).GetCScore(); ContextScore cScore2 = ((CSpellScore)o2).GetCScore(); double score1 = 0.6 * oScore1.GetScore() + 0.25 * fScore1.GetScore() + 0.15 * cScore1.GetScore(); double score2 = 0.6 * oScore2.GetScore() + 0.25 * fScore2.GetScore() + 0.15 * cScore2.GetScore(); // 1. compared by orthographic score, best // SCR-2: use a fixed number to ensure result is not 0. if (score2 > score1) { // from high to low @out = 1; } else if (score2 < score1) { @out = -1; } // 2. alphabetic order else { string cand1 = ((CSpellScore)o1).GetCandStr(); string cand2 = ((CSpellScore)o2).GetCandStr(); @out = cand2.CompareTo(cand1); } return(@out); }
/// <summary> /// Compare two object o1 and o2. Both objects o1 and o2 are /// CSpellScore. The compare algorithm: /// </summary> /// <param name="o1"> first object to be compared </param> /// <param name="o2"> second object to be compared /// </param> /// <returns> a negative integer, 0, or positive integer to represent the /// object o1 is less, equals, or greater than object 02. </returns> public virtual int Compare(CSpellScore o1, CSpellScore o2) { OrthographicScoreComparator <OrthographicScore> osc = new OrthographicScoreComparator <OrthographicScore>(); OrthographicScore oScore1 = ((CSpellScore)o1).GetOScore(); OrthographicScore oScore2 = ((CSpellScore)o2).GetOScore(); int @out = osc.Compare(oScore1, oScore2); // the following code are simplfied by above // We still keep this java file in case we need better implementation // in the future /// <summary> /// int out = 0; /// // 1. compared by orthographic score, best /// OrthographicScore oScore1 = ((CSpellScore) o1).GetOScore(); /// OrthographicScore oScore2 = ((CSpellScore) o2).GetOScore(); /// if(oScore1.GetScore() != oScore2.GetScore()) /// { /// OrthographicScoreComparator<OrthographicScore> osc /// = new OrthographicScoreComparator<OrthographicScore>(); /// out = osc.compare(oScore1, oScore2); /// } /// else /// { /// String cand1 = ((CSpellScore) o1).GetCandStr(); /// String cand2 = ((CSpellScore) o2).GetCandStr(); /// out = cand2.compareTo(cand1); /// } /// /// </summary> return(@out); }
// return candidate set with cSpell score // wordStr is the srcTxt used to calculate the score between it and cand public static HashSet <CSpellScore> GetCandidateScoreSet(string wordStr, HashSet <string> candidates, WordWcMap wordWcMap, int tarPos, int tarSize, List <TokenObj> nonSpaceTokenList, Word2Vec word2VecIm, Word2Vec word2VecOm, bool word2VecSkipWord, int contextRadius, double wf1, double wf2, double wf3, bool debugFlag) { HashSet <CSpellScore> candScoreSet = new HashSet <CSpellScore>(); foreach (string cand in candidates) { // find context for each candidates DoubleVec contextVec = Word2VecContext.GetContextVec(tarPos, tarSize, nonSpaceTokenList, word2VecIm, contextRadius, word2VecSkipWord, debugFlag); CSpellScore cs = new CSpellScore(wordStr, cand, wordWcMap, contextVec, word2VecOm, wf1, wf2, wf3); candScoreSet.Add(cs); } return(candScoreSet); }
// by combination, O, N, F, C private int compareByCombo(CSpellScore o1, CSpellScore o2) { int @out = 0; OrthographicScore oScore1 = ((CSpellScore)o1).GetOScore(); OrthographicScore oScore2 = ((CSpellScore)o2).GetOScore(); NoisyChannelScore nScore1 = ((CSpellScore)o1).GetNScore(); NoisyChannelScore nScore2 = ((CSpellScore)o2).GetNScore(); FrequencyScore fScore1 = ((CSpellScore)o1).GetFScore(); FrequencyScore fScore2 = ((CSpellScore)o2).GetFScore(); ContextScore cScore1 = ((CSpellScore)o1).GetCScore(); ContextScore cScore2 = ((CSpellScore)o2).GetCScore(); // 1. compared by orthographic score, best if (oScore1.GetScore() != oScore2.GetScore()) { OrthographicScoreComparator <OrthographicScore> osc = new OrthographicScoreComparator <OrthographicScore>(); @out = osc.Compare(oScore1, oScore2); } // 2. compared by noise channel score, 2nd best else if (nScore1.GetScore() != nScore2.GetScore()) { NoisyChannelScoreComparator <NoisyChannelScore> nsc = new NoisyChannelScoreComparator <NoisyChannelScore>(); @out = nsc.Compare(nScore1, nScore2); } // 3. compared by pure frequency score, 3rd best else if (fScore1.GetScore() != fScore2.GetScore()) { FrequencyScoreComparator <FrequencyScore> fsc = new FrequencyScoreComparator <FrequencyScore>(); @out = fsc.Compare(fScore1, fScore2); } // 4. compared by context score, 4 last else if (cScore1.GetScore() != cScore2.GetScore()) { ContextScoreComparator <ContextScore> csc = new ContextScoreComparator <ContextScore>(); @out = csc.Compare(cScore1, cScore2); } // 5. alphabetic order else { string cand1 = ((CSpellScore)o1).GetCandStr(); string cand2 = ((CSpellScore)o2).GetCandStr(); @out = cand2.CompareTo(cand1); } return(@out); }
private int compareByFrequency(CSpellScore o1, CSpellScore o2) { int @out = 0; FrequencyScore fScore1 = ((CSpellScore)o1).GetFScore(); FrequencyScore fScore2 = ((CSpellScore)o2).GetFScore(); // 1. compared by context score, 4 last if (fScore1.GetScore() != fScore2.GetScore()) { FrequencyScoreComparator <FrequencyScore> fsc = new FrequencyScoreComparator <FrequencyScore>(); @out = fsc.Compare(fScore1, fScore2); } // 2. alphabetic order else { string cand1 = ((CSpellScore)o1).GetCandStr(); string cand2 = ((CSpellScore)o2).GetCandStr(); @out = cand2.CompareTo(cand1); } return(@out); }
private int compareByContext(CSpellScore o1, CSpellScore o2) { int @out = 0; ContextScore cScore1 = ((CSpellScore)o1).GetCScore(); ContextScore cScore2 = ((CSpellScore)o2).GetCScore(); // 1. compared by context score, 4 last if (cScore1.GetScore() != cScore2.GetScore()) { ContextScoreComparator <ContextScore> csc = new ContextScoreComparator <ContextScore>(); @out = csc.Compare(cScore1, cScore2); } // 2. alphabetic order else { string cand1 = ((CSpellScore)o1).GetCandStr(); string cand2 = ((CSpellScore)o2).GetCandStr(); @out = cand2.CompareTo(cand1); } return(@out); }
private int compareByOrthographics(CSpellScore o1, CSpellScore o2) { int @out = 0; OrthographicScore oScore1 = ((CSpellScore)o1).GetOScore(); OrthographicScore oScore2 = ((CSpellScore)o2).GetOScore(); // 1. compared by orthographic score, best if (oScore1.GetScore() != oScore2.GetScore()) { OrthographicScoreComparator <OrthographicScore> osc = new OrthographicScoreComparator <OrthographicScore>(); @out = osc.Compare(oScore1, oScore2); } // 2. alphabetic order else { string cand1 = ((CSpellScore)o1).GetCandStr(); string cand2 = ((CSpellScore)o2).GetCandStr(); @out = cand2.CompareTo(cand1); } return(@out); }
// return the best ranked str from candidates using word2Vec score // inTokenList, includes space token, is not coreTerm.Lc // return the orignal inStr if no candidate has score > 0.0d public static string GetTopRankStr(string inStr, HashSet <string> candidates, CSpellApi cSpellApi, int tarPos, List <TokenObj> nonSpaceTokenList, bool debugFlag) { // init WordWcMap wordWcMap = cSpellApi.GetWordWcMap(); Word2Vec word2VecIm = cSpellApi.GetWord2VecIm(); Word2Vec word2VecOm = cSpellApi.GetWord2VecOm(); int contextRadius = cSpellApi.GetRw1To1ContextRadius(); bool word2VecSkipWord = cSpellApi.GetWord2VecSkipWord(); int maxCandNo = cSpellApi.GetCanMaxCandNo(); double wf1 = cSpellApi.GetOrthoScoreEdDistFac(); double wf2 = cSpellApi.GetOrthoScorePhoneticFac(); double wf3 = cSpellApi.GetOrthoScoreOverlapFac(); int tarSize = 1; // only for one-to-one, no merge here string topRankStr = inStr; // use cSpell top candidates int topNo = 1; // top sort string inStrLc = inStr.ToLower(); List <CSpellScore> cSpellScoreList = RankByCSpellRealWord1To1.GetCandidateScoreList(inStrLc, candidates, wordWcMap, tarPos, tarSize, nonSpaceTokenList, word2VecIm, word2VecOm, word2VecSkipWord, contextRadius, wf1, wf2, wf3, debugFlag); // Find the correction str and correct if (cSpellScoreList.Count > 0) { // the rw top rank must be in both NC and orthographic CSpellScore topScore = cSpellScoreList[0]; double topFScore = topScore.GetFScore().GetScore(); //frequency double topTScore = topScore.GetOScore().GetTokenScore(); // Token double topPScore = topScore.GetOScore().GetPhoneticScore(); //Phone double topOScore = topScore.GetOScore().GetOverlapScore(); //overlap ContextScore orgContextScore = null; // check the frequency // get the max score of frequency, eidt, phonetic, and overlap // the top rank must have all top score for above if ((topFScore == CSpellScore.GetMaxFScore(cSpellScoreList)) && (topTScore == CSpellScore.GetMaxEScore(cSpellScoreList)) && (topPScore == CSpellScore.GetMaxPScore(cSpellScoreList)) && (topOScore == CSpellScore.GetMaxOScore(cSpellScoreList))) { ContextScore topContextScore = topScore.GetCScore(); // 1.1 wordVec for context DoubleVec contextVec = Word2VecContext.GetContextVec(tarPos, tarSize, nonSpaceTokenList, word2VecIm, contextRadius, word2VecSkipWord, debugFlag); // 1.2 wordVec for the original words before one-to-one orgContextScore = new ContextScore(inStr, contextVec, word2VecOm); FrequencyScore orgFScore = new FrequencyScore(inStr, wordWcMap); // pass the orgContextScore if (IsTopCandValid(inStr, orgContextScore, topScore, orgFScore, cSpellApi, debugFlag) == true) { // no correction: if score is not good enough for corection topRankStr = topScore.GetCandStr(); // debug print for ananlysis /// <summary> ///* /// System.out.println("======= cSpellScoreList.size(): " /// + cSpellScoreList.size() + " ========"); /// System.out.println(inStr /// + "," + String.format("%1.8f", orgFScore.GetScore()) /// + "," + String.format("%1.8f", orgContextScore.GetScore())); /// System.out.println(CSpellScore.GetScoreHeader()); /// for(CSpellScore cSpellScore: cSpellScoreList) /// { /// System.out.println(cSpellScore.ToString(",")); /// } /// ** /// </summary> } } // debug print if (debugFlag == true) { // print focus token (original) if (orgContextScore != null) { DebugPrint.PrintScore(orgContextScore.ToString(), debugFlag); } else { DebugPrint.PrintScore("No score for focus (" + inStr + ")", debugFlag); } // print candidate var list = cSpellScoreList.Take(maxCandNo).Select(obj => obj.ToString()).ToList(); foreach (var item in list) { DebugPrint.PrintScore(item, debugFlag); } } } return(topRankStr); }
// private methods private static bool IsTopCandValidByScores(ContextScore orgContextScore, FrequencyScore orgFreqScore, ContextScore topContextScore, CSpellScore topCSpellScore, CSpellApi cSpellApi) { // init bool flag = false; double rw1To1CandCsFactor = cSpellApi.GetRankRw1To1CandCsFac(); double rw1To1WordMinCs = cSpellApi.GetRankRw1To1WordMinCs(); double rw1To1CandMinCs = cSpellApi.GetRankRw1To1CandMinCs(); double rw1To1CandCsDist = cSpellApi.GetRankRw1To1CandCsDist(); double rw1To1CandFsFactor = cSpellApi.GetRankRw1To1CandFsFac(); double rw1To1CandMinFs = cSpellApi.GetRankRw1To1CandMinFs(); double rw1To1CandFsDist = cSpellApi.GetRankRw1To1CandFsDist(); double orgScore = orgContextScore.GetScore(); double topScore = topContextScore.GetScore(); // another rule for word2Vec on real-word // check contect score: // 1. the topScore is bigger enough to cover the orgScore // 2. the distance is > a value for confidence if (((topScore / -orgScore) > rw1To1CandCsFactor) && (orgScore > rw1To1WordMinCs) && (topScore > rw1To1CandMinCs) && ((topScore - orgScore) > rw1To1CandCsDist)) //609|796|0.6920 // check frequency, all positive: // 1. cand has better frequency // 2. the difference is withint a range { double orgFScore = orgFreqScore.GetScore(); double topFScore = topCSpellScore.GetFScore().GetScore(); if (((topFScore / orgFScore) > rw1To1CandFsFactor) && (topFScore > rw1To1CandMinFs) && ((topFScore > orgFScore) || ((orgFScore - topFScore) < rw1To1CandFsDist))) // within freq range { flag = true; } } return(flag); }
// Use context and frequency scor eto validate the top ranked candidate private static bool IsTopCandValid(string inStr, ContextScore orgContextScore, CSpellScore topCSpellScore, FrequencyScore orgFreqScore, CSpellApi cSpellApi, bool debugFlag) { ContextScore topContextScore = topCSpellScore.GetCScore(); // return false if no topCand found if ((topContextScore == null) || (topContextScore.GetTerm().Equals(inStr))) { return(false); } // Score rules for one-to-one double orgScore = orgContextScore.GetScore(); double topScore = topContextScore.GetScore(); bool flag = false; double rw1To1CFactor = cSpellApi.GetRankRw1To1CFac(); // 2.1 no 1-to-1 correction if orgScore is 0.0d, no word2Vec information if (orgScore < 0.0d) { // 2.2a one-to-one if the org score is negative and top score is positive if (topScore > 0.0d) { // further check by ratio, dist, and min. by CScore and FScore if (IsTopCandValidByScores(orgContextScore, orgFreqScore, topContextScore, topCSpellScore, cSpellApi) == true) { flag = true; } } // 2.2b 1-to-1 if the org score is negative, top score is better else if ((topScore < 0.0d) && (topScore > orgScore * rw1To1CFactor)) { flag = true; } } else if (orgScore > 0.0d) { // 2.3a merge if the org score is positive, better 0.01*topScore if (topScore * rw1To1CFactor > orgScore) { flag = true; } } return(flag); }