コード例 #1
0
ファイル: SymSpell.cs プロジェクト: rlebowitz/ai
 //save some time and space
 private void AddLowestDistance(DictionaryItem item, string suggestion, int suggestionint, string delete)
 {
     //remove all existing suggestions of higher distance, if verbose<2
     //index2word
     if ((verbose < 2) && (item.Suggestions.Count > 0) && (Wordlist[item.Suggestions[0]].Length - delete.Length > suggestion.Length - delete.Length)) item.Suggestions.Clear();
     //do not add suggestion of higher distance than existing, if verbose<2
     if ((verbose == 2) || (item.Suggestions.Count == 0) || (Wordlist[item.Suggestions[0]].Length - delete.Length >= suggestion.Length - delete.Length)) item.Suggestions.Add(suggestionint);
 }
コード例 #2
0
ファイル: SymSpell.cs プロジェクト: rlebowitz/ai
        private List<SuggestItem> Lookup(string input, int editDistanceMax)
        {
            //save some time
            if (input.Length - editDistanceMax > Maxlength) return new List<SuggestItem>();

            var candidates = new List<string>();
            var hashset1 = new HashSet<string>();

            var suggestions = new List<SuggestItem>();
            var hashset2 = new HashSet<string>();

            //add original term
            candidates.Add(input);

            while (candidates.Count > 0)
            {
                var candidate = candidates[0];
                candidates.RemoveAt(0);

                //save some time
                //early termination
                //suggestion distance=candidate.distance... candidate.distance+editDistanceMax                
                //if canddate distance is already higher than suggestion distance, than there are no better suggestions to be expected
                if ((verbose < 2) && (suggestions.Count > 0) && (input.Length - candidate.Length > suggestions[0].Distance)) goto sort;


                //read candidate entry from dictionary
                object valueo;
                if (Dictionary.TryGetValue(candidate, out valueo))
                {
                    var value = new DictionaryItem();
                    if (valueo is int) value.Suggestions.Add((int)valueo); else value = (DictionaryItem)valueo;

                    //if count>0 then candidate entry is correct dictionary term, not only delete item
                    if ((value.Count > 0) && hashset2.Add(candidate))
                    {
                        //add correct dictionary term term to suggestion list
                        var si = new SuggestItem(candidate, value.Count, input.Length - candidate.Length);
                        suggestions.Add(si);
                        //early termination
                        if ((verbose < 2) && (input.Length - candidate.Length == 0)) goto sort;
                    }

                    //iterate through suggestions (to other correct dictionary items) of delete item and add them to suggestion list
                    foreach (int suggestionint in value.Suggestions)
                    {
                        //save some time 
                        //skipping double items early: different deletes of the input term can lead to the same suggestion
                        //index2word
                        string suggestion = Wordlist[suggestionint];
                        if (hashset2.Add(suggestion))
                        {
                            //True Damerau-Levenshtein Edit Distance: adjust distance, if both distances>0
                            //We allow simultaneous edits (deletes) of editDistanceMax on on both the dictionary and the input term. 
                            //For replaces and adjacent transposes the resulting edit distance stays <= editDistanceMax.
                            //For inserts and deletes the resulting edit distance might exceed editDistanceMax.
                            //To prevent suggestions of a higher edit distance, we need to calculate the resulting edit distance, if there are simultaneous edits on both sides.
                            //Example: (bank==bnak and bank==bink, but bank!=kanb and bank!=xban and bank!=baxn for editDistanceMaxe=1)
                            //Two deletes on each side of a pair makes them all equal, but the first two pairs have edit distance=1, the others edit distance=2.
                            int distance = 0;
                            if (suggestion != input)
                            {
                                if (suggestion.Length == candidate.Length) distance = input.Length - candidate.Length;
                                else if (input.Length == candidate.Length) distance = suggestion.Length - candidate.Length;
                                else
                                {
                                    //common prefixes and suffixes are ignored, because this speeds up the Damerau-levenshtein-Distance calculation without changing it.
                                    int ii = 0;
                                    int jj = 0;
                                    while ((ii < suggestion.Length) && (ii < input.Length) && (suggestion[ii] == input[ii])) ii++;
                                    while ((jj < suggestion.Length - ii) && (jj < input.Length - ii) && (suggestion[suggestion.Length - jj - 1] == input[input.Length - jj - 1])) jj++;
                                    if ((ii > 0) || (jj > 0)) { distance = DamerauLevenshteinDistance(suggestion.Substring(ii, suggestion.Length - ii - jj), input.Substring(ii, input.Length - ii - jj)); } else distance = DamerauLevenshteinDistance(suggestion, input);

                                }
                            }

                            //save some time.
                            //remove all existing suggestions of higher distance, if verbose<2
                            if ((verbose < 2) && (suggestions.Count > 0) && (suggestions[0].Distance > distance)) suggestions.Clear();
                            //do not process higher distances than those already found, if verbose<2
                            if ((verbose < 2) && (suggestions.Count > 0) && (distance > suggestions[0].Distance)) continue;

                            if (distance > editDistanceMax) continue;
                            object value2;
                            if (!Dictionary.TryGetValue(suggestion, out value2)) continue;
                            if (value2 == null) continue;
                            var item = (DictionaryItem)value2;
                            var si = new SuggestItem(suggestion, item.Count, distance);
                            suggestions.Add(si);
                        }
                    }//end foreach
                }//end if         

                //add edits 
                //derive edits (deletes) from candidate (input) and add them to candidates list
                //this is a recursive process until the maximum edit distance has been reached
                if (input.Length - candidate.Length < editDistanceMax)
                {
                    //save some time
                    //do not create edits with edit distance smaller than suggestions already found
                    if ((verbose < 2) && (suggestions.Count > 0) && (input.Length - candidate.Length >= suggestions[0].Distance)) continue;

                    candidates.AddRange(candidate.Select((t, i) => candidate.Remove(i, 1)).Where(delete => hashset1.Add(delete)));
                }
            }//end while

            //sort by ascending edit distance, then by descending word frequency
            sort: if (verbose < 2) suggestions.Sort((x, y) => -x.Count.CompareTo(y.Count)); else suggestions.Sort((x, y) => 2 * x.Distance.CompareTo(y.Distance) - x.Count.CompareTo(y.Count));
            if ((verbose == 0) && (suggestions.Count > 1)) return suggestions.GetRange(0, 1);
            return suggestions;
        }
コード例 #3
0
ファイル: SymSpell.cs プロジェクト: rlebowitz/ai
        //for every word there all deletes with an edit distance of 1..editDistanceMax created and added to the dictionary
        //every delete entry has a suggestions list, which points to the original term(s) it was created from
        //The dictionary may be dynamically updated (word frequency and new words) at any time by calling createDictionaryEntry
        private bool CreateDictionaryEntry(string key)
        {
            bool result = false;
            DictionaryItem value = null;
            object valueo;
            if (Dictionary.TryGetValue(key, out valueo))
            {
                //int or dictionaryItem? delete existed before word!
                if (valueo is int)
                {
                    var tmp = (int)valueo;
                    value = new DictionaryItem();
                    value.Suggestions.Add(tmp);
                    Dictionary[key] = value;
                }

                //already exists:
                //1. word appears several times
                //2. word1==deletes(word2) 
                else
                {
                    value = (valueo as DictionaryItem);
                }
                //prevent overflow
                if (value != null && value.Count < int.MaxValue)
                {
                    value.Count++;
                }
            }
            else if (Wordlist.Count < int.MaxValue)
            {
                value = new DictionaryItem();
                value.Count++;
                Dictionary.Add(key, value);

                if (key.Length > Maxlength) Maxlength = key.Length;
            }


            //edits/suggestions are created only once, no matter how often word occurs
            //edits/suggestions are created only as soon as the word occurs in the corpus, 
            //even if the same term existed before in the dictionary as an edit from another word
            //a treshold might be specifid, when a term occurs so frequently in the corpus that it is considered a valid word for spelling correction
            if (value != null && value.Count == 1)
            {
                //word2index
                Wordlist.Add(key);
                if (key.Equals("I", StringComparison.InvariantCultureIgnoreCase))
                {
                    Console.WriteLine(key);
                }
                var keyint = Wordlist.Count - 1;

                result = true;

                //create deletes
                foreach (var delete in Edits(key, 0, new HashSet<string>()))
                {
                    object value2;
                    if (Dictionary.TryGetValue(delete, out value2))
                    {
                        //already exists:
                        //1. word1==deletes(word2) 
                        //2. deletes(word1)==deletes(word2) 
                        //int or dictionaryItem? single delete existed before!
                        if (value2 is int)
                        {
                            //transformes int to dictionaryItem
                            var tmp = (int)value2;
                            DictionaryItem di = new DictionaryItem();
                            di.Suggestions.Add(tmp);
                            Dictionary[delete] = di;
                            if (!di.Suggestions.Contains(keyint))
                            {
                                AddLowestDistance(di, key, keyint, delete);
                            }
                        }
                        else
                        {
                            var dictionaryItem = value2 as DictionaryItem;
                            if (dictionaryItem != null && !dictionaryItem.Suggestions.Contains(keyint))
                            {
                                AddLowestDistance((DictionaryItem) value2, key, keyint, delete);
                            }
                        }
                    }
                    else
                    {
                        Dictionary.Add(delete, keyint);
                    }
                }
            }
            return result;
        }