Beispiel #1
0
        static void InitAllDics()
        {
            //
            if (s_dicInit || s_dicProvider == null)
            {
                return;
            }

            if (s_thaiDic == null)
            {
                var customDic = new CustomDic();
                customDic.SetCharRange(ThaiDictionaryBreakingEngine.FirstChar, ThaiDictionaryBreakingEngine.LastChar);
                customDic.LoadSortedUniqueWordList(s_dicProvider.GetSortedUniqueWordList("thai"));
                s_thaiDic = customDic;
            }
            if (s_laoDic == null)
            {
                var customDic = new CustomDic();
                customDic.SetCharRange(LaoDictionaryBreakingEngine.FirstChar, LaoDictionaryBreakingEngine.LastChar);
                customDic.LoadSortedUniqueWordList(s_dicProvider.GetSortedUniqueWordList("lao"));
                s_laoDic = customDic;
            }


            if (s_enAbbrvDic == null)
            {
                s_enAbbrvDic = new CustomAbbrvDic();
                s_enAbbrvDic.LoadSortedUniqueWordList(s_dicProvider.GetSortedUniqueWordList("abbrv-en"));
            }

            s_dicInit = true;
        }
Beispiel #2
0
        static void InitAllDics()
        {
            if (thaiDicBreakingEngine == null)
            {
                var customDic = new CustomDic();
                thaiDicBreakingEngine = new ThaiDictionaryBreakingEngine();
                thaiDicBreakingEngine.SetDictionaryData(customDic);//add customdic to the breaker
                customDic.SetCharRange(thaiDicBreakingEngine.FirstUnicodeChar, thaiDicBreakingEngine.LastUnicodeChar);

                using (Stream data = s_dataProvider.GetDataStream("thaidict.txt"))
                {
                    customDic.LoadFromDataStream(data);
                }
            }
            if (laoDicBreakingEngine == null)
            {
                var customDic = new CustomDic();
                laoDicBreakingEngine = new LaoDictionaryBreakingEngine();
                laoDicBreakingEngine.SetDictionaryData(customDic);//add customdic to the breaker
                customDic.SetCharRange(laoDicBreakingEngine.FirstUnicodeChar, laoDicBreakingEngine.LastUnicodeChar);
                using (Stream data = s_dataProvider.GetDataStream("laodict.txt"))
                {
                    customDic.LoadFromDataStream(data);
                }
            }
        }
Beispiel #3
0
        static void InitAllDics()
        {
            if (thaiDicBreakingEngine == null)
            {
                var customDic = new CustomDic();
                thaiDicBreakingEngine = new ThaiDictionaryBreakingEngine();
                thaiDicBreakingEngine.SetDictionaryData(customDic);//add customdic to the breaker
                customDic.SetCharRange(thaiDicBreakingEngine.FirstUnicodeChar, thaiDicBreakingEngine.LastUnicodeChar);
                customDic.LoadSortedUniqueWordList(s_dicProvider.GetSortedUniqueWordList("thai"));
            }

            if (laoDicBreakingEngine == null)
            {
                var customDic = new CustomDic();
                laoDicBreakingEngine = new LaoDictionaryBreakingEngine();
                laoDicBreakingEngine.SetDictionaryData(customDic);//add customdic to the breaker
                customDic.SetCharRange(laoDicBreakingEngine.FirstUnicodeChar, laoDicBreakingEngine.LastUnicodeChar);
                customDic.LoadSortedUniqueWordList(s_dicProvider.GetSortedUniqueWordList("lao"));
            }
        }
Beispiel #4
0
        WordGroup _resultWordGroup;//after call DoIndex()
        internal void DoIndex(CustomDicTextBuffer textBuffer, CustomDic owner)
        {
            //recursive
            if (this.PrefixLen > 7)
            {
                DoIndexOfSmallAmount(textBuffer);
#if DEBUG
                dbugDataState = debugDataState.TooLongPrefix;
#endif
                return;
            }
            //-----------------------------------------------

            bool hasEvalPrefix = false;
            if (subGroups == null)
            {
                subGroups = new DevelopingWordGroup[owner.LastChar - owner.FirstChar + 1];
            }
            //--------------------------------
            int j             = wordSpanList.Count;
            int thisPrefixLen = this.PrefixLen;
            int doSepAt       = thisPrefixLen;
            for (int i = 0; i < j; ++i)
            {
                WordSpan sp = wordSpanList[i];
                if (sp.len > doSepAt)
                {
                    char c       = sp.GetChar(doSepAt, textBuffer);
                    int  c_index = c - owner.FirstChar;
                    DevelopingWordGroup found = subGroups[c_index];
                    if (found == null)
                    {
                        //not found
                        found = new DevelopingWordGroup(new WordSpan(sp.startAt, (byte)(doSepAt + 1)));
                        subGroups[c_index] = found;
                    }
                    found.AddWordSpan(sp);
                }
                else
                {
                    if (!hasEvalPrefix)
                    {
                        if (sp.SameTextContent(this.prefixSpan, textBuffer))
                        {
                            hasEvalPrefix     = true;
                            this.PrefixIsWord = true;
                        }
                    }
                }
            }
#if DEBUG
            this.dbugDataState = debugDataState.Indexed;
#endif
            wordSpanList.Clear();
            wordSpanList = null;
            //--------------------------------
            //do sup index
            //foreach (WordGroup subgroup in this.wordGroups.Values)
            bool hasSomeSubGroup = false;
            foreach (DevelopingWordGroup subgroup in this.subGroups)
            {
                if (subgroup != null)
                {
                    hasSomeSubGroup = true;

                    //****
                    //performance factor here,****
                    //in this current version
                    //if we not call DoIndex(),
                    //this subgroup need linear search-> so it slow
                    //so we call DoIndex until member count in the group <=3
                    //then it search faster,
                    //but dictionary-building time may increase.

                    if (subgroup.WordSpanListCount > 2)
                    {
                        subgroup.DoIndex(textBuffer, owner);
                    }
                    else
                    {
#if DEBUG
                        subgroup.dbugDataState = debugDataState.SmallAmountOfMembers;
#endif
                        subgroup.DoIndexOfSmallAmount(textBuffer);
                    }
                }
            }
            //--------------------------------
#if DEBUG
            this.dbugDataState = debugDataState.Indexed;
#endif
            if (!hasSomeSubGroup)
            {
                //clear
                subGroups = null;
            }

            //--------------------------------
            WordGroup[] newsubGroups = null;
            if (subGroups != null)
            {
                newsubGroups = new WordGroup[subGroups.Length];
                for (int i = subGroups.Length - 1; i >= 0; --i)
                {
                    DevelopingWordGroup subg = subGroups[i];
                    if (subg != null)
                    {
                        newsubGroups[i] = subg.ResultWordGroup;
                    }
                }
            }
            //--------------------------------
            this._resultWordGroup = new WordGroup(
                this.prefixSpan,
                newsubGroups,
                null,
                this.PrefixIsWord);
        }
Beispiel #5
0
 public void SetDictionaryData(CustomDic customDic)
 {
     _customDic = customDic;
 }