Exemplo n.º 1
0
        // public method
        // the input mergeObjList is in the same order of index as inTokenList
        // TBD: has bug: "imple ment ation" => implementimplementation
        public static List <TokenObj> CorrectTokenListByMerge(List <TokenObj> inTokenList, List <MergeObj> mergeObjList, string procHistStr, bool debugFlag, CSpellApi cSpellApi)
        {
            // 0. unify the mergeObjList to remove contain and overlap
            List <MergeObj> mergeObjListC = CleanUpMergeObjList(mergeObjList);

            List <TokenObj> outTokenList = new List <TokenObj>();
            // 1. go through all mergeObj
            int curIndex = 0;

            foreach (MergeObj mergeObj in mergeObjListC)
            {
                //System.out.println(mergeObj.ToString());
                int startIndex = mergeObj.GetStartIndex();
                int endIndex   = mergeObj.GetEndIndex();
                // 1. update tokens before merge start
                for (int i = curIndex; i < startIndex; i++)
                {
                    outTokenList.Add(inTokenList[i]);
                }
                // 2. update merge at target
                string   mergeWord     = mergeObj.GetMergeWord();
                string   orgMergeWord  = mergeObj.GetOrgMergeWord();
                string   tarWord       = mergeObj.GetTarWord();
                TokenObj mergeTokenObj = new TokenObj(orgMergeWord, mergeWord);
                // update process history
                for (int i = startIndex; i <= endIndex; i++)
                {
                    // merge focus token
                    if (i == mergeObj.GetTarIndex())
                    {
                        cSpellApi.UpdateCorrectNo();
                        mergeTokenObj.AddProcToHist(procHistStr + TokenObj.MERGE_START_STR + tarWord + TokenObj.MERGE_END_STR);
                        //DebugPrint.PrintCorrect("NW",
                        DebugPrint.PrintCorrect(procHistStr, "MergeCorrector (" + tarWord + ")", orgMergeWord, mergeWord, debugFlag);
                    }
                    else                         // not merge focus token, context
                    {
                        TokenObj      contextToken    = inTokenList[i];
                        List <string> contextProcHist = contextToken.GetProcHist();
                        foreach (string procHist in contextProcHist)
                        {
                            mergeTokenObj.AddProcToHist(procHist + TokenObj.MERGE_START_STR + contextToken.GetTokenStr() + TokenObj.MERGE_END_STR);
                        }
                    }
                }
                outTokenList.Add(mergeTokenObj);
                curIndex = endIndex + 1;
            }
            // 2. add tokens after the last merge Obj
            for (int i = curIndex; i < inTokenList.Count; i++)
            {
                outTokenList.Add(inTokenList[i]);
            }
            return(outTokenList);
        }
Exemplo n.º 2
0
        // public method
        /// <summary>
        /// The core method to correct a word by following steps:
        /// <ul>
        /// <li>Convert inToken to coreTerm
        /// <li>detect if real-word
        /// <li>get candidates
        ///     <ul>
        ///     <li>get candidates from one-to-one.
        ///     </ul>
        /// <li>Rank candidates
        ///     <ul>
        ///     <li>context
        ///     </ul>
        /// <li>Update information
        ///
        /// </ul>
        /// </summary>
        /// <param name="inTokenObj">    the input tokenObj (single word) </param>
        /// <param name="cSpellApi"> CSpell Api object </param>
        /// <param name="debugFlag"> flag for debug print </param>
        /// <param name="tarPos"> the position for target token </param>
        /// <param name="nonSpaceTokenList"> token list without space token(s)
        /// </param>
        /// <returns>    the corrected word in tokenObj if suggested word found.
        ///             Otherwise, the original input token is returned. </returns>
        // return the original term if no good correctin are found
        public static TokenObj GetCorrectTerm(TokenObj inTokenObj, CSpellApi cSpellApi, bool debugFlag, int tarPos, List <TokenObj> nonSpaceTokenList)
        {
            // init
            int funcMode = cSpellApi.GetFuncMode();

            // get inWord from inTokenObj and init outTokenObj
            string   inWord      = inTokenObj.GetTokenStr();
            TokenObj outTokenObj = new TokenObj(inTokenObj);
            // 1. convert a word to coreTerm (no leading/ending space, punc, digit)
            int         ctType      = CoreTermUtil.CT_TYPE_SPACE_PUNC_DIGIT;
            CoreTermObj coreTermObj = new CoreTermObj(inWord, ctType);
            string      coreStr     = coreTermObj.GetCoreTerm();

            // 2. real-word detection and correction
            // check if the coreTerm is real-word
            if ((inTokenObj.GetProcHist().Count == 0) && (RealWord1To1Detector.IsDetect(inWord, coreStr, cSpellApi, debugFlag) == true))
            {
                cSpellApi.UpdateDetectNo();
                // TBD, should take care of possessive xxx's here
                // 3 get 1-to-1 candidates set from correction
                // TBD. realWordFlag to use metaphone ...
                // this process is very slow, 7 min., need to improved
                HashSet <string> candSet = RealWord1To1Candidates.GetCandidates(coreStr, cSpellApi);
                /// <summary>
                ///** development analysis print out to see total RW
                ///            totalRwNo_++;
                ///            int candSize = candSet.size();
                ///            if(candSize != 0)
                ///            {
                ///                totalCandNo_ += candSize;
                ///                maxCandSize_
                ///                    = ((candSize > maxCandSize_)?candSize:maxCandSize_);
                ///                System.out.println("---- totalRwNo|totalCandNo(" + coreStr
                ///                    + "): " + totalRwNo_ + "|" + candSize + "|"
                ///                    + totalCandNo_ + "|" + maxCandSize_);
                ///                System.out.println(candSet);
                ///            }
                /// ***
                /// </summary>
                // 4. Ranking: get top ranked candidates as corrected terms
                // in case of using context
                string topRankStr = RankRealWord1To1ByCSpell.GetTopRankStr(coreStr, candSet, cSpellApi, tarPos, nonSpaceTokenList, debugFlag);
                // 5 update coreTerm and convert back to tokenObj
                coreTermObj.SetCoreTerm(topRankStr);
                string outWord = coreTermObj.ToString();
                // 6. update info if there is a real-word correction
                if (inWord.Equals(outWord, StringComparison.OrdinalIgnoreCase) == false)
                {
                    cSpellApi.UpdateCorrectNo();
                    outTokenObj.SetTokenStr(outWord);
                    outTokenObj.AddProcToHist(TokenObj.HIST_RW_1);                     // 1-to-1
                    DebugPrint.PrintCorrect("RW", "RealWord1To1Corrector", inWord, outWord, debugFlag);
                }
            }
            return(outTokenObj);
        }
        public static TokenObj Process(TokenObj inTokenObj)
        {
            string   inTokenStr  = inTokenObj.GetTokenStr();
            string   outTokenStr = Process(inTokenStr);
            TokenObj outTokenObj = new TokenObj(inTokenObj);

            //update info if there is a process
            if (inTokenStr.Equals(outTokenStr) == false)
            {
                outTokenObj.SetTokenStr(outTokenStr);
                outTokenObj.AddProcToHist(TokenObj.HIST_ND_S_E_P);
            }
            return(outTokenObj);
        }
        public static TokenObj Process(TokenObj inTokenObj, int maxProcess, bool debugFlag)
        {
            string   inTokenStr  = inTokenObj.GetTokenStr();
            string   outTokenStr = Process(inTokenStr, maxProcess);
            TokenObj outTokenObj = new TokenObj(inTokenObj);

            //update info if there is a process
            if (inTokenStr.Equals(outTokenStr) == false)
            {
                outTokenObj.SetTokenStr(outTokenStr);
                outTokenObj.AddProcToHist(TokenObj.HIST_ND_S_E_P);
                DebugPrint.PrintCorrect("ND", "EndingPuncSplitter", inTokenStr, outTokenStr, debugFlag);
            }
            return(outTokenObj);
        }
Exemplo n.º 5
0
        public static TokenObj Process(TokenObj inTokenObj, Dictionary <string, string> informalExpMap, bool debugFlag)
        {
            string   inTokenStr  = inTokenObj.GetTokenStr();
            string   outTokenStr = ProcessWord(inTokenStr, informalExpMap);
            TokenObj outTokenObj = new TokenObj(inTokenObj);

            //update info if there is a process
            if (inTokenStr.Equals(outTokenStr) == false)
            {
                outTokenObj.SetTokenStr(outTokenStr);
                outTokenObj.AddProcToHist(TokenObj.HIST_ND_INFORMAL_EXP);
                DebugPrint.PrintCorrect("ND", "InformalExpHandler", inTokenStr, outTokenStr, debugFlag);
            }
            return(outTokenObj);
        }
Exemplo n.º 6
0
        public static TokenObj Process(TokenObj inTokenObj, bool debugFlag)
        {
            // get string from tokenObj
            string inTokenStr  = inTokenObj.GetTokenStr();
            string outTokenStr = ProcessWord(inTokenStr);
            //update info if there is a XMl/Html process
            TokenObj outTokenObj = new TokenObj(inTokenObj);

            if (inTokenStr.Equals(outTokenStr) == false)
            {
                outTokenObj.SetTokenStr(outTokenStr);
                outTokenObj.AddProcToHist(TokenObj.HIST_ND_XML_HTML);
                DebugPrint.PrintCorrect("ND", "XmlHtmlHandler", inTokenStr, outTokenStr, debugFlag);
            }
            return(outTokenObj);
        }
        // public method
        /// <summary>
        /// The core method to correct a word by following steps:
        /// <ul>
        /// <li>Convert inToken to coreTerm
        /// <li>detect if real-word
        /// <li>get split candidates
        /// <li>Rank candidates
        ///     <ul>
        ///     <li>context
        ///     </ul>
        /// <li>Update information
        ///
        /// </ul>
        /// </summary>
        /// <param name="inTokenObj">    the input tokenObj (single word) </param>
        /// <param name="cSpellApi"> cSpell API object </param>
        /// <param name="debugFlag"> flag for debug print </param>
        /// <param name="tarPos"> position of the target token to be split </param>
        /// <param name="nonSpaceTokenList"> the token list without space tokens
        /// </param>
        /// <returns>    the split words in tokenObj.  </returns>
        // return the original term if no good correctin are found
        public static TokenObj GetCorrectTerm(TokenObj inTokenObj, CSpellApi cSpellApi, bool debugFlag, int tarPos, List <TokenObj> nonSpaceTokenList)
        {
            // init
            int funcMode = cSpellApi.GetFuncMode();

            // get inWord from inTokenObj and init outTokenObj
            string   inWord      = inTokenObj.GetTokenStr();
            TokenObj outTokenObj = new TokenObj(inTokenObj);
            // 1. convert a word to coreTerm (no leading/ending space, punc, digit)
            int         ctType      = CoreTermUtil.CT_TYPE_SPACE_PUNC_DIGIT;
            CoreTermObj coreTermObj = new CoreTermObj(inWord, ctType);
            string      coreStr     = coreTermObj.GetCoreTerm();

            // 2. non-word detection and correction
            // check if the coreTerm is real-word
            if ((inTokenObj.GetProcHist().Count == 0) && (RealWordSplitDetector.IsDetect(inWord, coreStr, cSpellApi, debugFlag) == true))
            {
                cSpellApi.UpdateDetectNo();
                // TBD, should take care of possessive xxx's here
                // 3. get split candidates set from correction
                int maxSplitNo            = cSpellApi.GetCanRwMaxSplitNo();
                HashSet <string> splitSet = RealWordSplitCandidates.GetCandidates(coreStr, cSpellApi, maxSplitNo);
                // get candidates from split
                // 4. Ranking: get top ranked candidates as corrected terms
                // in case of using context
                string topRankStr = RankRealWordSplitByMode.GetTopRankStr(coreStr, splitSet, cSpellApi, debugFlag, tarPos, nonSpaceTokenList);
                // 5 update coreTerm and convert back to tokenObj
                coreTermObj.SetCoreTerm(topRankStr);
                string outWord = coreTermObj.ToString();
                // 6. update info if there is a real-word correction
                if (inWord.Equals(outWord) == false)
                {
                    cSpellApi.UpdateCorrectNo();
                    outTokenObj.SetTokenStr(outWord);
                    outTokenObj.AddProcToHist(TokenObj.HIST_RW_S);                     //split
                    DebugPrint.PrintCorrect("RW", "RealWordSplitCorrector", inWord, outWord, debugFlag);
                }
            }
            return(outTokenObj);
        }
Exemplo n.º 8
0
        /// <summary>
        /// This method uses context scores to find the correct term.
        /// </summary>
        /// <param name="inTokenObj">    the input tokenObj (single word) </param>
        /// <param name="cSpellApi"> CSpell Api object </param>
        /// <param name="debugFlag"> flag for debug print </param>
        /// <param name="tarPos"> position for target token </param>
        /// <param name="nonSpaceTokenList"> token list without space token(s)
        /// </param>
        /// <returns>    the corrected word in tokenObj if the coreTerm is OOV
        ///             and suggested word found. Otherwise, the original input token
        ///             is returned. </returns>
        public static TokenObj GetCorrectTerm(TokenObj inTokenObj, CSpellApi cSpellApi, bool debugFlag, int tarPos, List <TokenObj> nonSpaceTokenList)
        {
            // init
            int funcMode = cSpellApi.GetFuncMode();

            // get inWord from inTokenObj and init outTokenObj
            string   inWord      = inTokenObj.GetTokenStr();
            TokenObj outTokenObj = new TokenObj(inTokenObj);
            // 1. convert a word to coreTerm (no leading/ending space, punc, digit)
            int         ctType      = CoreTermUtil.CT_TYPE_SPACE_PUNC_DIGIT;
            CoreTermObj coreTermObj = new CoreTermObj(inWord, ctType);
            string      coreStr     = coreTermObj.GetCoreTerm();

            // 2. non-word detection and correction
            // check if the coreTerm is spelling errors - non-word
            //!NonWordDetector.IsValidWord(inWord, coreStr, cSpellApi, debugFlag);
            // TBD .. need to separate 1-to-1 and split
            if (NonWordDetector.IsDetect(inWord, coreStr, cSpellApi, debugFlag) == true)
            {
                cSpellApi.UpdateDetectNo();
                // TBD, should take care of possessive xxx's here
                // 3.1 get 1-to-1 candidates set from correction, no split
                HashSet <string> candSet = NonWord1To1Candidates.GetCandidates(coreStr, cSpellApi);
                // add split
                // TBD ...
                if (funcMode != CSpellApi.FUNC_MODE_NW_1)
                {
                    // 3.2 get candidates from split
                    int maxSplitNo            = cSpellApi.GetCanNwMaxSplitNo();
                    HashSet <string> splitSet = NonWordSplitCandidates.GetCandidates(coreStr, cSpellApi, maxSplitNo);
                    // 3.4 set split candidates to candidate
                    if (funcMode == CSpellApi.FUNC_MODE_NW_S)
                    {
                        candSet = new HashSet <string>(splitSet);
                    }
                    else                         // 3.4 add split candidates
                    {
                        candSet.addAll(splitSet);
                    }
                }
                // 4. Ranking: get top ranked candidates as corrected terms
                // 4.1 from orthoGraphic

                /*
                 * // not used context
                 * String topRankStr = RankByMode.GetTopRankStr(coreStr, candSet,
                 *  cSpellApi, debugFlag);
                 */
                // in case of using context
                string topRankStr = RankNonWordByMode.GetTopRankStr(coreStr, candSet, cSpellApi, debugFlag, tarPos, nonSpaceTokenList);
                // 5 update coreTerm and convert back to tokenObj
                coreTermObj.SetCoreTerm(topRankStr);
                string outWord = coreTermObj.ToString();
                // 6. update info if there is a process
                if (inWord.Equals(outWord) == false)
                {
                    outTokenObj.SetTokenStr(outWord);
                    if (TermUtil.IsMultiword(outWord) == true)
                    {
                        cSpellApi.UpdateCorrectNo();
                        outTokenObj.AddProcToHist(TokenObj.HIST_NW_S);                         //split
                        DebugPrint.PrintCorrect("NW", "NonWordCorrector-Split", inWord, outWord, debugFlag);
                    }
                    else                         // 1To1 correct
                    {
                        cSpellApi.UpdateCorrectNo();
                        outTokenObj.AddProcToHist(TokenObj.HIST_NW_1);
                        DebugPrint.PrintCorrect("NW", "NonWordCorrector-1To1", inWord, outWord, debugFlag);
                    }
                }
            }
            return(outTokenObj);
        }