Beispiel #1
0
        // *** End of Dec-2011 ***

        public void Tag(Corpus corpus, out int lemmaCorrect, out int lemmaCorrectLowercase, out int lemmaWords, bool xmlMode)
        {
            DateTime startTime = DateTime.Now;

            mLogger.Debug("Tag", "Označujem besedilo ...");
            lemmaCorrect          = 0;
            lemmaCorrectLowercase = 0;
            lemmaWords            = 0;
            for (int i = 0; i < corpus.TaggedWords.Count; i++)
            {
                mLogger.ProgressFast(Logger.Level.Info, /*sender=*/ this, "Tag", "{0} / {1}", i + 1, corpus.TaggedWords.Count);
                BinaryVector        featureVector = corpus.GenerateFeatureVector(i, mFeatureSpace, /*extendFeatureSpace=*/ false, mSuffixTree);
                Prediction <string> result        = mModel.Predict(featureVector);
                if ((corpus.TaggedWords[i].MoreInfo != null && corpus.TaggedWords[i].MoreInfo.Punctuation) ||
                    (corpus.TaggedWords[i].MoreInfo == null && mNonWordRegex.Match(corpus.TaggedWords[i].WordLower).Success)) // non-word
                {
                    bool flag = false;
                    foreach (KeyDat <double, string> item in result)
                    {
                        if (corpus.TaggedWords[i].Word == item.Dat || corpus.TaggedWords[i].Word + "<eos>" == item.Dat)
                        {
                            corpus.TaggedWords[i].Tag = item.Dat;
                            flag = true;
                            break;
                        }
                    }
                    if (!flag)
                    {
                        corpus.TaggedWords[i].Tag = corpus.TaggedWords[i].Word;
                    }
                }
                else // word
                {
                    string       wordLower = corpus.TaggedWords[i].WordLower;
                    Set <string> filter    = mSuffixTree.Contains(wordLower) ? mSuffixTree.GetTags(wordLower) : null;
                    result = ProcessResult(result, filter);//???!!!
                    string goldTag = corpus.TaggedWords[i].Tag;
                    string word    = corpus.TaggedWords[i].Word;
                    string rule;
                    if (filter == null)
                    {
                        filter = Rules.ApplyTaggerRules(CreateFilterFromResult(result), word, out rule);
                    }
                    else
                    {
                        filter = Rules.ApplyTaggerRules(filter, word, out rule);
                        if (filter.Count == 0)
                        {
                            filter = Rules.ApplyTaggerRules(CreateFilterFromResult(result), word, out rule);
                        }
                    }
                    result = ProcessResult(result, filter);//???!!!
                    string predictedTag;
                    if (result.Count == 0)
                    {
                        predictedTag = Rules.GetMostFrequentTag(wordLower, filter);
                    }
                    else
                    {
                        predictedTag = result.BestClassLabel;
                    }
                    corpus.TaggedWords[i].Tag = predictedTag;
                    if (mLemmatizer != null)
                    {
                        string lemma;
                        lemma = mConsiderTags ? mLemmatizer.Lemmatize(wordLower, predictedTag) : mLemmatizer.Lemmatize(wordLower);
                        lemma = Rules.FixLemma(lemma, corpus.TaggedWords[i].Word, predictedTag);
                        if (string.IsNullOrEmpty(lemma))
                        {
                            lemma = wordLower;
                        }
                        if (xmlMode)
                        {
                            lemmaWords++;
                            if (lemma == corpus.TaggedWords[i].Lemma)
                            {
                                lemmaCorrect++;
                            }
                            if (corpus.TaggedWords[i].Lemma != null && lemma.ToLower() == corpus.TaggedWords[i].Lemma.ToLower())
                            {
                                lemmaCorrectLowercase++;
                            }
                        }
                        corpus.TaggedWords[i].Lemma = lemma;
                    }
                }
            }
            TimeSpan span = DateTime.Now - startTime;

            mLogger.Debug("Tag", "Trajanje označevanja: {0:00}:{1:00}:{2:00}.{3:000}.", span.Hours, span.Minutes, span.Seconds, span.Milliseconds);
        }
Beispiel #2
0
        public BinaryVector GenerateFeatureVector(int wordIdx, Dictionary <string, int> featureSpace, bool extendFeatureSpace, PatriciaTree suffixTree)
        {
            Utils.ThrowException((wordIdx < 0 || wordIdx >= mTaggedWords.Count) ? new ArgumentOutOfRangeException("wordIdx") : null);
            Utils.ThrowException(suffixTree == null ? new ArgumentNullException("suffixTree") : null);
            ArrayList <int> featureVector = new ArrayList <int>();

            for (int offset = -3; offset <= 3; offset++) // consider context of 3 + 1 + 3 words
            {
                int idx = wordIdx + offset;
                // *** unigrams ***
                if (idx >= 0 && idx < mTaggedWords.Count)
                {
                    AddFeature(string.Format("w({0}) {1}", offset, mTaggedWords[idx].WordLower), featureSpace, extendFeatureSpace, featureVector);
                    for (int i = 1; i <= 4; i++) // consider prefixes and suffixes of up to 4 letters
                    {
                        string prefix = GetPrefix(mTaggedWords[idx].WordLower, i);
                        AddFeature(string.Format("p{0}({1}) {2}", i, offset, prefix), featureSpace, extendFeatureSpace, featureVector);
                        string suffix = GetSuffix(mTaggedWords[idx].WordLower, i);
                        AddFeature(string.Format("s{0}({1}) {2}", i, offset, suffix), featureSpace, extendFeatureSpace, featureVector);
                    }
                    if (offset < 0) // tag is available iff offset < 0
                    {
                        AddFeature(string.Format("t({0}) {1}", offset, mTaggedWords[idx].Tag), featureSpace, extendFeatureSpace, featureVector);
                        if (mTaggedWords[idx].Tag.Length > 0)
                        {
                            AddFeature(string.Format("t1({0}) {1}", offset, mTaggedWords[idx].Tag[0]), featureSpace, extendFeatureSpace, featureVector);
                        }
                    }
                    else // tag not available; use "maybe" features and ambiguity class instead
                    {
                        string word = mTaggedWords[idx].WordLower;
                        Set <string> .ReadOnly tags = suffixTree.GetTags(word);
                        foreach (string tag in tags)
                        {
                            AddFeature(string.Format("m({0}) {1}", offset, tag), featureSpace, extendFeatureSpace, featureVector);
                            if (tag.Length > 0)
                            {
                                AddFeature(string.Format("m1({0}) {1}", offset, tag[0]), featureSpace, extendFeatureSpace, featureVector);
                            }
                        }
                        string ambiguityClass = suffixTree.GetAmbiguityClass(word);
                        AddFeature(string.Format("t({0}) {1}", offset, ambiguityClass), featureSpace, extendFeatureSpace, featureVector);
                    }
                }
            }
#if NGRAM_FEATURES
            // *** bigrams and trigrams ***
            for (int n = 2; n <= 3; n++)
            {
                for (int offset = -2; offset <= 3 - n; offset++) // consider 4 bigrams and 3 trigrams
                {
                    string   wordFeature   = string.Format("w({0},{1})", n, offset);
                    string   tagFeature    = string.Format("t({0},{1})", n, offset);
                    string[] prefixFeature = new string[4];
                    string[] suffixFeature = new string[4];
                    for (int i = 0; i < 4; i++) // consider prefixes and suffixes of up to 4 letters
                    {
                        prefixFeature[i] = string.Format("p{0}({1},{2})", i, n, offset);
                        suffixFeature[i] = string.Format("s{0}({1},{2})", i, n, offset);
                    }
                    if (wordIdx + offset >= 0 && wordIdx + offset + (n - 1) < mTaggedWords.Count)
                    {
                        for (int i = 0; i < n; i++)
                        {
                            int    idx  = wordIdx + offset + i;
                            string word = mTaggedWords[idx].WordLower;
                            wordFeature += " " + word;
                            for (int j = 0; j < 4; j++) // prefixes and suffixes
                            {
                                prefixFeature[j] += " " + GetPrefix(word, j);
                                suffixFeature[j] += " " + GetSuffix(word, j);
                            }
                            if (offset + i < 0) // tag is available iff offset + i < 0
                            {
                                tagFeature += " " + mTaggedWords[idx].Tag;
                            }
                            else // tag not available; use ambiguity class instead
                            {
                                string ambiguityClass = suffixTree.GetAmbiguityClass(word);
                                tagFeature += " " + ambiguityClass;
                            }
                        }
                        AddFeature(wordFeature, featureSpace, extendFeatureSpace, featureVector);
                        AddFeature(tagFeature, featureSpace, extendFeatureSpace, featureVector);
                        for (int i = 0; i < 4; i++) // add prefix and suffix features
                        {
                            AddFeature(prefixFeature[i], featureSpace, extendFeatureSpace, featureVector);
                            AddFeature(suffixFeature[i], featureSpace, extendFeatureSpace, featureVector);
                        }
                    }
                }
            }
#endif
            // character features
            foreach (char ch in mTaggedWords[wordIdx].Word)
            {
                // contains non-alphanum char?
                if (!char.IsLetterOrDigit(ch))
                {
                    AddFeature(string.Format("c{0}", ch), featureSpace, extendFeatureSpace, featureVector);
                }
                // contains number?
                if (char.IsDigit(ch))
                {
                    AddFeature("cd", featureSpace, extendFeatureSpace, featureVector);
                }
                // contains uppercase char?
                if (char.IsUpper(ch))
                {
                    AddFeature("cu", featureSpace, extendFeatureSpace, featureVector);
                }
            }
            // starts with capital letter?
            if (mTaggedWords[wordIdx].Word.Length > 0 && char.IsUpper(mTaggedWords[wordIdx].Word[0]))
            {
                AddFeature("cl", featureSpace, extendFeatureSpace, featureVector);
            }
            // starts with capital letter and not first word?
            if (wordIdx > 0 && !mTaggedWords[wordIdx - 1].Tag.EndsWith("<eos>") && mTaggedWords[wordIdx].Word.Length > 0 && char.IsUpper(mTaggedWords[wordIdx].Word[0]))
            {
                AddFeature("cl+", featureSpace, extendFeatureSpace, featureVector);
            }
            return(new BinaryVector(featureVector));
        }