public override float Score(IntTaggedWord iTW, int loc, string word, string featureSpec) { int wordId = iTW.Word(); int tagId = iTW.Tag(); // Force 1-best path to go through the boundary symbol // (deterministic tagging) int boundaryId = wordIndex.IndexOf(LexiconConstants.Boundary); int boundaryTagId = tagIndex.IndexOf(LexiconConstants.BoundaryTag); if (wordId == boundaryId && tagId == boundaryTagId) { return(0.0f); } // Morphological features string tag = tagIndex.Get(iTW.Tag()); Pair <string, string> lemmaMorph = MorphoFeatureSpecification.SplitMorphString(word, featureSpec); string lemma = lemmaMorph.First(); int lemmaId = wordIndex.IndexOf(lemma); string richMorphTag = lemmaMorph.Second(); string reducedMorphTag = morphoSpec.StrToFeatures(richMorphTag).ToString().Trim(); reducedMorphTag = reducedMorphTag.Length == 0 ? NoMorphAnalysis : reducedMorphTag; int morphId = morphIndex.AddToIndex(reducedMorphTag); // Score the factors and create the rule score p_W_T double p_W_Tf = Math.Log(ProbWordTag(word, loc, wordId, tagId)); // double p_L_T = Math.log(probLemmaTag(word, loc, tagId, lemmaId)); double p_L_T = 0.0; double p_M_T = Math.Log(ProbMorphTag(tagId, morphId)); double p_W_T = p_W_Tf + p_L_T + p_M_T; // String tag = tagIndex.get(tagId); // Filter low probability taggings return(p_W_T > -100.0 ? (float)p_W_T : float.NegativeInfinity); }
public virtual float Score(IntTaggedWord iTW, int loc, string word, string featureSpec) { EnsureProbs(iTW.Word()); double max = Counters.Max(logProbs); double score = logProbs.GetCount(iTW.TagString(tagIndex)); if (score > max - iteratorCutoffFactor) { return((float)score); } else { return(float.NegativeInfinity); } }
/// <summary>Adds the tagging with count to the data structures in this Lexicon.</summary> protected internal virtual void AddTagging(bool seen, IntTaggedWord itw, double count) { if (seen) { seenCounter.IncrementCount(itw, count); if (itw.Tag() == nullTag) { words.Add(itw); } else { if (itw.Word() == nullWord) { tags.Add(itw); } } } else { // rules.add(itw); uwModel.AddTagging(seen, itw, count); } }