public override string GetValue(Token token, IndexedSentence sentence) { int currentTokenPosition = sentence[token]; int position = currentTokenPosition + offset; if (position >= 0 && position < sentence.Count) return transformer.Transform(sentence[position]); return null; }
public override string GetValue(Token token, IndexedSentence sentence) { string[] values = new string[features.Length]; for (int i = 0; i < features.Length; i++) { values[i] = features[i].GetValue(token, sentence); } return(string.Join(valueDelimiter, values)); }
public override string GetValue(Token token, IndexedSentence sentence) { int currentTokenPosition = sentence[token]; int position = currentTokenPosition + offset; if (position >= 0 && position < sentence.Count) { return(transformer.Transform(sentence[position])); } return(null); }
public override string GetValue(Token token, IndexedSentence sentence) { int position = sentence[token]; for (int i = position - 1; i >= 0; i--) { if (sentence[i].PredictedTag != null && (sentence[i].PredictedTag.PartOfSpeech == 'n' || sentence[i].PredictedTag.PartOfSpeech == 'p') && (sentence[i].PredictedTag.Msd[4] != 'g' && sentence[i].PredictedTag.Msd[4] != 'l')) { return(sentence[i].PredictedTag.Msd); } } return(null); }
public override string GetValue(Token token, IndexedSentence sentence) { int position = sentence[token]; for (int i = position - 1, prevPos = -1; i >= 0; i--) { if (sentence[i].PredictedTag != null && sentence[i].PredictedTag.PartOfSpeech == 'n') { prevPos = i; } else { if (prevPos != -1 && sentence[prevPos].PredictedTag != null) { return(sentence[prevPos].PredictedTag.Msd); } } } return(null); }
public abstract string GetValue(Token token, IndexedSentence sentence);
public override string GetValue(Token token, IndexedSentence sentence) { int position = sentence[token]; for (int i = position - 1, prevPos = -1; i >= 0; i--) { if (sentence[i].PredictedTag != null && sentence[i].PredictedTag.PartOfSpeech == 'n') { prevPos = i; } else { if (prevPos != -1 && sentence[prevPos].PredictedTag != null) { return sentence[prevPos].PredictedTag.Msd; } } } return null; }
public override string GetValue(Token token, IndexedSentence sentence) { int position = sentence[token]; for (int i = position - 1; i >= 0; i--) { if (sentence[i].PredictedTag != null && (sentence[i].PredictedTag.PartOfSpeech == 'n' || sentence[i].PredictedTag.PartOfSpeech == 'p') && (sentence[i].PredictedTag.Msd[4] != 'g' && sentence[i].PredictedTag.Msd[4] != 'l')) { return sentence[i].PredictedTag.Msd; } } return null; }
public override string GetValue(Token token, IndexedSentence sentence) { return template.GetValue(token, sentence); }
public override string GetValue(Token token, IndexedSentence sentence) { string[] values = new string[features.Length]; for (int i = 0; i < features.Length; i++) values[i] = features[i].GetValue(token, sentence); return string.Join(valueDelimiter, values); }
public override string GetValue(Token token, IndexedSentence sentence) { return transformer.Transform(token); }
public override string GetValue(Token token, IndexedSentence sentence) { return(transformer.Transform(token)); }
public override string GetValue(Token token, IndexedSentence sentence) { return(template.GetValue(token, sentence)); }
private Features GetFeatures(List<FeatureTemplate> list, Token token, IndexedSentence sentence, bool? local = null) { Features features = new Features(list.Count); foreach (FeatureTemplate template in list) { if (local == null || local == template.IsLocal) { features.Add(template.Name, template.GetValue(token, sentence)); } } return features; }
public void Tag(IEnumerable<Sentence> sentences) { if (perceptronMsd == null) throw new InvalidOperationException("Model not loaded."); Parallel.ForEach(sentences, (sentence, index) => { IndexedSentence normalizedSentence = new IndexedSentence((Reverse ? (sentence as IEnumerable<Token>).Reverse() : sentence).Select(t => Normalize(t))); foreach (Token token in normalizedSentence) { Tag[] possibleTags = token.PossibleTags; if (possibleTags == null || possibleTags.Length == 0) possibleTags = perceptronMsd.Tags.ToArray(); if (possibleTags.Length == 1) { token.PredictedTag = possibleTags[0]; } else { double? maxScoreMsd = null; Tag bestMsd = null; Features localFeaturesTag = GetFeatures(featureTemplatesTag, token, normalizedSentence, true); foreach (Tag tag in possibleTags) { Tag tagMsd = new Tag(tag.Msd); token.PredictedTag = tagMsd; Features featuresTag = GetFeatures(featureTemplatesTag, token, normalizedSentence, false); featuresTag.AddRange(localFeaturesTag); double scoreMsd = perceptronMsd.Score(featuresTag, tagMsd); if (maxScoreMsd == null || scoreMsd > maxScoreMsd.Value) { maxScoreMsd = scoreMsd; bestMsd = tagMsd; } } double? maxScoreLemma = null; string bestLemma = null; token.PredictedTag = bestMsd; Features localFeaturesLemma = GetFeatures(featureTemplatesLemma, token, normalizedSentence, true); foreach (Tag tag in possibleTags) { if (tag.Msd != bestMsd.Msd || tag.Lemma == null) continue; token.PredictedTag = new Tag(token.CorrectTag.Msd, tag.Lemma); Features featuresLemma = GetFeatures(featureTemplatesLemma, token, normalizedSentence, false); featuresLemma.AddRange(localFeaturesLemma); double scoreLemma = perceptronLemma.Score(featuresLemma, tag.Lemma); if (maxScoreLemma == null || scoreLemma > maxScoreLemma.Value) { maxScoreLemma = scoreLemma; bestLemma = tag.Lemma; } } token.PredictedTag = new Tag(bestMsd.Msd, bestLemma); } int pos = normalizedSentence[token]; if (Reverse) pos = normalizedSentence.Count - pos - 1; Token original = sentence[pos]; original.PredictedTag = token.PredictedTag; Unnormalize(original); } }); }