public override string GetValue(Token token, IndexedSentence sentence)
{
int currentTokenPosition = sentence[token];
int position = currentTokenPosition + offset;
if (position >= 0 && position < sentence.Count)
return transformer.Transform(sentence[position]);
return null;
}
public override string GetValue(Token token, IndexedSentence sentence)
{
string[] values = new string[features.Length];
for (int i = 0; i < features.Length; i++)
{
values[i] = features[i].GetValue(token, sentence);
}
return(string.Join(valueDelimiter, values));
}
public override string GetValue(Token token, IndexedSentence sentence)
{
int currentTokenPosition = sentence[token];
int position = currentTokenPosition + offset;
if (position >= 0 && position < sentence.Count)
{
return(transformer.Transform(sentence[position]));
}
return(null);
}
public override string GetValue(Token token, IndexedSentence sentence)
{
int position = sentence[token];
for (int i = position - 1; i >= 0; i--)
{
if (sentence[i].PredictedTag != null && (sentence[i].PredictedTag.PartOfSpeech == 'n' || sentence[i].PredictedTag.PartOfSpeech == 'p') && (sentence[i].PredictedTag.Msd[4] != 'g' && sentence[i].PredictedTag.Msd[4] != 'l'))
{
return(sentence[i].PredictedTag.Msd);
}
}
return(null);
}
public override string GetValue(Token token, IndexedSentence sentence)
{
int position = sentence[token];
for (int i = position - 1, prevPos = -1; i >= 0; i--)
{
if (sentence[i].PredictedTag != null && sentence[i].PredictedTag.PartOfSpeech == 'n')
{
prevPos = i;
}
else
{
if (prevPos != -1 && sentence[prevPos].PredictedTag != null)
{
return(sentence[prevPos].PredictedTag.Msd);
}
}
}
return(null);
}
public abstract string GetValue(Token token, IndexedSentence sentence);
public override string GetValue(Token token, IndexedSentence sentence)
{
int position = sentence[token];
for (int i = position - 1, prevPos = -1; i >= 0; i--)
{
if (sentence[i].PredictedTag != null && sentence[i].PredictedTag.PartOfSpeech == 'n')
{
prevPos = i;
}
else
{
if (prevPos != -1 && sentence[prevPos].PredictedTag != null)
{
return sentence[prevPos].PredictedTag.Msd;
}
}
}
return null;
}
public override string GetValue(Token token, IndexedSentence sentence)
{
int position = sentence[token];
for (int i = position - 1; i >= 0; i--)
{
if (sentence[i].PredictedTag != null && (sentence[i].PredictedTag.PartOfSpeech == 'n' || sentence[i].PredictedTag.PartOfSpeech == 'p') && (sentence[i].PredictedTag.Msd[4] != 'g' && sentence[i].PredictedTag.Msd[4] != 'l'))
{
return sentence[i].PredictedTag.Msd;
}
}
return null;
}
public override string GetValue(Token token, IndexedSentence sentence)
{
return template.GetValue(token, sentence);
}
public override string GetValue(Token token, IndexedSentence sentence)
{
string[] values = new string[features.Length];
for (int i = 0; i < features.Length; i++)
values[i] = features[i].GetValue(token, sentence);
return string.Join(valueDelimiter, values);
}
public override string GetValue(Token token, IndexedSentence sentence)
{
return transformer.Transform(token);
}
public override string GetValue(Token token, IndexedSentence sentence)
{
return(transformer.Transform(token));
}
public override string GetValue(Token token, IndexedSentence sentence)
{
return(template.GetValue(token, sentence));
}
private Features GetFeatures(List<FeatureTemplate> list, Token token, IndexedSentence sentence, bool? local = null)
{
Features features = new Features(list.Count);
foreach (FeatureTemplate template in list)
{
if (local == null || local == template.IsLocal)
{
features.Add(template.Name, template.GetValue(token, sentence));
}
}
return features;
}
public void Tag(IEnumerable<Sentence> sentences)
{
if (perceptronMsd == null)
throw new InvalidOperationException("Model not loaded.");
Parallel.ForEach(sentences, (sentence, index) =>
{
IndexedSentence normalizedSentence =
new IndexedSentence((Reverse ? (sentence as IEnumerable<Token>).Reverse() : sentence).Select(t => Normalize(t)));
foreach (Token token in normalizedSentence)
{
Tag[] possibleTags = token.PossibleTags;
if (possibleTags == null || possibleTags.Length == 0)
possibleTags = perceptronMsd.Tags.ToArray();
if (possibleTags.Length == 1)
{
token.PredictedTag = possibleTags[0];
}
else
{
double? maxScoreMsd = null;
Tag bestMsd = null;
Features localFeaturesTag = GetFeatures(featureTemplatesTag, token, normalizedSentence, true);
foreach (Tag tag in possibleTags)
{
Tag tagMsd = new Tag(tag.Msd);
token.PredictedTag = tagMsd;
Features featuresTag = GetFeatures(featureTemplatesTag, token, normalizedSentence, false);
featuresTag.AddRange(localFeaturesTag);
double scoreMsd = perceptronMsd.Score(featuresTag, tagMsd);
if (maxScoreMsd == null || scoreMsd > maxScoreMsd.Value)
{
maxScoreMsd = scoreMsd;
bestMsd = tagMsd;
}
}
double? maxScoreLemma = null;
string bestLemma = null;
token.PredictedTag = bestMsd;
Features localFeaturesLemma = GetFeatures(featureTemplatesLemma, token, normalizedSentence, true);
foreach (Tag tag in possibleTags)
{
if (tag.Msd != bestMsd.Msd || tag.Lemma == null)
continue;
token.PredictedTag = new Tag(token.CorrectTag.Msd, tag.Lemma);
Features featuresLemma = GetFeatures(featureTemplatesLemma, token, normalizedSentence, false);
featuresLemma.AddRange(localFeaturesLemma);
double scoreLemma = perceptronLemma.Score(featuresLemma, tag.Lemma);
if (maxScoreLemma == null || scoreLemma > maxScoreLemma.Value)
{
maxScoreLemma = scoreLemma;
bestLemma = tag.Lemma;
}
}
token.PredictedTag = new Tag(bestMsd.Msd, bestLemma);
}
int pos = normalizedSentence[token];
if (Reverse) pos = normalizedSentence.Count - pos - 1;
Token original = sentence[pos];
original.PredictedTag = token.PredictedTag;
Unnormalize(original);
}
});
}
