/// <summary>
/// Adds the appropriate features for the token at the specified index with the
/// specified array of previous outcomes to the specified list of features.
/// </summary>
/// <param name="features">The list of features to be added to.</param>
/// <param name="tokens">The tokens of the sentence or other text unit being processed.</param>
/// <param name="index">The index of the token which is currently being processed.</param>
/// <param name="previousOutcomes">The outcomes for the tokens prior to the specified index.</param>
public override void CreateFeatures(List <string> features, string[] tokens, int index, string[] previousOutcomes)
{
var wordShape = FeatureGeneratorUtil.TokenFeature(tokens[index]);
var wordClasses = BrownTokenClasses.GetWordClasses(tokens[index], brownLexicon);
features.AddRange(wordClasses.Select(brownClass => "c,browncluster=" + wordShape + "," + brownClass));
}
/// <summary>
/// Adds the appropriate features for the token at the specified index with the
/// specified array of previous outcomes to the specified list of features.
/// </summary>
/// <param name="features">The list of features to be added to.</param>
/// <param name="tokens">The tokens of the sentence or other text unit being processed.</param>
/// <param name="index">The index of the token which is currently being processed.</param>
/// <param name="previousOutcomes">The outcomes for the tokens prior to the specified index.</param>
public override void CreateFeatures(List <string> features, string[] tokens, int index, string[] previousOutcomes)
{
var wordClasses = BrownTokenClasses.GetWordClasses(tokens[index], brownLexicon);
if (index > 0)
{
var prevWordClasses = BrownTokenClasses.GetWordClasses(tokens[index - 1], brownLexicon);
for (var i = 0; i < wordClasses.Count && i < prevWordClasses.Count; i++)
{
features.Add("pbrowncluster,browncluster=" + prevWordClasses[i] + "," + wordClasses[i]);
}
}
if (index + 1 < tokens.Length)
{
var nextWordClasses = BrownTokenClasses.GetWordClasses(tokens[index + 1], brownLexicon);
for (var i = 0; i < wordClasses.Count && i < nextWordClasses.Count; i++)
{
features.Add("browncluster,nbrowncluster=" + wordClasses[i] + "," + nextWordClasses[i]);
}
}
}
/// <summary>
/// Adds the appropriate features for the token at the specified index with the
/// specified array of previous outcomes to the specified list of features.
/// </summary>
/// <param name="features">The list of features to be added to.</param>
/// <param name="tokens">The tokens of the sentence or other text unit being processed.</param>
/// <param name="index">The index of the token which is currently being processed.</param>
/// <param name="previousOutcomes">The outcomes for the tokens prior to the specified index.</param>
public override void CreateFeatures(List <string> features, string[] tokens, int index, string[] previousOutcomes)
{
var wordClasses = BrownTokenClasses.GetWordClasses(tokens[index], brownLexicon);
features.AddRange(wordClasses.Select(wordClass => "browncluster=" + wordClass));
}