/// <summary> /// Adds the appropriate features for the token at the specified index with the /// specified array of previous outcomes to the specified list of features. /// </summary> /// <param name="features">The list of features to be added to.</param> /// <param name="tokens">The tokens of the sentence or other text unit being processed.</param> /// <param name="index">The index of the token which is currently being processed.</param> /// <param name="previousOutcomes">The outcomes for the tokens prior to the specified index.</param> public override void CreateFeatures(List <string> features, string[] tokens, int index, string[] previousOutcomes) { var wordShape = FeatureGeneratorUtil.TokenFeature(tokens[index]); var wordClasses = BrownTokenClasses.GetWordClasses(tokens[index], brownLexicon); features.AddRange(wordClasses.Select(brownClass => "c,browncluster=" + wordShape + "," + brownClass)); }
/// <summary> /// Adds the appropriate features for the token at the specified index with the /// specified array of previous outcomes to the specified list of features. /// </summary> /// <param name="features">The list of features to be added to.</param> /// <param name="tokens">The tokens of the sentence or other text unit being processed.</param> /// <param name="index">The index of the token which is currently being processed.</param> /// <param name="previousOutcomes">The outcomes for the tokens prior to the specified index.</param> public override void CreateFeatures(List <string> features, string[] tokens, int index, string[] previousOutcomes) { var wordClasses = BrownTokenClasses.GetWordClasses(tokens[index], brownLexicon); if (index > 0) { var prevWordClasses = BrownTokenClasses.GetWordClasses(tokens[index - 1], brownLexicon); for (var i = 0; i < wordClasses.Count && i < prevWordClasses.Count; i++) { features.Add("pbrowncluster,browncluster=" + prevWordClasses[i] + "," + wordClasses[i]); } } if (index + 1 < tokens.Length) { var nextWordClasses = BrownTokenClasses.GetWordClasses(tokens[index + 1], brownLexicon); for (var i = 0; i < wordClasses.Count && i < nextWordClasses.Count; i++) { features.Add("browncluster,nbrowncluster=" + wordClasses[i] + "," + nextWordClasses[i]); } } }
/// <summary> /// Adds the appropriate features for the token at the specified index with the /// specified array of previous outcomes to the specified list of features. /// </summary> /// <param name="features">The list of features to be added to.</param> /// <param name="tokens">The tokens of the sentence or other text unit being processed.</param> /// <param name="index">The index of the token which is currently being processed.</param> /// <param name="previousOutcomes">The outcomes for the tokens prior to the specified index.</param> public override void CreateFeatures(List <string> features, string[] tokens, int index, string[] previousOutcomes) { var wordClasses = BrownTokenClasses.GetWordClasses(tokens[index], brownLexicon); features.AddRange(wordClasses.Select(wordClass => "browncluster=" + wordClass)); }