public virtual void Annotate(Annotation annotation)
{
if (annotation.ContainsKey(typeof(CoreAnnotations.SentencesAnnotation)))
{
// TODO: parallelize
IList <ICoreMap> sentences = annotation.Get(typeof(CoreAnnotations.SentencesAnnotation));
foreach (ICoreMap sentence in sentences)
{
Tree binarized = sentence.Get(typeof(TreeCoreAnnotations.BinarizedTreeAnnotation));
if (binarized == null)
{
throw new AssertionError("Binarized sentences not built by parser");
}
Tree collapsedUnary = transformer.TransformTree(binarized);
SentimentCostAndGradient scorer = new SentimentCostAndGradient(model, null);
scorer.ForwardPropagateTree(collapsedUnary);
sentence.Set(typeof(SentimentCoreAnnotations.SentimentAnnotatedTree), collapsedUnary);
int sentiment = RNNCoreAnnotations.GetPredictedClass(collapsedUnary);
sentence.Set(typeof(SentimentCoreAnnotations.SentimentClass), SentimentUtils.SentimentString(model, sentiment));
Tree tree = sentence.Get(typeof(TreeCoreAnnotations.TreeAnnotation));
if (tree != null)
{
collapsedUnary.SetSpans();
// map the sentiment annotations onto the tree
IDictionary <IntPair, string> spanSentiment = Generics.NewHashMap();
foreach (Tree bt in collapsedUnary)
{
IntPair p = bt.GetSpan();
int sen = RNNCoreAnnotations.GetPredictedClass(bt);
string sentStr = SentimentUtils.SentimentString(model, sen);
if (!spanSentiment.Contains(p))
{
// we'll take the first = highest one discovered
spanSentiment[p] = sentStr;
}
}
if (((CoreLabel)tree.Label()).ContainsKey(typeof(CoreAnnotations.SpanAnnotation)))
{
throw new InvalidOperationException("This code assumes you don't have SpanAnnotation");
}
tree.SetSpans();
foreach (Tree t in tree)
{
IntPair p = t.GetSpan();
string str = spanSentiment[p];
if (str != null)
{
CoreLabel cl = (CoreLabel)t.Label();
cl.Set(typeof(SentimentCoreAnnotations.SentimentClass), str);
cl.Remove(typeof(CoreAnnotations.SpanAnnotation));
}
}
}
}
}
else
{
throw new Exception("unable to find sentences in: " + annotation);
}
}
public void ProperNounTests_BasicFunctionality()
{
// Arrange: Create list of tokens
List <string> tokens = new List <string> {
"FirstToken", "Brian", "break", "Brian", "break", "Plymouth", "break", "Campari", "break", "Carpano", "Antica", "cocktail"
};
List <string> expectedTokens = new List <string> {
"Brian", "Plymouth", "Campari", "Carpano Antica"
};
// Act: Transform list of tokens into proper noun dictionary
Dictionary <string, int> properNounTokens = SentimentUtils.GetProperNouns(tokens);
// Assert: Check to see if dictionary has what we expect
// Shouldn't contain "FirstToken", first token is ignored
Assert.IsFalse(properNounTokens.ContainsKey("FirstToken"));
// Number of keys in dictionary should be equal to # of distinct proper nouns, and compound proper nouns
Assert.AreEqual(expectedTokens.Count, properNounTokens.Keys.Count);
// Dictionary should have a key for all expected values
expectedTokens.ForEach(s => Assert.IsTrue(properNounTokens.ContainsKey(s)));
// "Brian" should have 2 as a value (it appears twice)
Assert.AreEqual(2, properNounTokens["Brian"]);
}
public void ProperNounTests_IgnoredProperNounWithinCompoundProperNoun()
{
// Arrange: Create list of tokens that resemble crappy sentences
List <string> tokens = new List <string> {
"FirstToken", "Brian", "Monday", "Clark", "that's", "not", "me."
};
// Act: Transform list of tokens into proper noun dictionary
Dictionary <string, int> properNounTokens = SentimentUtils.GetProperNouns(tokens);
// Assert: Check to see if dictionary has what we expect
// Should contain 2 token: "Brian", "Clark",
Assert.AreEqual(2, properNounTokens.Keys.Count);
// Shouldn't contain "Monday"
Assert.IsFalse(properNounTokens.ContainsKey("Monday"));
// Shouldn't contain "Brian Monday Clark"
Assert.IsFalse(properNounTokens.ContainsKey("Brian Monday Clark"));
// See if dictionary contains only legit proper noun: Brian, Clark
Assert.IsTrue(properNounTokens.ContainsKey("Brian"));
Assert.IsTrue(properNounTokens.ContainsKey("Clark"));
}
public void ProperNounTests_StartOfSentence()
{
// Arrange: Create list of tokens that resemble crappy sentences
List <string> tokens = new List <string> {
"Yes", "that's", "me.", "That", "is", "my", "Negroni."
};
// Act: Transform list of tokens into proper noun dictionary
Dictionary <string, int> properNounTokens = SentimentUtils.GetProperNouns(tokens);
// Assert: Check to see if dictionary has what we expect
// Shouldn't contain "Yes", first token is ignored
Assert.IsFalse(properNounTokens.ContainsKey("Yes"));
// Should contain 1 token: "Negroni",
Assert.AreEqual(1, properNounTokens.Keys.Count);
// See if dictionary contains only legit proper noun: Negroni
Assert.IsTrue(properNounTokens.ContainsKey("Negroni"));
}
public void ProperNounTests_CompoundToken()
{
// Arrange: Create list of tokens that are made up one or more capitalized words
List <string> tokens = new List <string> {
"FirstToken", "Software", "Developer", "Brian", "break", "Russian", "Agent", "Donald", "Trump", "break", "New", "York"
};
// Act: Transform list of tokens into proper noun dictionary
Dictionary <string, int> properNounTokens = SentimentUtils.GetProperNouns(tokens);
// Assert: Check to see if dictionary has what we expect
// Shouldn't contain "FirstToken", first token is ignored
Assert.IsFalse(properNounTokens.ContainsKey("FirstToken"));
// Should contain 3 tokens: "Software Developer Brian, Russian Agent Donald Trump, and New York",
Assert.AreEqual(3, properNounTokens.Keys.Count);
// See if dictionary contains these complex proper nouns properly grouped into keys
Assert.IsTrue(properNounTokens.ContainsKey("Software Developer Brian"));
Assert.IsTrue(properNounTokens.ContainsKey("Russian Agent Donald Trump"));
Assert.IsTrue(properNounTokens.ContainsKey("New York"));
}
