public override void PopulatePredictedLabels(IList <Tree> trees)
{
if (trees.Count != this.predicted.Count)
{
throw new ArgumentException("Number of gold and predicted trees not equal!");
}
for (int i = 0; i < trees.Count; i++)
{
IEnumerator <Tree> goldTree = trees[i].GetEnumerator();
IEnumerator <Tree> predictedTree = this.predicted[i].GetEnumerator();
while (goldTree.MoveNext() || predictedTree.MoveNext())
{
Tree goldNode = goldTree.Current;
Tree predictedNode = predictedTree.Current;
if (goldNode == null || predictedNode == null)
{
throw new ArgumentException("Trees not of equal length");
}
if (goldNode.IsLeaf())
{
continue;
}
CoreLabel label = (CoreLabel)goldNode.Label();
label.Set(typeof(RNNCoreAnnotations.PredictedClass), RNNCoreAnnotations.GetPredictedClass(predictedNode));
}
}
}
public Tvn.Cosine.Text.Nlp.Document Process(string text)
{
lock (syncLock)
{
var sentences = new List <Sentence>();
var tokens = new List <Token>();
var annotation = new Annotation(text);
pipeline.annotate(annotation);
var sentencesStanford = (java.util.List)annotation.get(typeof(CoreAnnotations.SentencesAnnotation));
if (sentencesStanford != null && sentencesStanford.size() > 0)
{
for (int i = 0; i < sentencesStanford.size(); ++i)
{
var sentence = (CoreMap)sentencesStanford.get(i);
var sentiment = (string)sentence.get(typeof(SentimentClass));
var tree = (Tree)sentence.get(typeof(SentimentAnnotatedTree));
var score = RNNCoreAnnotations.getPredictions(tree).getMatrix().getData();
var sentDic = new Dictionary <Sentiment, double>();
var tokensSentence = getTokens((java.util.List)sentence.get(typeof(CoreAnnotations.TokensAnnotation)));
var ner = getNamedEntities(tokensSentence);
for (uint s = 0; s < score.Length; ++s)
{
sentDic[new Sentiment(s, s.ToString())] = score[s];
}
sentences.Add(new Sentence(sentence.ToString(), tokensSentence, ner, new Sentiment(0, sentiment), sentDic));
tokens.AddRange(tokensSentence);
}
}
return(new Tvn.Cosine.Text.Nlp.Document(text, sentences, tokens));
}
}
protected internal virtual void CountTree(Tree tree)
{
if (tree.IsLeaf())
{
return;
}
foreach (Tree child in tree.Children())
{
CountTree(child);
}
int gold = RNNCoreAnnotations.GetGoldClass(tree);
int predicted = RNNCoreAnnotations.GetPredictedClass(tree);
if (gold >= 0)
{
if (gold.Equals(predicted))
{
labelsCorrect++;
}
else
{
labelsIncorrect++;
}
labelConfusion[gold][predicted]++;
}
}
public static int GetSentiment(string review, dynamic pipeline)
{
int mainSentiment = 0;
if (review != null && review.Length > 0)
{
int longest = 0;
Annotation annotation = pipeline.process(review);
var Sentence = annotation.get(typeof(CoreAnnotations.SentencesAnnotation)) as ArrayList;
foreach (CoreMap sen in Sentence)
{
Tree tree = (Tree)sen.get(typeof(SentimentCoreAnnotations.SentimentAnnotatedTree));
Console.WriteLine(tree);
int sentiment = RNNCoreAnnotations.getPredictedClass(tree);
string partText = sen.ToString();
if (partText.Length > longest)
{
mainSentiment = sentiment;
longest = partText.Length;
}
}
}
return(mainSentiment);
}
protected internal virtual int CountLengthAccuracy(Tree tree)
{
if (tree.IsLeaf())
{
return(0);
}
int gold = RNNCoreAnnotations.GetGoldClass(tree);
int predicted = RNNCoreAnnotations.GetPredictedClass(tree);
int length;
if (tree.IsPreTerminal())
{
length = 1;
}
else
{
length = 0;
foreach (Tree child in tree.Children())
{
length += CountLengthAccuracy(child);
}
}
if (gold >= 0)
{
if (gold.Equals(predicted))
{
lengthLabelsCorrect.IncrementCount(length);
}
else
{
lengthLabelsIncorrect.IncrementCount(length);
}
}
return(length);
}
public virtual void Annotate(Annotation annotation)
{
if (annotation.ContainsKey(typeof(CoreAnnotations.SentencesAnnotation)))
{
// TODO: parallelize
IList <ICoreMap> sentences = annotation.Get(typeof(CoreAnnotations.SentencesAnnotation));
foreach (ICoreMap sentence in sentences)
{
Tree binarized = sentence.Get(typeof(TreeCoreAnnotations.BinarizedTreeAnnotation));
if (binarized == null)
{
throw new AssertionError("Binarized sentences not built by parser");
}
Tree collapsedUnary = transformer.TransformTree(binarized);
SentimentCostAndGradient scorer = new SentimentCostAndGradient(model, null);
scorer.ForwardPropagateTree(collapsedUnary);
sentence.Set(typeof(SentimentCoreAnnotations.SentimentAnnotatedTree), collapsedUnary);
int sentiment = RNNCoreAnnotations.GetPredictedClass(collapsedUnary);
sentence.Set(typeof(SentimentCoreAnnotations.SentimentClass), SentimentUtils.SentimentString(model, sentiment));
Tree tree = sentence.Get(typeof(TreeCoreAnnotations.TreeAnnotation));
if (tree != null)
{
collapsedUnary.SetSpans();
// map the sentiment annotations onto the tree
IDictionary <IntPair, string> spanSentiment = Generics.NewHashMap();
foreach (Tree bt in collapsedUnary)
{
IntPair p = bt.GetSpan();
int sen = RNNCoreAnnotations.GetPredictedClass(bt);
string sentStr = SentimentUtils.SentimentString(model, sen);
if (!spanSentiment.Contains(p))
{
// we'll take the first = highest one discovered
spanSentiment[p] = sentStr;
}
}
if (((CoreLabel)tree.Label()).ContainsKey(typeof(CoreAnnotations.SpanAnnotation)))
{
throw new InvalidOperationException("This code assumes you don't have SpanAnnotation");
}
tree.SetSpans();
foreach (Tree t in tree)
{
IntPair p = t.GetSpan();
string str = spanSentiment[p];
if (str != null)
{
CoreLabel cl = (CoreLabel)t.Label();
cl.Set(typeof(SentimentCoreAnnotations.SentimentClass), str);
cl.Remove(typeof(CoreAnnotations.SpanAnnotation));
}
}
}
}
}
else
{
throw new Exception("unable to find sentences in: " + annotation);
}
}
public static void SetPredictedLabels(Tree tree)
{
if (tree.IsLeaf())
{
return;
}
foreach (Tree child in tree.Children())
{
SetPredictedLabels(child);
}
tree.Label().SetValue(int.ToString(RNNCoreAnnotations.GetPredictedClass(tree)));
}
public SentimentActor()
{
Receive<string>(i =>
{
var r = JsonConvert.DeserializeObject<SharedMessages.SentimentRequest>(i);
if (r != null)
{
var res = new SharedMessages.SentimentResponse()
{
id = r.id,
feed = r.feed,
section = r.section
};
foreach (var line in r.linesToProcess)
{
var annotation = new Annotation(line);
Program.pipeline.annotate(annotation);
var sentences = annotation.get(typeof(CoreAnnotations.SentencesAnnotation)) as java.util.ArrayList;
// For each sentence in sentences, annotate and return the sentiment value
foreach (var s in sentences)
{
var sentence = s as Annotation;
var sentenceTree = sentence.get(typeof(SentimentCoreAnnotations.SentimentAnnotatedTree)) as Tree;
var sentiment = RNNCoreAnnotations.getPredictedClass(sentenceTree);
var preds = RNNCoreAnnotations.getPredictions(sentenceTree);
var sent = "";
if (sentiment == 0)
sent = "Negative";
else if (sentiment == 1)
sent = "Somewhat negative";
else if (sentiment == 2)
sent = "Neutral";
else if (sentiment == 3)
sent = "Somewhat positive";
else if (sentiment == 4)
sent = "Positive";
res.results.Add(sentiment);
}
}
Sender.Tell("sent:" + JsonConvert.SerializeObject(res));
}
});
}
protected internal virtual void CountRoot(Tree tree)
{
int gold = RNNCoreAnnotations.GetGoldClass(tree);
int predicted = RNNCoreAnnotations.GetPredictedClass(tree);
if (gold >= 0)
{
if (gold.Equals(predicted))
{
rootLabelsCorrect++;
}
else
{
rootLabelsIncorrect++;
}
rootLabelConfusion[gold][predicted]++;
}
}
public Emotion GetEmotion(string text)
{
var annotation = new Annotation(text);
stanfordNLP.annotate(annotation);
return
((annotation.get(sentencesAnnotationClassName) as ArrayList)
.toArray().Select(ParseEmotion).FirstOrDefault());
Emotion ParseEmotion(object s)
{
var sentence = s as Annotation;
var sentenceTree = sentence.get(emotionAnnotationTreeClassName) as Tree;
var emotion = RNNCoreAnnotations.getPredictedClass(sentenceTree);
return(new Emotion(emotion));
}
}
public static string findSentiment(String line)
{
// // Path to models extracted from `stanford-parser-3.5.1-models.jar`
// var jarRoot = @"..\..\..\..\paket-files\nlp.stanford.edu\stanford-parser-full-2015-01-30\models\";
// var modelsDirectory = jarRoot + @"\edu\stanford\nlp\models";
//var modelsDirectory = @"\edu\stanford\nlp\models";
var jarRoot = @"..\..\src\stanford-corenlp\";
//var modelsDirectory = jarRoot + @"\edu\stanford\nlp\models";
// Loading english PCFG parser from file
//var lp = LexicalizedParser.loadModel(modelsDirectory + @"\lexparser\englishPCFG.ser.gz");
//try
{
Properties prop = new Properties();
prop.setProperty("annotators", "tokenize, ssplit, parse, sentiment");
var curDir = Environment.CurrentDirectory;
System.IO.Directory.SetCurrentDirectory(jarRoot);
StanfordCoreNLP pipeline = new StanfordCoreNLP(prop);
System.IO.Directory.SetCurrentDirectory(curDir);
String[] polarity = { "Very Negative", "Negative", "Neutral", "Positive", "Very Positive" };
int score = 0;
if ((line != null) && (line.Length > 0))
{
Annotation annotation = new Annotation(line);
pipeline.annotate(annotation);
foreach (CoreMap sent in (dynamic)annotation.get(new CoreAnnotations.SentencesAnnotation().getClass()))
{
Tree tree = (Tree)sent.get(new SentimentCoreAnnotations.AnnotatedTree().getClass());
score = RNNCoreAnnotations.getPredictedClass(tree);
//Console.WriteLine("The polarity of the satement is "+polarity[score]);
}
}
return(polarity[score]);
}
//catch { return "nothing"; }
}
// static methods
/// <summary>
/// Sets the labels on the tree (except the leaves) to be the integer
/// value of the sentiment prediction.
/// </summary>
/// <remarks>
/// Sets the labels on the tree (except the leaves) to be the integer
/// value of the sentiment prediction. Makes it easy to print out
/// with Tree.toString()
/// </remarks>
private static void SetSentimentLabels(Tree tree)
{
if (tree.IsLeaf())
{
return;
}
foreach (Tree child in tree.Children())
{
SetSentimentLabels(child);
}
ILabel label = tree.Label();
if (!(label is CoreLabel))
{
throw new ArgumentException("Required a tree with CoreLabels");
}
CoreLabel cl = (CoreLabel)label;
cl.SetValue(int.ToString(RNNCoreAnnotations.GetPredictedClass(tree)));
}
/// <summary>Outputs the scores from the tree.</summary>
/// <remarks>
/// Outputs the scores from the tree. Counts the tree nodes the
/// same as setIndexLabels.
/// </remarks>
private static int OutputTreeScores(TextWriter @out, Tree tree, int index)
{
if (tree.IsLeaf())
{
return(index);
}
@out.Write(" " + index + ':');
SimpleMatrix vector = RNNCoreAnnotations.GetPredictions(tree);
for (int i = 0; i < vector.GetNumElements(); ++i)
{
@out.Write(" " + Nf.Format(vector.Get(i)));
}
@out.WriteLine();
index++;
foreach (Tree child in tree.Children())
{
index = OutputTreeScores(@out, child, index);
}
return(index);
}
private static double SumError(Tree tree)
{
if (tree.IsLeaf())
{
return(0.0);
}
else
{
if (tree.IsPreTerminal())
{
return(RNNCoreAnnotations.GetPredictionError(tree));
}
else
{
double error = 0.0;
foreach (Tree child in tree.Children())
{
error += SumError(child);
}
return(RNNCoreAnnotations.GetPredictionError(tree) + error);
}
}
}
private void BgWorkerClean_DoWork(object sender, DoWorkEventArgs e)
{
DictionaryData DictData = (DictionaryData)e.Argument;
//report what we're working on
FilenameLabel.Invoke((MethodInvoker) delegate
{
FilenameLabel.Text = "Loading CoreNLP models... please wait...";
});
//largely taken from here: https://github.com/sergey-tihon/Stanford.NLP.NET/issues/39
var jarRoot = @"stanford-corenlp-full-2018-02-27\";
var props = new java.util.Properties();
props.setProperty("annotators", "tokenize, ssplit, parse, sentiment");
props.setProperty("sutime.binders", "0");
var curDir = Environment.CurrentDirectory;
Directory.SetCurrentDirectory(Path.Combine(Path.GetDirectoryName(AppDomain.CurrentDomain.BaseDirectory), jarRoot));
var pipeline = new StanfordCoreNLP(props);
//selects the text encoding based on user selection
Encoding SelectedEncoding = null;
this.Invoke((MethodInvoker) delegate()
{
SelectedEncoding = Encoding.GetEncoding(EncodingDropdown.SelectedItem.ToString());
});
//get the list of files
var SearchDepth = SearchOption.TopDirectoryOnly;
if (ScanSubfolderCheckbox.Checked)
{
SearchDepth = SearchOption.AllDirectories;
}
var files = Directory.EnumerateFiles(DictData.TextFileFolder, "*.txt", SearchDepth);
//try
//{
//open up the output file
using (StreamWriter outputFile = new StreamWriter(new FileStream(DictData.OutputFileLocation, FileMode.Create), SelectedEncoding))
{
using (StreamWriter outputFileSentences = new StreamWriter(new FileStream(AddSuffix(DictData.OutputFileLocation, "_Sentences"), FileMode.Create), SelectedEncoding))
{
//write the header row to the output file
StringBuilder HeaderString = new StringBuilder();
HeaderString.Append("\"Filename\",\"Sentences\",\"Classification\",\"Classification_M\",\"Classification_SD\"");
outputFile.WriteLine(HeaderString.ToString());
StringBuilder HeaderStringSentence = new StringBuilder();
HeaderStringSentence.Append("\"Filename\",\"SentNumber\",\"SentenceText\",\"Classification\",\"Class_Prob\",\"Class_Number\"");
outputFileSentences.WriteLine(HeaderStringSentence.ToString());
foreach (string fileName in files)
{
//set up our variables to report
string Filename_Clean = Path.GetFileName(fileName);
Dictionary <string, int> DictionaryResults = new Dictionary <string, int>();
//report what we're working on
FilenameLabel.Invoke((MethodInvoker) delegate
{
FilenameLabel.Text = "Analyzing: " + Filename_Clean;
});
//read in the text file, convert everything to lowercase
string InputText = System.IO.File.ReadAllText(fileName, SelectedEncoding).Trim();
// _ _ _____ _
// / \ _ __ __ _| |_ _ _______ |_ _|____ _| |_
// / _ \ | '_ \ / _` | | | | |_ / _ \ | |/ _ \ \/ / __|
// / ___ \| | | | (_| | | |_| |/ / __/ | | __/> <| |_
// /_/ \_\_| |_|\__,_|_|\__, /___\___| |_|\___/_/\_\\__|
// |___/
var annotation = new edu.stanford.nlp.pipeline.Annotation(InputText);
pipeline.annotate(annotation);
List <double> SentimentValues = new List <double>();
var sentences = annotation.get(new CoreAnnotations.SentencesAnnotation().getClass()) as ArrayList;
int SentenceCount = 0;
foreach (CoreMap sentence in sentences)
{
SentenceCount++;
Tree tree = sentence.get(new SentimentCoreAnnotations.SentimentAnnotatedTree().getClass()) as Tree;
//add this sentence to our overall list of sentiment scores
SentimentValues.Add(RNNCoreAnnotations.getPredictedClass(tree));
// __ __ _ _ ___ _ _
// \ \ / / __(_) |_ ___ / _ \ _ _| |_ _ __ _ _| |_
// \ \ /\ / / '__| | __/ _ \ | | | | | | | __| '_ \| | | | __|
// \ V V /| | | | || __/ | |_| | |_| | |_| |_) | |_| | |_
// \_/\_/ |_| |_|\__\___| \___/ \__,_|\__| .__/ \__,_|\__|
// |_|
string[] OutputString_SentenceLevel = new string[6];
string Classification = GetClassification((double)RNNCoreAnnotations.getPredictedClass(tree));
OutputString_SentenceLevel[0] = "\"" + Filename_Clean + "\"";
OutputString_SentenceLevel[1] = SentenceCount.ToString();
OutputString_SentenceLevel[2] = "\"" + sentence.ToString().Replace("\"", "\"\"") + "\"";
OutputString_SentenceLevel[3] = Classification;
OutputString_SentenceLevel[4] = RNNCoreAnnotations.getPredictedClassProb(tree.label()).ToString();
OutputString_SentenceLevel[5] = RNNCoreAnnotations.getPredictedClass(tree).ToString();
outputFileSentences.WriteLine(String.Join(",", OutputString_SentenceLevel));
}
//write output at the file level
string[] OutputString = new string[5];
OutputString[0] = "\"" + Filename_Clean + "\"";
OutputString[1] = SentenceCount.ToString();
OutputString[2] = GetClassification(SentimentValues.Average());
OutputString[3] = SentimentValues.Average().ToString();
OutputString[4] = StandardDeviation(SentimentValues).ToString();
outputFile.WriteLine(String.Join(",", OutputString));
}
//this is the closing bracket for the sentence-level "using" filestream
}
//this is the closing bracket for the document-level "using" filestream
}
//}
//catch
//{
// MessageBox.Show("Senti-Gent encountered an issue somewhere while trying to analyze your texts. The most common cause of this is trying to open your output file while Senti-Gent is still running. Did any of your input files move, or is your output file being opened/modified by another application?", "Error while analyzing", MessageBoxButtons.OK, MessageBoxIcon.Error);
//}
}
/// <summary>
/// This is the method to call for assigning labels and node vectors
/// to the Tree.
/// </summary>
/// <remarks>
/// This is the method to call for assigning labels and node vectors
/// to the Tree. After calling this, each of the non-leaf nodes will
/// have the node vector and the predictions of their classes
/// assigned to that subtree's node. The annotations filled in are
/// the RNNCoreAnnotations.NodeVector, Predictions, and
/// PredictedClass. In general, PredictedClass will be the most
/// useful annotation except when training.
/// </remarks>
public virtual void ForwardPropagateTree(Tree tree)
{
SimpleMatrix nodeVector;
// initialized below or Exception thrown // = null;
SimpleMatrix classification;
// initialized below or Exception thrown // = null;
if (tree.IsLeaf())
{
// We do nothing for the leaves. The preterminals will
// calculate the classification for this word/tag. In fact, the
// recursion should not have gotten here (unless there are
// degenerate trees of just one leaf)
log.Info("SentimentCostAndGradient: warning: We reached leaves in forwardPropagate: " + tree);
throw new AssertionError("We should not have reached leaves in forwardPropagate");
}
else
{
if (tree.IsPreTerminal())
{
classification = model.GetUnaryClassification(tree.Label().Value());
string word = tree.Children()[0].Label().Value();
SimpleMatrix wordVector = model.GetWordVector(word);
nodeVector = NeuralUtils.ElementwiseApplyTanh(wordVector);
}
else
{
if (tree.Children().Length == 1)
{
log.Info("SentimentCostAndGradient: warning: Non-preterminal nodes of size 1: " + tree);
throw new AssertionError("Non-preterminal nodes of size 1 should have already been collapsed");
}
else
{
if (tree.Children().Length == 2)
{
ForwardPropagateTree(tree.Children()[0]);
ForwardPropagateTree(tree.Children()[1]);
string leftCategory = tree.Children()[0].Label().Value();
string rightCategory = tree.Children()[1].Label().Value();
SimpleMatrix W = model.GetBinaryTransform(leftCategory, rightCategory);
classification = model.GetBinaryClassification(leftCategory, rightCategory);
SimpleMatrix leftVector = RNNCoreAnnotations.GetNodeVector(tree.Children()[0]);
SimpleMatrix rightVector = RNNCoreAnnotations.GetNodeVector(tree.Children()[1]);
SimpleMatrix childrenVector = NeuralUtils.ConcatenateWithBias(leftVector, rightVector);
if (model.op.useTensors)
{
SimpleTensor tensor = model.GetBinaryTensor(leftCategory, rightCategory);
SimpleMatrix tensorIn = NeuralUtils.Concatenate(leftVector, rightVector);
SimpleMatrix tensorOut = tensor.BilinearProducts(tensorIn);
nodeVector = NeuralUtils.ElementwiseApplyTanh(W.Mult(childrenVector).Plus(tensorOut));
}
else
{
nodeVector = NeuralUtils.ElementwiseApplyTanh(W.Mult(childrenVector));
}
}
else
{
log.Info("SentimentCostAndGradient: warning: Tree not correctly binarized: " + tree);
throw new AssertionError("Tree not correctly binarized");
}
}
}
}
SimpleMatrix predictions = NeuralUtils.Softmax(classification.Mult(NeuralUtils.ConcatenateWithBias(nodeVector)));
int index = GetPredictedClass(predictions);
if (!(tree.Label() is CoreLabel))
{
log.Info("SentimentCostAndGradient: warning: No CoreLabels in nodes: " + tree);
throw new AssertionError("Expected CoreLabels in the nodes");
}
CoreLabel label = (CoreLabel)tree.Label();
label.Set(typeof(RNNCoreAnnotations.Predictions), predictions);
label.Set(typeof(RNNCoreAnnotations.PredictedClass), index);
label.Set(typeof(RNNCoreAnnotations.NodeVector), nodeVector);
}
private void BackpropDerivativesAndError(Tree tree, TwoDimensionalMap <string, string, SimpleMatrix> binaryTD, TwoDimensionalMap <string, string, SimpleMatrix> binaryCD, TwoDimensionalMap <string, string, SimpleTensor> binaryTensorTD, IDictionary
<string, SimpleMatrix> unaryCD, IDictionary <string, SimpleMatrix> wordVectorD, SimpleMatrix deltaUp)
{
if (tree.IsLeaf())
{
return;
}
SimpleMatrix currentVector = RNNCoreAnnotations.GetNodeVector(tree);
string category = tree.Label().Value();
category = model.BasicCategory(category);
// Build a vector that looks like 0,0,1,0,0 with an indicator for the correct class
SimpleMatrix goldLabel = new SimpleMatrix(model.numClasses, 1);
int goldClass = RNNCoreAnnotations.GetGoldClass(tree);
if (goldClass >= 0)
{
goldLabel.Set(goldClass, 1.0);
}
double nodeWeight = model.op.trainOptions.GetClassWeight(goldClass);
SimpleMatrix predictions = RNNCoreAnnotations.GetPredictions(tree);
// If this is an unlabeled class, set deltaClass to 0. We could
// make this more efficient by eliminating various of the below
// calculations, but this would be the easiest way to handle the
// unlabeled class
SimpleMatrix deltaClass = goldClass >= 0 ? predictions.Minus(goldLabel).Scale(nodeWeight) : new SimpleMatrix(predictions.NumRows(), predictions.NumCols());
SimpleMatrix localCD = deltaClass.Mult(NeuralUtils.ConcatenateWithBias(currentVector).Transpose());
double error = -(NeuralUtils.ElementwiseApplyLog(predictions).ElementMult(goldLabel).ElementSum());
error = error * nodeWeight;
RNNCoreAnnotations.SetPredictionError(tree, error);
if (tree.IsPreTerminal())
{
// below us is a word vector
unaryCD[category] = unaryCD[category].Plus(localCD);
string word = tree.Children()[0].Label().Value();
word = model.GetVocabWord(word);
//SimpleMatrix currentVectorDerivative = NeuralUtils.elementwiseApplyTanhDerivative(currentVector);
//SimpleMatrix deltaFromClass = model.getUnaryClassification(category).transpose().mult(deltaClass);
//SimpleMatrix deltaFull = deltaFromClass.extractMatrix(0, model.op.numHid, 0, 1).plus(deltaUp);
//SimpleMatrix wordDerivative = deltaFull.elementMult(currentVectorDerivative);
//wordVectorD.put(word, wordVectorD.get(word).plus(wordDerivative));
SimpleMatrix currentVectorDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(currentVector);
SimpleMatrix deltaFromClass = model.GetUnaryClassification(category).Transpose().Mult(deltaClass);
deltaFromClass = deltaFromClass.ExtractMatrix(0, model.op.numHid, 0, 1).ElementMult(currentVectorDerivative);
SimpleMatrix deltaFull = deltaFromClass.Plus(deltaUp);
SimpleMatrix oldWordVectorD = wordVectorD[word];
if (oldWordVectorD == null)
{
wordVectorD[word] = deltaFull;
}
else
{
wordVectorD[word] = oldWordVectorD.Plus(deltaFull);
}
}
else
{
// Otherwise, this must be a binary node
string leftCategory = model.BasicCategory(tree.Children()[0].Label().Value());
string rightCategory = model.BasicCategory(tree.Children()[1].Label().Value());
if (model.op.combineClassification)
{
unaryCD[string.Empty] = unaryCD[string.Empty].Plus(localCD);
}
else
{
binaryCD.Put(leftCategory, rightCategory, binaryCD.Get(leftCategory, rightCategory).Plus(localCD));
}
SimpleMatrix currentVectorDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(currentVector);
SimpleMatrix deltaFromClass = model.GetBinaryClassification(leftCategory, rightCategory).Transpose().Mult(deltaClass);
deltaFromClass = deltaFromClass.ExtractMatrix(0, model.op.numHid, 0, 1).ElementMult(currentVectorDerivative);
SimpleMatrix deltaFull = deltaFromClass.Plus(deltaUp);
SimpleMatrix leftVector = RNNCoreAnnotations.GetNodeVector(tree.Children()[0]);
SimpleMatrix rightVector = RNNCoreAnnotations.GetNodeVector(tree.Children()[1]);
SimpleMatrix childrenVector = NeuralUtils.ConcatenateWithBias(leftVector, rightVector);
SimpleMatrix W_df = deltaFull.Mult(childrenVector.Transpose());
binaryTD.Put(leftCategory, rightCategory, binaryTD.Get(leftCategory, rightCategory).Plus(W_df));
SimpleMatrix deltaDown;
if (model.op.useTensors)
{
SimpleTensor Wt_df = GetTensorGradient(deltaFull, leftVector, rightVector);
binaryTensorTD.Put(leftCategory, rightCategory, binaryTensorTD.Get(leftCategory, rightCategory).Plus(Wt_df));
deltaDown = ComputeTensorDeltaDown(deltaFull, leftVector, rightVector, model.GetBinaryTransform(leftCategory, rightCategory), model.GetBinaryTensor(leftCategory, rightCategory));
}
else
{
deltaDown = model.GetBinaryTransform(leftCategory, rightCategory).Transpose().Mult(deltaFull);
}
SimpleMatrix leftDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(leftVector);
SimpleMatrix rightDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(rightVector);
SimpleMatrix leftDeltaDown = deltaDown.ExtractMatrix(0, deltaFull.NumRows(), 0, 1);
SimpleMatrix rightDeltaDown = deltaDown.ExtractMatrix(deltaFull.NumRows(), deltaFull.NumRows() * 2, 0, 1);
BackpropDerivativesAndError(tree.Children()[0], binaryTD, binaryCD, binaryTensorTD, unaryCD, wordVectorD, leftDerivative.ElementMult(leftDeltaDown));
BackpropDerivativesAndError(tree.Children()[1], binaryTD, binaryCD, binaryTensorTD, unaryCD, wordVectorD, rightDerivative.ElementMult(rightDeltaDown));
}
}
/// <summary>Converts the given annotation to an XML document using the specified options</summary>
public static Document AnnotationToDoc(Annotation annotation, AnnotationOutputter.Options options)
{
//
// create the XML document with the root node pointing to the namespace URL
//
Element root = new Element("root", NamespaceUri);
Document xmlDoc = new Document(root);
ProcessingInstruction pi = new ProcessingInstruction("xml-stylesheet", "href=\"" + StylesheetName + "\" type=\"text/xsl\"");
xmlDoc.InsertChild(pi, 0);
Element docElem = new Element("document", NamespaceUri);
root.AppendChild(docElem);
SetSingleElement(docElem, "docId", NamespaceUri, annotation.Get(typeof(CoreAnnotations.DocIDAnnotation)));
SetSingleElement(docElem, "docDate", NamespaceUri, annotation.Get(typeof(CoreAnnotations.DocDateAnnotation)));
SetSingleElement(docElem, "docSourceType", NamespaceUri, annotation.Get(typeof(CoreAnnotations.DocSourceTypeAnnotation)));
SetSingleElement(docElem, "docType", NamespaceUri, annotation.Get(typeof(CoreAnnotations.DocTypeAnnotation)));
SetSingleElement(docElem, "author", NamespaceUri, annotation.Get(typeof(CoreAnnotations.AuthorAnnotation)));
SetSingleElement(docElem, "location", NamespaceUri, annotation.Get(typeof(CoreAnnotations.LocationAnnotation)));
if (options.includeText)
{
SetSingleElement(docElem, "text", NamespaceUri, annotation.Get(typeof(CoreAnnotations.TextAnnotation)));
}
Element sentencesElem = new Element("sentences", NamespaceUri);
docElem.AppendChild(sentencesElem);
//
// save the info for each sentence in this doc
//
if (annotation.Get(typeof(CoreAnnotations.SentencesAnnotation)) != null)
{
int sentCount = 1;
foreach (ICoreMap sentence in annotation.Get(typeof(CoreAnnotations.SentencesAnnotation)))
{
Element sentElem = new Element("sentence", NamespaceUri);
sentElem.AddAttribute(new Attribute("id", int.ToString(sentCount)));
int lineNumber = sentence.Get(typeof(CoreAnnotations.LineNumberAnnotation));
if (lineNumber != null)
{
sentElem.AddAttribute(new Attribute("line", int.ToString(lineNumber)));
}
sentCount++;
// add the word table with all token-level annotations
Element wordTable = new Element("tokens", NamespaceUri);
IList <CoreLabel> tokens = sentence.Get(typeof(CoreAnnotations.TokensAnnotation));
for (int j = 0; j < tokens.Count; j++)
{
Element wordInfo = new Element("token", NamespaceUri);
AddWordInfo(wordInfo, tokens[j], j + 1, NamespaceUri);
wordTable.AppendChild(wordInfo);
}
sentElem.AppendChild(wordTable);
// add tree info
Tree tree = sentence.Get(typeof(TreeCoreAnnotations.TreeAnnotation));
if (tree != null)
{
// add the constituent tree for this sentence
Element parseInfo = new Element("parse", NamespaceUri);
AddConstituentTreeInfo(parseInfo, tree, options.constituentTreePrinter);
sentElem.AppendChild(parseInfo);
}
SemanticGraph basicDependencies = sentence.Get(typeof(SemanticGraphCoreAnnotations.BasicDependenciesAnnotation));
if (basicDependencies != null)
{
// add the dependencies for this sentence
Element depInfo = BuildDependencyTreeInfo("basic-dependencies", sentence.Get(typeof(SemanticGraphCoreAnnotations.BasicDependenciesAnnotation)), tokens, NamespaceUri);
if (depInfo != null)
{
sentElem.AppendChild(depInfo);
}
depInfo = BuildDependencyTreeInfo("collapsed-dependencies", sentence.Get(typeof(SemanticGraphCoreAnnotations.CollapsedDependenciesAnnotation)), tokens, NamespaceUri);
if (depInfo != null)
{
sentElem.AppendChild(depInfo);
}
depInfo = BuildDependencyTreeInfo("collapsed-ccprocessed-dependencies", sentence.Get(typeof(SemanticGraphCoreAnnotations.CollapsedCCProcessedDependenciesAnnotation)), tokens, NamespaceUri);
if (depInfo != null)
{
sentElem.AppendChild(depInfo);
}
depInfo = BuildDependencyTreeInfo("enhanced-dependencies", sentence.Get(typeof(SemanticGraphCoreAnnotations.EnhancedDependenciesAnnotation)), tokens, NamespaceUri);
if (depInfo != null)
{
sentElem.AppendChild(depInfo);
}
depInfo = BuildDependencyTreeInfo("enhanced-plus-plus-dependencies", sentence.Get(typeof(SemanticGraphCoreAnnotations.EnhancedPlusPlusDependenciesAnnotation)), tokens, NamespaceUri);
if (depInfo != null)
{
sentElem.AppendChild(depInfo);
}
}
// add Open IE triples
ICollection <RelationTriple> openieTriples = sentence.Get(typeof(NaturalLogicAnnotations.RelationTriplesAnnotation));
if (openieTriples != null)
{
Element openieElem = new Element("openie", NamespaceUri);
AddTriples(openieTriples, openieElem, NamespaceUri);
sentElem.AppendChild(openieElem);
}
// add KBP triples
ICollection <RelationTriple> kbpTriples = sentence.Get(typeof(CoreAnnotations.KBPTriplesAnnotation));
if (kbpTriples != null)
{
Element kbpElem = new Element("kbp", NamespaceUri);
AddTriples(kbpTriples, kbpElem, NamespaceUri);
sentElem.AppendChild(kbpElem);
}
// add the MR entities and relations
IList <EntityMention> entities = sentence.Get(typeof(MachineReadingAnnotations.EntityMentionsAnnotation));
IList <RelationMention> relations = sentence.Get(typeof(MachineReadingAnnotations.RelationMentionsAnnotation));
if (entities != null && !entities.IsEmpty())
{
Element mrElem = new Element("MachineReading", NamespaceUri);
Element entElem = new Element("entities", NamespaceUri);
AddEntities(entities, entElem, NamespaceUri);
mrElem.AppendChild(entElem);
if (relations != null)
{
Element relElem = new Element("relations", NamespaceUri);
AddRelations(relations, relElem, NamespaceUri, options.relationsBeam);
mrElem.AppendChild(relElem);
}
sentElem.AppendChild(mrElem);
}
// Adds sentiment as an attribute of this sentence.
Tree sentimentTree = sentence.Get(typeof(SentimentCoreAnnotations.SentimentAnnotatedTree));
if (sentimentTree != null)
{
int sentiment = RNNCoreAnnotations.GetPredictedClass(sentimentTree);
sentElem.AddAttribute(new Attribute("sentimentValue", int.ToString(sentiment)));
string sentimentClass = sentence.Get(typeof(SentimentCoreAnnotations.SentimentClass));
sentElem.AddAttribute(new Attribute("sentiment", sentimentClass.ReplaceAll(" ", string.Empty)));
}
// add the sentence to the root
sentencesElem.AppendChild(sentElem);
}
}
//
// add the coref graph
//
IDictionary <int, CorefChain> corefChains = annotation.Get(typeof(CorefCoreAnnotations.CorefChainAnnotation));
if (corefChains != null)
{
IList <ICoreMap> sentences = annotation.Get(typeof(CoreAnnotations.SentencesAnnotation));
Element corefInfo = new Element("coreference", NamespaceUri);
AddCorefGraphInfo(options, corefInfo, sentences, corefChains, NamespaceUri);
docElem.AppendChild(corefInfo);
}
//
// save any document-level annotations here
//
return(xmlDoc);
}
public Payload RunPlugin(Payload Input)
{
Payload pData = new Payload();
pData.FileID = Input.FileID;
bool trackSegmentID = false;
if (Input.SegmentID.Count > 0)
{
trackSegmentID = true;
}
else
{
pData.SegmentID = Input.SegmentID;
}
for (int i = 0; i < Input.StringList.Count; i++)
{
//seems to prematurely exit sometimes. checking to see what might cause that -- maybe blank docs?
if (!string.IsNullOrEmpty(Input.StringList[i]) && !string.IsNullOrWhiteSpace(Input.StringList[i]))
{
Annotation annotation = new edu.stanford.nlp.pipeline.Annotation();
ArrayList sentences = new ArrayList();
List <double> SentimentValues = new List <double>();
annotation = new edu.stanford.nlp.pipeline.Annotation(Input.StringList[i]);
pipeline.annotate(annotation);
sentences = annotation.get(new CoreAnnotations.SentencesAnnotation().getClass()) as ArrayList;
int SentenceCount = 0;
foreach (CoreMap sentence in sentences)
{
SentenceCount++;
Tree tree = sentence.get(new SentimentCoreAnnotations.SentimentAnnotatedTree().getClass()) as Tree;
//add this sentence to our overall list of sentiment scores
SentimentValues.Add(RNNCoreAnnotations.getPredictedClass(tree));
string[] OutputString_SentenceLevel = new string[10] {
"", "", "", "", "", "", "", "", "", ""
};
string Classification = GetClassification((double)RNNCoreAnnotations.getPredictedClass(tree));
//this pulls out the prediction probabilites for each class
string Predictions = RNNCoreAnnotations.getPredictionsAsStringList(tree).ToString();
string[] Predictions_Split = Predictions.Replace("[", "").Replace("]", "").Split(',');
if (useBuiltInSentenceSplitter)
{
OutputString_SentenceLevel[0] = SentenceCount.ToString();
}
else
{
//if we're using an external sentence tokenizer, then every segment is
//going to be treated as its own sentence.
OutputString_SentenceLevel[0] = (i + 1).ToString();
}
OutputString_SentenceLevel[1] = Classification;
OutputString_SentenceLevel[2] = RNNCoreAnnotations.getPredictedClassProb(tree.label()).ToString();
OutputString_SentenceLevel[3] = RNNCoreAnnotations.getPredictedClass(tree).ToString();
OutputString_SentenceLevel[4] = Predictions_Split[0];
OutputString_SentenceLevel[5] = Predictions_Split[1];
OutputString_SentenceLevel[6] = Predictions_Split[2];
OutputString_SentenceLevel[7] = Predictions_Split[3];
OutputString_SentenceLevel[8] = Predictions_Split[4];
if (includeSentenceText)
{
OutputString_SentenceLevel[9] = sentence.ToString();
}
pData.StringArrayList.Add(OutputString_SentenceLevel);
pData.SegmentNumber.Add(Input.SegmentNumber[i]);
if (trackSegmentID)
{
pData.SegmentID.Add(Input.SegmentID[i]);
}
}
}
else
{
pData.StringArrayList.Add(new string[10] {
"", "", "", "", "", "", "", "", "", ""
});
pData.SegmentNumber.Add(Input.SegmentNumber[i]);
if (trackSegmentID)
{
pData.SegmentID.Add(Input.SegmentID[i]);
}
}
}
return(pData);
}
