public virtual double[] ParamsToVector()
{
int totalSize = TotalParamSize();
return(NeuralUtils.ParamsToVector(totalSize, binaryTransform.ValueIterator(), binaryClassification.ValueIterator(), SimpleTensor.IteratorSimpleMatrix(binaryTensors.ValueIterator()), unaryClassification.Values.GetEnumerator(), wordVectors.Values
.GetEnumerator()));
}
private SimpleMatrix ConcatenateContextWords(SimpleMatrix childVec, IntPair span, IList <string> words)
{
// TODO: factor out getting the words
SimpleMatrix left = (span.GetSource() < 0) ? dvModel.GetStartWordVector() : dvModel.GetWordVector(words[span.GetSource()]);
SimpleMatrix right = (span.GetTarget() >= words.Count) ? dvModel.GetEndWordVector() : dvModel.GetWordVector(words[span.GetTarget()]);
return(NeuralUtils.Concatenate(childVec, left, right));
}
public virtual SimpleMatrix GetMentionEmbeddings(Mention m, SimpleMatrix docEmbedding)
{
IEnumerator <SemanticGraphEdge> depIterator = m.enhancedDependency.IncomingEdgeIterator(m.headIndexedWord);
SemanticGraphEdge depRelation = depIterator.MoveNext() ? depIterator.Current : null;
return(NeuralUtils.Concatenate(GetAverageEmbedding(m.sentenceWords, m.startIndex, m.endIndex), GetAverageEmbedding(m.sentenceWords, m.startIndex - 5, m.startIndex), GetAverageEmbedding(m.sentenceWords, m.endIndex, m.endIndex + 5), GetAverageEmbedding
(m.sentenceWords.SubList(0, m.sentenceWords.Count - 1)), docEmbedding, GetWordEmbedding(m.sentenceWords, m.headIndex), GetWordEmbedding(m.sentenceWords, m.startIndex), GetWordEmbedding(m.sentenceWords, m.endIndex - 1), GetWordEmbedding(m.sentenceWords
, m.startIndex - 1), GetWordEmbedding(m.sentenceWords, m.endIndex), GetWordEmbedding(m.sentenceWords, m.startIndex - 2), GetWordEmbedding(m.sentenceWords, m.endIndex + 1), GetWordEmbedding(depRelation == null ? null : depRelation.GetSource(
).Word())));
}
public virtual void VectorToParams(double[] theta)
{
if (op.trainOptions.trainWordVectors)
{
NeuralUtils.VectorToParams(theta, binaryTransform.ValueIterator(), unaryTransform.Values.GetEnumerator(), binaryScore.ValueIterator(), unaryScore.Values.GetEnumerator(), wordVectors.Values.GetEnumerator());
}
else
{
NeuralUtils.VectorToParams(theta, binaryTransform.ValueIterator(), unaryTransform.Values.GetEnumerator(), binaryScore.ValueIterator(), unaryScore.Values.GetEnumerator());
}
}
private static double Score(SimpleMatrix features, IList <SimpleMatrix> weights)
{
for (int i = 0; i < weights.Count; i += 2)
{
features = weights[i].Mult(features).Plus(weights[i + 1]);
if (weights[i].NumRows() > 1)
{
features = NeuralUtils.ElementwiseApplyReLU(features);
}
}
return(features.ElementSum());
}
private static SimpleTensor GetTensorGradient(SimpleMatrix deltaFull, SimpleMatrix leftVector, SimpleMatrix rightVector)
{
int size = deltaFull.GetNumElements();
SimpleTensor Wt_df = new SimpleTensor(size * 2, size * 2, size);
// TODO: combine this concatenation with computeTensorDeltaDown?
SimpleMatrix fullVector = NeuralUtils.Concatenate(leftVector, rightVector);
for (int slice = 0; slice < size; ++slice)
{
Wt_df.SetSlice(slice, fullVector.Scale(deltaFull.Get(slice)).Mult(fullVector.Transpose()));
}
return(Wt_df);
}
public virtual double[] ParamsToVector()
{
int totalSize = TotalParamSize();
if (op.trainOptions.trainWordVectors)
{
return(NeuralUtils.ParamsToVector(totalSize, binaryTransform.ValueIterator(), unaryTransform.Values.GetEnumerator(), binaryScore.ValueIterator(), unaryScore.Values.GetEnumerator(), wordVectors.Values.GetEnumerator()));
}
else
{
return(NeuralUtils.ParamsToVector(totalSize, binaryTransform.ValueIterator(), unaryTransform.Values.GetEnumerator(), binaryScore.ValueIterator(), unaryScore.Values.GetEnumerator()));
}
}
public void CalculateOutputs()
{
if (type != NeuronType.output_neuron)
{
for (int i = 0; i < outputs.Length; i++)
{
outputs[i] = NeuralUtils.CustomSigmoid(bias + data * weights[i], -0.5f, 2, 0.3f);
}
}
else
{
outputs[0] = NeuralUtils.CustomSigmoid(bias + data, -0.5f, 2, 0.3f);
}
}
private static SimpleMatrix ComputeTensorDeltaDown(SimpleMatrix deltaFull, SimpleMatrix leftVector, SimpleMatrix rightVector, SimpleMatrix W, SimpleTensor Wt)
{
SimpleMatrix WTDelta = W.Transpose().Mult(deltaFull);
SimpleMatrix WTDeltaNoBias = WTDelta.ExtractMatrix(0, deltaFull.NumRows() * 2, 0, 1);
int size = deltaFull.GetNumElements();
SimpleMatrix deltaTensor = new SimpleMatrix(size * 2, 1);
SimpleMatrix fullVector = NeuralUtils.Concatenate(leftVector, rightVector);
for (int slice = 0; slice < size; ++slice)
{
SimpleMatrix scaledFullVector = fullVector.Scale(deltaFull.Get(slice));
deltaTensor = deltaTensor.Plus(Wt.GetSlice(slice).Plus(Wt.GetSlice(slice).Transpose()).Mult(scaledFullVector));
}
return(deltaTensor.Plus(WTDeltaNoBias));
}
public virtual SimpleMatrix GetPairFeatures(Pair <int, int> pair, Document document, IDictionary <int, IList <Mention> > mentionsByHeadIndex)
{
Mention m1 = document.predictedMentionsByID[pair.first];
Mention m2 = document.predictedMentionsByID[pair.second];
IList <int> featureVals = PairwiseFeatures(document, m1, m2, dictionaries, conll);
SimpleMatrix features = new SimpleMatrix(featureVals.Count, 1);
for (int i = 0; i < featureVals.Count; i++)
{
features.Set(i, featureVals[i]);
}
features = NeuralUtils.Concatenate(features, EncodeDistance(m2.sentNum - m1.sentNum), EncodeDistance(m2.mentionNum - m1.mentionNum - 1), new SimpleMatrix(new double[][] { new double[] { m1.sentNum == m2.sentNum && m1.endIndex > m2.startIndex
? 1 : 0 } }), GetMentionFeatures(m1, document, mentionsByHeadIndex), GetMentionFeatures(m2, document, mentionsByHeadIndex), EncodeGenre(document));
return(features);
}
/// <exception cref="System.IO.IOException"/>
public static SimpleMatrix LoadMatrix(string binaryName, string textName)
{
File matrixFile = new File(binaryName);
if (matrixFile.Exists())
{
return(SimpleMatrix.LoadBinary(matrixFile.GetPath()));
}
matrixFile = new File(textName);
if (matrixFile.Exists())
{
return(NeuralUtils.LoadTextMatrix(matrixFile));
}
throw new Exception("Could not find either " + binaryName + " or " + textName);
}
protected internal override void Calculate(double[] theta)
{
model.VectorToParams(theta);
SentimentCostAndGradient.ModelDerivatives derivatives;
if (model.op.trainOptions.nThreads == 1)
{
derivatives = ScoreDerivatives(trainingBatch);
}
else
{
// TODO: because some addition operations happen in different
// orders now, this results in slightly different values, which
// over time add up to significantly different models even when
// given the same random seed. Probably not a big deal.
// To be more specific, for trees T1, T2, T3, ... Tn,
// when using one thread, we sum the derivatives T1 + T2 ...
// When using multiple threads, we first sum T1 + ... + Tk,
// then sum Tk+1 + ... + T2k, etc, for split size k.
// The splits are then summed in order.
// This different sum order results in slightly different numbers.
MulticoreWrapper <IList <Tree>, SentimentCostAndGradient.ModelDerivatives> wrapper = new MulticoreWrapper <IList <Tree>, SentimentCostAndGradient.ModelDerivatives>(model.op.trainOptions.nThreads, new SentimentCostAndGradient.ScoringProcessor(this
));
// use wrapper.nThreads in case the number of threads was automatically changed
foreach (IList <Tree> chunk in CollectionUtils.PartitionIntoFolds(trainingBatch, wrapper.NThreads()))
{
wrapper.Put(chunk);
}
wrapper.Join();
derivatives = new SentimentCostAndGradient.ModelDerivatives(model);
while (wrapper.Peek())
{
SentimentCostAndGradient.ModelDerivatives batchDerivatives = wrapper.Poll();
derivatives.Add(batchDerivatives);
}
}
// scale the error by the number of sentences so that the
// regularization isn't drowned out for large training batchs
double scale = (1.0 / trainingBatch.Count);
value = derivatives.error * scale;
value += ScaleAndRegularize(derivatives.binaryTD, model.binaryTransform, scale, model.op.trainOptions.regTransformMatrix, false);
value += ScaleAndRegularize(derivatives.binaryCD, model.binaryClassification, scale, model.op.trainOptions.regClassification, true);
value += ScaleAndRegularizeTensor(derivatives.binaryTensorTD, model.binaryTensors, scale, model.op.trainOptions.regTransformTensor);
value += ScaleAndRegularize(derivatives.unaryCD, model.unaryClassification, scale, model.op.trainOptions.regClassification, false, true);
value += ScaleAndRegularize(derivatives.wordVectorD, model.wordVectors, scale, model.op.trainOptions.regWordVector, true, false);
derivative = NeuralUtils.ParamsToVector(theta.Length, derivatives.binaryTD.ValueIterator(), derivatives.binaryCD.ValueIterator(), SimpleTensor.IteratorSimpleMatrix(derivatives.binaryTensorTD.ValueIterator()), derivatives.unaryCD.Values.GetEnumerator
(), derivatives.wordVectorD.Values.GetEnumerator());
}
private void ConnectNode(
RectTransform target,
Neuron targetNeuron,
INeuralGenome genome)
{
var outNeurons = genome.Network[targetNeuron]
.Select(x => x.receiver);
foreach (var neuronInnov in outNeurons)
{
var neuron = genome.Neurons[neuronInnov];
if (neuron == targetNeuron)
{
continue;
}
var conn = ConnectionManager.CreateConnection(
target,
nodes[neuron]);
conn.points[0].weight = 0.6f;
conn.points[1].weight = 0.6f;
conn.line.widthMultiplier = connectionWidth;
conn.points[0].direction =
ConnectionPoint.ConnectionDirection.East;
conn.points[1].direction =
ConnectionPoint.ConnectionDirection.West;
var synapse = genome.Network[targetNeuron]
.First(x => x.receiver == neuronInnov);
var gene = NeuralUtils.FindGene(genome.Genes, synapse);
var color = GetColor(gene);
conn.points[0].color = color;
conn.points[1].color = color;
if (!synapse.isEnabled)
{
conn.line.widthMultiplier *= 0.3f;
}
}
}
public virtual double GetAnaphoricityScore(SimpleMatrix mentionEmbedding, SimpleMatrix anaphoricityFeatures)
{
return(Score(NeuralUtils.Concatenate(mentionEmbedding, anaphoricityFeatures), anaphoricityModel));
}
public virtual void VectorToParams(double[] theta)
{
NeuralUtils.VectorToParams(theta, binaryTransform.ValueIterator(), binaryClassification.ValueIterator(), SimpleTensor.IteratorSimpleMatrix(binaryTensors.ValueIterator()), unaryClassification.Values.GetEnumerator(), wordVectors.Values.GetEnumerator
());
}
// fill value & derivative
protected internal override void Calculate(double[] theta)
{
dvModel.VectorToParams(theta);
double localValue = 0.0;
double[] localDerivative = new double[theta.Length];
TwoDimensionalMap <string, string, SimpleMatrix> binaryW_dfsG;
TwoDimensionalMap <string, string, SimpleMatrix> binaryW_dfsB;
binaryW_dfsG = TwoDimensionalMap.TreeMap();
binaryW_dfsB = TwoDimensionalMap.TreeMap();
TwoDimensionalMap <string, string, SimpleMatrix> binaryScoreDerivativesG;
TwoDimensionalMap <string, string, SimpleMatrix> binaryScoreDerivativesB;
binaryScoreDerivativesG = TwoDimensionalMap.TreeMap();
binaryScoreDerivativesB = TwoDimensionalMap.TreeMap();
IDictionary <string, SimpleMatrix> unaryW_dfsG;
IDictionary <string, SimpleMatrix> unaryW_dfsB;
unaryW_dfsG = new SortedDictionary <string, SimpleMatrix>();
unaryW_dfsB = new SortedDictionary <string, SimpleMatrix>();
IDictionary <string, SimpleMatrix> unaryScoreDerivativesG;
IDictionary <string, SimpleMatrix> unaryScoreDerivativesB;
unaryScoreDerivativesG = new SortedDictionary <string, SimpleMatrix>();
unaryScoreDerivativesB = new SortedDictionary <string, SimpleMatrix>();
IDictionary <string, SimpleMatrix> wordVectorDerivativesG = new SortedDictionary <string, SimpleMatrix>();
IDictionary <string, SimpleMatrix> wordVectorDerivativesB = new SortedDictionary <string, SimpleMatrix>();
foreach (TwoDimensionalMap.Entry <string, string, SimpleMatrix> entry in dvModel.binaryTransform)
{
int numRows = entry.GetValue().NumRows();
int numCols = entry.GetValue().NumCols();
binaryW_dfsG.Put(entry.GetFirstKey(), entry.GetSecondKey(), new SimpleMatrix(numRows, numCols));
binaryW_dfsB.Put(entry.GetFirstKey(), entry.GetSecondKey(), new SimpleMatrix(numRows, numCols));
binaryScoreDerivativesG.Put(entry.GetFirstKey(), entry.GetSecondKey(), new SimpleMatrix(1, numRows));
binaryScoreDerivativesB.Put(entry.GetFirstKey(), entry.GetSecondKey(), new SimpleMatrix(1, numRows));
}
foreach (KeyValuePair <string, SimpleMatrix> entry_1 in dvModel.unaryTransform)
{
int numRows = entry_1.Value.NumRows();
int numCols = entry_1.Value.NumCols();
unaryW_dfsG[entry_1.Key] = new SimpleMatrix(numRows, numCols);
unaryW_dfsB[entry_1.Key] = new SimpleMatrix(numRows, numCols);
unaryScoreDerivativesG[entry_1.Key] = new SimpleMatrix(1, numRows);
unaryScoreDerivativesB[entry_1.Key] = new SimpleMatrix(1, numRows);
}
if (op.trainOptions.trainWordVectors)
{
foreach (KeyValuePair <string, SimpleMatrix> entry_2 in dvModel.wordVectors)
{
int numRows = entry_2.Value.NumRows();
int numCols = entry_2.Value.NumCols();
wordVectorDerivativesG[entry_2.Key] = new SimpleMatrix(numRows, numCols);
wordVectorDerivativesB[entry_2.Key] = new SimpleMatrix(numRows, numCols);
}
}
// Some optimization methods prints out a line without an end, so our
// debugging statements are misaligned
Timing scoreTiming = new Timing();
scoreTiming.Doing("Scoring trees");
int treeNum = 0;
MulticoreWrapper <Tree, Pair <DeepTree, DeepTree> > wrapper = new MulticoreWrapper <Tree, Pair <DeepTree, DeepTree> >(op.trainOptions.trainingThreads, new DVParserCostAndGradient.ScoringProcessor(this));
foreach (Tree tree in trainingBatch)
{
wrapper.Put(tree);
}
wrapper.Join();
scoreTiming.Done();
while (wrapper.Peek())
{
Pair <DeepTree, DeepTree> result = wrapper.Poll();
DeepTree goldTree = result.first;
DeepTree bestTree = result.second;
StringBuilder treeDebugLine = new StringBuilder();
Formatter formatter = new Formatter(treeDebugLine);
bool isDone = (Math.Abs(bestTree.GetScore() - goldTree.GetScore()) <= 0.00001 || goldTree.GetScore() > bestTree.GetScore());
string done = isDone ? "done" : string.Empty;
formatter.Format("Tree %6d Highest tree: %12.4f Correct tree: %12.4f %s", treeNum, bestTree.GetScore(), goldTree.GetScore(), done);
log.Info(treeDebugLine.ToString());
if (!isDone)
{
// if the gold tree is better than the best hypothesis tree by
// a large enough margin, then the score difference will be 0
// and we ignore the tree
double valueDelta = bestTree.GetScore() - goldTree.GetScore();
//double valueDelta = Math.max(0.0, - scoreGold + bestScore);
localValue += valueDelta;
// get the context words for this tree - should be the same
// for either goldTree or bestTree
IList <string> words = GetContextWords(goldTree.GetTree());
// The derivatives affected by this tree are only based on the
// nodes present in this tree, eg not all matrix derivatives
// will be affected by this tree
BackpropDerivative(goldTree.GetTree(), words, goldTree.GetVectors(), binaryW_dfsG, unaryW_dfsG, binaryScoreDerivativesG, unaryScoreDerivativesG, wordVectorDerivativesG);
BackpropDerivative(bestTree.GetTree(), words, bestTree.GetVectors(), binaryW_dfsB, unaryW_dfsB, binaryScoreDerivativesB, unaryScoreDerivativesB, wordVectorDerivativesB);
}
++treeNum;
}
double[] localDerivativeGood;
double[] localDerivativeB;
if (op.trainOptions.trainWordVectors)
{
localDerivativeGood = NeuralUtils.ParamsToVector(theta.Length, binaryW_dfsG.ValueIterator(), unaryW_dfsG.Values.GetEnumerator(), binaryScoreDerivativesG.ValueIterator(), unaryScoreDerivativesG.Values.GetEnumerator(), wordVectorDerivativesG.Values
.GetEnumerator());
localDerivativeB = NeuralUtils.ParamsToVector(theta.Length, binaryW_dfsB.ValueIterator(), unaryW_dfsB.Values.GetEnumerator(), binaryScoreDerivativesB.ValueIterator(), unaryScoreDerivativesB.Values.GetEnumerator(), wordVectorDerivativesB.Values
.GetEnumerator());
}
else
{
localDerivativeGood = NeuralUtils.ParamsToVector(theta.Length, binaryW_dfsG.ValueIterator(), unaryW_dfsG.Values.GetEnumerator(), binaryScoreDerivativesG.ValueIterator(), unaryScoreDerivativesG.Values.GetEnumerator());
localDerivativeB = NeuralUtils.ParamsToVector(theta.Length, binaryW_dfsB.ValueIterator(), unaryW_dfsB.Values.GetEnumerator(), binaryScoreDerivativesB.ValueIterator(), unaryScoreDerivativesB.Values.GetEnumerator());
}
// correct - highest
for (int i = 0; i < localDerivativeGood.Length; i++)
{
localDerivative[i] = localDerivativeB[i] - localDerivativeGood[i];
}
// TODO: this is where we would combine multiple costs if we had parallelized the calculation
value = localValue;
derivative = localDerivative;
// normalizing by training batch size
value = (1.0 / trainingBatch.Count) * value;
ArrayMath.MultiplyInPlace(derivative, (1.0 / trainingBatch.Count));
// add regularization to cost:
double[] currentParams = dvModel.ParamsToVector();
double regCost = 0;
foreach (double currentParam in currentParams)
{
regCost += currentParam * currentParam;
}
regCost = op.trainOptions.regCost * 0.5 * regCost;
value += regCost;
// add regularization to gradient
ArrayMath.MultiplyInPlace(currentParams, op.trainOptions.regCost);
ArrayMath.PairwiseAddInPlace(derivative, currentParams);
}
internal static SimpleMatrix RandomWordVector(int size, Random rand)
{
return(NeuralUtils.RandomGaussian(size, 1, rand).Scale(0.1));
}
//log.info(this);
/// <summary>Dumps *all* the matrices in a mostly readable format.</summary>
public override string ToString()
{
StringBuilder output = new StringBuilder();
if (binaryTransform.Size() > 0)
{
if (binaryTransform.Size() == 1)
{
output.Append("Binary transform matrix\n");
}
else
{
output.Append("Binary transform matrices\n");
}
foreach (TwoDimensionalMap.Entry <string, string, SimpleMatrix> matrix in binaryTransform)
{
if (!matrix.GetFirstKey().Equals(string.Empty) || !matrix.GetSecondKey().Equals(string.Empty))
{
output.Append(matrix.GetFirstKey() + " " + matrix.GetSecondKey() + ":\n");
}
output.Append(NeuralUtils.ToString(matrix.GetValue(), "%.8f"));
}
}
if (binaryTensors.Size() > 0)
{
if (binaryTensors.Size() == 1)
{
output.Append("Binary transform tensor\n");
}
else
{
output.Append("Binary transform tensors\n");
}
foreach (TwoDimensionalMap.Entry <string, string, SimpleTensor> matrix in binaryTensors)
{
if (!matrix.GetFirstKey().Equals(string.Empty) || !matrix.GetSecondKey().Equals(string.Empty))
{
output.Append(matrix.GetFirstKey() + " " + matrix.GetSecondKey() + ":\n");
}
output.Append(matrix.GetValue().ToString("%.8f"));
}
}
if (binaryClassification.Size() > 0)
{
if (binaryClassification.Size() == 1)
{
output.Append("Binary classification matrix\n");
}
else
{
output.Append("Binary classification matrices\n");
}
foreach (TwoDimensionalMap.Entry <string, string, SimpleMatrix> matrix in binaryClassification)
{
if (!matrix.GetFirstKey().Equals(string.Empty) || !matrix.GetSecondKey().Equals(string.Empty))
{
output.Append(matrix.GetFirstKey() + " " + matrix.GetSecondKey() + ":\n");
}
output.Append(NeuralUtils.ToString(matrix.GetValue(), "%.8f"));
}
}
if (unaryClassification.Count > 0)
{
if (unaryClassification.Count == 1)
{
output.Append("Unary classification matrix\n");
}
else
{
output.Append("Unary classification matrices\n");
}
foreach (KeyValuePair <string, SimpleMatrix> matrix in unaryClassification)
{
if (!matrix.Key.Equals(string.Empty))
{
output.Append(matrix.Key + ":\n");
}
output.Append(NeuralUtils.ToString(matrix.Value, "%.8f"));
}
}
output.Append("Word vectors\n");
foreach (KeyValuePair <string, SimpleMatrix> matrix_1 in wordVectors)
{
output.Append("'" + matrix_1.Key + "'");
output.Append("\n");
output.Append(NeuralUtils.ToString(matrix_1.Value, "%.8f"));
output.Append("\n");
}
return(output.ToString());
}
public void DrawGenome(INeuralGenome genome)
{
RemoveAllNodes();
if (autoAdjustMaxWeight)
{
maxWeight = GetAverageWeight(genome.Genes);
}
int count = 0;
foreach (var neuron in genome.NeuronLst)
{
var newNode = Instantiate(nodePrefab, nodesParent)
.GetComponent <RectTransform>();
newNode.name = "Node " + count;
var nodeText = neuron.innovNb.ToString();
if (neuron.neurType == ENeurType.bias)
{
nodeText = "bias" + nodeText;
}
else if (neuron.neurType == ENeurType.input)
{
nodeText = "in" + nodeText;
}
else if (neuron.neurType == ENeurType.output)
{
nodeText = "out" + nodeText;
}
newNode.GetChild(0).GetComponent <Text>().text = nodeText;
nodes.Add(neuron, newNode);
count++;
}
var inputNeurons = NeuralUtils.GetNeurons(
genome, ENeurType.input);
var outputNeurons = NeuralUtils.GetNeurons(
genome, ENeurType.output);
var bias = NeuralUtils.GetBias(genome);
foreach (var kv in nodes)
{
ConnectNode(kv.Value, kv.Key, genome);
if (inputNeurons.Contains(kv.Key))
{
int i = inputNeurons.IndexOf(kv.Key) + 1;
kv.Value.localPosition =
inputNodesPos.localPosition +
Vector3.down * distBetweenNodes * i;
}
else if (outputNeurons.Contains(kv.Key))
{
int i = outputNeurons.IndexOf(kv.Key);
kv.Value.localPosition =
outputNodesPos.localPosition +
Vector3.down * distBetweenNodes * i;
}
else if (bias == kv.Key)
{
kv.Value.localPosition = inputNodesPos.localPosition;
}
else
{
kv.Value.localPosition = GetPosOfNewNode();
}
}
}
/// <summary>
/// This is the method to call for assigning labels and node vectors
/// to the Tree.
/// </summary>
/// <remarks>
/// This is the method to call for assigning labels and node vectors
/// to the Tree. After calling this, each of the non-leaf nodes will
/// have the node vector and the predictions of their classes
/// assigned to that subtree's node. The annotations filled in are
/// the RNNCoreAnnotations.NodeVector, Predictions, and
/// PredictedClass. In general, PredictedClass will be the most
/// useful annotation except when training.
/// </remarks>
public virtual void ForwardPropagateTree(Tree tree)
{
SimpleMatrix nodeVector;
// initialized below or Exception thrown // = null;
SimpleMatrix classification;
// initialized below or Exception thrown // = null;
if (tree.IsLeaf())
{
// We do nothing for the leaves. The preterminals will
// calculate the classification for this word/tag. In fact, the
// recursion should not have gotten here (unless there are
// degenerate trees of just one leaf)
log.Info("SentimentCostAndGradient: warning: We reached leaves in forwardPropagate: " + tree);
throw new AssertionError("We should not have reached leaves in forwardPropagate");
}
else
{
if (tree.IsPreTerminal())
{
classification = model.GetUnaryClassification(tree.Label().Value());
string word = tree.Children()[0].Label().Value();
SimpleMatrix wordVector = model.GetWordVector(word);
nodeVector = NeuralUtils.ElementwiseApplyTanh(wordVector);
}
else
{
if (tree.Children().Length == 1)
{
log.Info("SentimentCostAndGradient: warning: Non-preterminal nodes of size 1: " + tree);
throw new AssertionError("Non-preterminal nodes of size 1 should have already been collapsed");
}
else
{
if (tree.Children().Length == 2)
{
ForwardPropagateTree(tree.Children()[0]);
ForwardPropagateTree(tree.Children()[1]);
string leftCategory = tree.Children()[0].Label().Value();
string rightCategory = tree.Children()[1].Label().Value();
SimpleMatrix W = model.GetBinaryTransform(leftCategory, rightCategory);
classification = model.GetBinaryClassification(leftCategory, rightCategory);
SimpleMatrix leftVector = RNNCoreAnnotations.GetNodeVector(tree.Children()[0]);
SimpleMatrix rightVector = RNNCoreAnnotations.GetNodeVector(tree.Children()[1]);
SimpleMatrix childrenVector = NeuralUtils.ConcatenateWithBias(leftVector, rightVector);
if (model.op.useTensors)
{
SimpleTensor tensor = model.GetBinaryTensor(leftCategory, rightCategory);
SimpleMatrix tensorIn = NeuralUtils.Concatenate(leftVector, rightVector);
SimpleMatrix tensorOut = tensor.BilinearProducts(tensorIn);
nodeVector = NeuralUtils.ElementwiseApplyTanh(W.Mult(childrenVector).Plus(tensorOut));
}
else
{
nodeVector = NeuralUtils.ElementwiseApplyTanh(W.Mult(childrenVector));
}
}
else
{
log.Info("SentimentCostAndGradient: warning: Tree not correctly binarized: " + tree);
throw new AssertionError("Tree not correctly binarized");
}
}
}
}
SimpleMatrix predictions = NeuralUtils.Softmax(classification.Mult(NeuralUtils.ConcatenateWithBias(nodeVector)));
int index = GetPredictedClass(predictions);
if (!(tree.Label() is CoreLabel))
{
log.Info("SentimentCostAndGradient: warning: No CoreLabels in nodes: " + tree);
throw new AssertionError("Expected CoreLabels in the nodes");
}
CoreLabel label = (CoreLabel)tree.Label();
label.Set(typeof(RNNCoreAnnotations.Predictions), predictions);
label.Set(typeof(RNNCoreAnnotations.PredictedClass), index);
label.Set(typeof(RNNCoreAnnotations.NodeVector), nodeVector);
}
public virtual void BackpropDerivative(Tree tree, IList <string> words, IdentityHashMap <Tree, SimpleMatrix> nodeVectors, TwoDimensionalMap <string, string, SimpleMatrix> binaryW_dfs, IDictionary <string, SimpleMatrix> unaryW_dfs, TwoDimensionalMap
<string, string, SimpleMatrix> binaryScoreDerivatives, IDictionary <string, SimpleMatrix> unaryScoreDerivatives, IDictionary <string, SimpleMatrix> wordVectorDerivatives, SimpleMatrix deltaUp)
{
if (tree.IsLeaf())
{
return;
}
if (tree.IsPreTerminal())
{
if (op.trainOptions.trainWordVectors)
{
string word = tree.Children()[0].Label().Value();
word = dvModel.GetVocabWord(word);
// SimpleMatrix currentVector = nodeVectors.get(tree);
// SimpleMatrix currentVectorDerivative = nonlinearityVectorToDerivative(currentVector);
// SimpleMatrix derivative = deltaUp.elementMult(currentVectorDerivative);
SimpleMatrix derivative = deltaUp;
wordVectorDerivatives[word] = wordVectorDerivatives[word].Plus(derivative);
}
return;
}
SimpleMatrix currentVector = nodeVectors[tree];
SimpleMatrix currentVectorDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(currentVector);
SimpleMatrix scoreW = dvModel.GetScoreWForNode(tree);
currentVectorDerivative = currentVectorDerivative.ElementMult(scoreW.Transpose());
// the delta that is used at the current nodes
SimpleMatrix deltaCurrent = deltaUp.Plus(currentVectorDerivative);
SimpleMatrix W = dvModel.GetWForNode(tree);
SimpleMatrix WTdelta = W.Transpose().Mult(deltaCurrent);
if (tree.Children().Length == 2)
{
//TODO: RS: Change to the nice "getWForNode" setup?
string leftLabel = dvModel.BasicCategory(tree.Children()[0].Label().Value());
string rightLabel = dvModel.BasicCategory(tree.Children()[1].Label().Value());
binaryScoreDerivatives.Put(leftLabel, rightLabel, binaryScoreDerivatives.Get(leftLabel, rightLabel).Plus(currentVector.Transpose()));
SimpleMatrix leftVector = nodeVectors[tree.Children()[0]];
SimpleMatrix rightVector = nodeVectors[tree.Children()[1]];
SimpleMatrix childrenVector = NeuralUtils.ConcatenateWithBias(leftVector, rightVector);
if (op.trainOptions.useContextWords)
{
childrenVector = ConcatenateContextWords(childrenVector, tree.GetSpan(), words);
}
SimpleMatrix W_df = deltaCurrent.Mult(childrenVector.Transpose());
binaryW_dfs.Put(leftLabel, rightLabel, binaryW_dfs.Get(leftLabel, rightLabel).Plus(W_df));
// and then recurse
SimpleMatrix leftDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(leftVector);
SimpleMatrix rightDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(rightVector);
SimpleMatrix leftWTDelta = WTdelta.ExtractMatrix(0, deltaCurrent.NumRows(), 0, 1);
SimpleMatrix rightWTDelta = WTdelta.ExtractMatrix(deltaCurrent.NumRows(), deltaCurrent.NumRows() * 2, 0, 1);
BackpropDerivative(tree.Children()[0], words, nodeVectors, binaryW_dfs, unaryW_dfs, binaryScoreDerivatives, unaryScoreDerivatives, wordVectorDerivatives, leftDerivative.ElementMult(leftWTDelta));
BackpropDerivative(tree.Children()[1], words, nodeVectors, binaryW_dfs, unaryW_dfs, binaryScoreDerivatives, unaryScoreDerivatives, wordVectorDerivatives, rightDerivative.ElementMult(rightWTDelta));
}
else
{
if (tree.Children().Length == 1)
{
string childLabel = dvModel.BasicCategory(tree.Children()[0].Label().Value());
unaryScoreDerivatives[childLabel] = unaryScoreDerivatives[childLabel].Plus(currentVector.Transpose());
SimpleMatrix childVector = nodeVectors[tree.Children()[0]];
SimpleMatrix childVectorWithBias = NeuralUtils.ConcatenateWithBias(childVector);
if (op.trainOptions.useContextWords)
{
childVectorWithBias = ConcatenateContextWords(childVectorWithBias, tree.GetSpan(), words);
}
SimpleMatrix W_df = deltaCurrent.Mult(childVectorWithBias.Transpose());
// System.out.println("unary backprop derivative for " + childLabel);
// System.out.println("Old transform:");
// System.out.println(unaryW_dfs.get(childLabel));
// System.out.println(" Delta:");
// System.out.println(W_df.scale(scale));
unaryW_dfs[childLabel] = unaryW_dfs[childLabel].Plus(W_df);
// and then recurse
SimpleMatrix childDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(childVector);
//SimpleMatrix childDerivative = childVector;
SimpleMatrix childWTDelta = WTdelta.ExtractMatrix(0, deltaCurrent.NumRows(), 0, 1);
BackpropDerivative(tree.Children()[0], words, nodeVectors, binaryW_dfs, unaryW_dfs, binaryScoreDerivatives, unaryScoreDerivatives, wordVectorDerivatives, childDerivative.ElementMult(childWTDelta));
}
}
}
private SimpleMatrix EncodeGenre(Document document)
{
return(conll ? NeuralUtils.OneHot(genres[document.docInfo["DOC_ID"].Split("/")[0]], genres.Count) : new SimpleMatrix(1, 1));
}
public virtual SimpleMatrix GetAnaphoricityFeatures(Mention m, Document document, IDictionary <int, IList <Mention> > mentionsByHeadIndex)
{
return(NeuralUtils.Concatenate(GetMentionFeatures(m, document, mentionsByHeadIndex), EncodeGenre(document)));
}
private void ForwardPropagateTree(Tree tree, IList <string> words, IdentityHashMap <Tree, SimpleMatrix> nodeVectors, IdentityHashMap <Tree, double> scores)
{
if (tree.IsLeaf())
{
return;
}
if (tree.IsPreTerminal())
{
Tree wordNode = tree.Children()[0];
string word = wordNode.Label().Value();
SimpleMatrix wordVector = dvModel.GetWordVector(word);
wordVector = NeuralUtils.ElementwiseApplyTanh(wordVector);
nodeVectors[tree] = wordVector;
return;
}
foreach (Tree child in tree.Children())
{
ForwardPropagateTree(child, words, nodeVectors, scores);
}
// at this point, nodeVectors contains the vectors for all of
// the children of tree
SimpleMatrix childVec;
if (tree.Children().Length == 2)
{
childVec = NeuralUtils.ConcatenateWithBias(nodeVectors[tree.Children()[0]], nodeVectors[tree.Children()[1]]);
}
else
{
childVec = NeuralUtils.ConcatenateWithBias(nodeVectors[tree.Children()[0]]);
}
if (op.trainOptions.useContextWords)
{
childVec = ConcatenateContextWords(childVec, tree.GetSpan(), words);
}
SimpleMatrix W = dvModel.GetWForNode(tree);
if (W == null)
{
string error = "Could not find W for tree " + tree;
if (op.testOptions.verbose)
{
log.Info(error);
}
throw new NoSuchParseException(error);
}
SimpleMatrix currentVector = W.Mult(childVec);
currentVector = NeuralUtils.ElementwiseApplyTanh(currentVector);
nodeVectors[tree] = currentVector;
SimpleMatrix scoreW = dvModel.GetScoreWForNode(tree);
if (scoreW == null)
{
string error = "Could not find scoreW for tree " + tree;
if (op.testOptions.verbose)
{
log.Info(error);
}
throw new NoSuchParseException(error);
}
double score = scoreW.Dot(currentVector);
//score = NeuralUtils.sigmoid(score);
scores[tree] = score;
}
public virtual double GetPairwiseScore(SimpleMatrix antecedentEmbedding, SimpleMatrix anaphorEmbedding, SimpleMatrix pairFeatures)
{
SimpleMatrix firstLayerOutput = NeuralUtils.ElementwiseApplyReLU(antecedentEmbedding.Plus(anaphorEmbedding).Plus(pairFeaturesMatrix.Mult(pairFeatures)).Plus(pairwiseFirstLayerBias));
return(Score(firstLayerOutput, pairwiseModel));
}
private SimpleMatrix GetMentionFeatures(Mention m, Document document, IDictionary <int, IList <Mention> > mentionsByHeadIndex)
{
return(NeuralUtils.Concatenate(NeuralUtils.OneHot((int)(m.mentionType), 4), EncodeDistance(m.endIndex - m.startIndex - 1), new SimpleMatrix(new double[][] { new double[] { m.mentionNum / (double)document.predictedMentionsByID.Count }, new double
[] { mentionsByHeadIndex[m.headIndex].Stream().AnyMatch(null) ? 1 : 0 } })));
}
private void BackpropDerivativesAndError(Tree tree, TwoDimensionalMap <string, string, SimpleMatrix> binaryTD, TwoDimensionalMap <string, string, SimpleMatrix> binaryCD, TwoDimensionalMap <string, string, SimpleTensor> binaryTensorTD, IDictionary
<string, SimpleMatrix> unaryCD, IDictionary <string, SimpleMatrix> wordVectorD, SimpleMatrix deltaUp)
{
if (tree.IsLeaf())
{
return;
}
SimpleMatrix currentVector = RNNCoreAnnotations.GetNodeVector(tree);
string category = tree.Label().Value();
category = model.BasicCategory(category);
// Build a vector that looks like 0,0,1,0,0 with an indicator for the correct class
SimpleMatrix goldLabel = new SimpleMatrix(model.numClasses, 1);
int goldClass = RNNCoreAnnotations.GetGoldClass(tree);
if (goldClass >= 0)
{
goldLabel.Set(goldClass, 1.0);
}
double nodeWeight = model.op.trainOptions.GetClassWeight(goldClass);
SimpleMatrix predictions = RNNCoreAnnotations.GetPredictions(tree);
// If this is an unlabeled class, set deltaClass to 0. We could
// make this more efficient by eliminating various of the below
// calculations, but this would be the easiest way to handle the
// unlabeled class
SimpleMatrix deltaClass = goldClass >= 0 ? predictions.Minus(goldLabel).Scale(nodeWeight) : new SimpleMatrix(predictions.NumRows(), predictions.NumCols());
SimpleMatrix localCD = deltaClass.Mult(NeuralUtils.ConcatenateWithBias(currentVector).Transpose());
double error = -(NeuralUtils.ElementwiseApplyLog(predictions).ElementMult(goldLabel).ElementSum());
error = error * nodeWeight;
RNNCoreAnnotations.SetPredictionError(tree, error);
if (tree.IsPreTerminal())
{
// below us is a word vector
unaryCD[category] = unaryCD[category].Plus(localCD);
string word = tree.Children()[0].Label().Value();
word = model.GetVocabWord(word);
//SimpleMatrix currentVectorDerivative = NeuralUtils.elementwiseApplyTanhDerivative(currentVector);
//SimpleMatrix deltaFromClass = model.getUnaryClassification(category).transpose().mult(deltaClass);
//SimpleMatrix deltaFull = deltaFromClass.extractMatrix(0, model.op.numHid, 0, 1).plus(deltaUp);
//SimpleMatrix wordDerivative = deltaFull.elementMult(currentVectorDerivative);
//wordVectorD.put(word, wordVectorD.get(word).plus(wordDerivative));
SimpleMatrix currentVectorDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(currentVector);
SimpleMatrix deltaFromClass = model.GetUnaryClassification(category).Transpose().Mult(deltaClass);
deltaFromClass = deltaFromClass.ExtractMatrix(0, model.op.numHid, 0, 1).ElementMult(currentVectorDerivative);
SimpleMatrix deltaFull = deltaFromClass.Plus(deltaUp);
SimpleMatrix oldWordVectorD = wordVectorD[word];
if (oldWordVectorD == null)
{
wordVectorD[word] = deltaFull;
}
else
{
wordVectorD[word] = oldWordVectorD.Plus(deltaFull);
}
}
else
{
// Otherwise, this must be a binary node
string leftCategory = model.BasicCategory(tree.Children()[0].Label().Value());
string rightCategory = model.BasicCategory(tree.Children()[1].Label().Value());
if (model.op.combineClassification)
{
unaryCD[string.Empty] = unaryCD[string.Empty].Plus(localCD);
}
else
{
binaryCD.Put(leftCategory, rightCategory, binaryCD.Get(leftCategory, rightCategory).Plus(localCD));
}
SimpleMatrix currentVectorDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(currentVector);
SimpleMatrix deltaFromClass = model.GetBinaryClassification(leftCategory, rightCategory).Transpose().Mult(deltaClass);
deltaFromClass = deltaFromClass.ExtractMatrix(0, model.op.numHid, 0, 1).ElementMult(currentVectorDerivative);
SimpleMatrix deltaFull = deltaFromClass.Plus(deltaUp);
SimpleMatrix leftVector = RNNCoreAnnotations.GetNodeVector(tree.Children()[0]);
SimpleMatrix rightVector = RNNCoreAnnotations.GetNodeVector(tree.Children()[1]);
SimpleMatrix childrenVector = NeuralUtils.ConcatenateWithBias(leftVector, rightVector);
SimpleMatrix W_df = deltaFull.Mult(childrenVector.Transpose());
binaryTD.Put(leftCategory, rightCategory, binaryTD.Get(leftCategory, rightCategory).Plus(W_df));
SimpleMatrix deltaDown;
if (model.op.useTensors)
{
SimpleTensor Wt_df = GetTensorGradient(deltaFull, leftVector, rightVector);
binaryTensorTD.Put(leftCategory, rightCategory, binaryTensorTD.Get(leftCategory, rightCategory).Plus(Wt_df));
deltaDown = ComputeTensorDeltaDown(deltaFull, leftVector, rightVector, model.GetBinaryTransform(leftCategory, rightCategory), model.GetBinaryTensor(leftCategory, rightCategory));
}
else
{
deltaDown = model.GetBinaryTransform(leftCategory, rightCategory).Transpose().Mult(deltaFull);
}
SimpleMatrix leftDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(leftVector);
SimpleMatrix rightDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(rightVector);
SimpleMatrix leftDeltaDown = deltaDown.ExtractMatrix(0, deltaFull.NumRows(), 0, 1);
SimpleMatrix rightDeltaDown = deltaDown.ExtractMatrix(deltaFull.NumRows(), deltaFull.NumRows() * 2, 0, 1);
BackpropDerivativesAndError(tree.Children()[0], binaryTD, binaryCD, binaryTensorTD, unaryCD, wordVectorD, leftDerivative.ElementMult(leftDeltaDown));
BackpropDerivativesAndError(tree.Children()[1], binaryTD, binaryCD, binaryTensorTD, unaryCD, wordVectorD, rightDerivative.ElementMult(rightDeltaDown));
}
}
private void Awake()
{
Application.runInBackground = true;
var protoCreature = QuadrotorFactory.CreateCreature(
Vector3.zero,
Quaternion.identity);
var config = new FCNetworkConfig();
config.Layers.Add(new FCLayerConfig {
NumNeurons = protoCreature.NumInputs
});
config.Layers.Add(new FCLayerConfig {
NumNeurons = 30
});
config.Layers.Add(new FCLayerConfig {
NumNeurons = protoCreature.NumOutputs
});
Destroy(protoCreature.gameObject);
// Create creature bodies
_creatures = new List <NeuralCreature>(_populationSize);
for (int i = 0; i < _populationSize; i++)
{
NeuralCreature nc = new NeuralCreature();
Vector3 spawnPos = GetSpawnPosition(i, _populationSize);
nc.Body = QuadrotorFactory.CreateCreature(
spawnPos,
GetSpawnRotation());
nc.Mind = new FCNetwork(config);
NeuralUtils.Initialize(nc.Mind, ref _rng);
_creatures.Add(nc);
}
// Create a random genotype to seed the population
_genotype = new FCNetwork(config);
NeuralUtils.Initialize(_genotype, ref _rng);
PrepareNewEpisode();
MutatePopulation();
// Store initial pose so we can reuse creature gameplay objects across tests
_creatureInitPose = new List <ITransform>();
CreatureFactory.SerializePose(_creatures[0].Body, _creatureInitPose);
Physics.autoSimulation = false;
Physics.autoSyncTransforms = false;
var colliders = FindObjectsOfType <CapsuleCollider>();
for (int i = 0; i < colliders.Length; i++)
{
// Bug this doesn't work, why?
colliders[i].material = _physicsMaterial;
}
StartEpisode();
}
