public static SimpleMatrix OneHot(int index, int size)
{
SimpleMatrix m = new SimpleMatrix(size, 1);
m.Set(index, 1);
return(m);
}
/// <summary>Applies the derivative of tanh to each of the elements in the vector.</summary>
/// <remarks>Applies the derivative of tanh to each of the elements in the vector. Returns a new matrix.</remarks>
public static SimpleMatrix ElementwiseApplyTanhDerivative(SimpleMatrix input)
{
SimpleMatrix output = new SimpleMatrix(input.NumRows(), input.NumCols());
output.Set(1.0);
output = output.Minus(input.ElementMult(input));
return(output);
}
/// <summary>Returns a vector with random Gaussian values, mean 0, std 1</summary>
public static SimpleMatrix RandomGaussian(int numRows, int numCols, Random rand)
{
SimpleMatrix result = new SimpleMatrix(numRows, numCols);
for (int i = 0; i < numRows; ++i)
{
for (int j = 0; j < numCols; ++j)
{
result.Set(i, j, rand.NextGaussian());
}
}
return(result);
}
/// <summary>Applies tanh to each of the entries in the matrix.</summary>
/// <remarks>Applies tanh to each of the entries in the matrix. Returns a new matrix.</remarks>
public static SimpleMatrix ElementwiseApplyTanh(SimpleMatrix input)
{
SimpleMatrix output = new SimpleMatrix(input);
for (int i = 0; i < output.NumRows(); ++i)
{
for (int j = 0; j < output.NumCols(); ++j)
{
output.Set(i, j, Math.Tanh(output.Get(i, j)));
}
}
return(output);
}
public virtual SimpleMatrix GetPairFeatures(Pair <int, int> pair, Document document, IDictionary <int, IList <Mention> > mentionsByHeadIndex)
{
Mention m1 = document.predictedMentionsByID[pair.first];
Mention m2 = document.predictedMentionsByID[pair.second];
IList <int> featureVals = PairwiseFeatures(document, m1, m2, dictionaries, conll);
SimpleMatrix features = new SimpleMatrix(featureVals.Count, 1);
for (int i = 0; i < featureVals.Count; i++)
{
features.Set(i, featureVals[i]);
}
features = NeuralUtils.Concatenate(features, EncodeDistance(m2.sentNum - m1.sentNum), EncodeDistance(m2.mentionNum - m1.mentionNum - 1), new SimpleMatrix(new double[][] { new double[] { m1.sentNum == m2.sentNum && m1.endIndex > m2.startIndex
? 1 : 0 } }), GetMentionFeatures(m1, document, mentionsByHeadIndex), GetMentionFeatures(m2, document, mentionsByHeadIndex), EncodeGenre(document));
return(features);
}
/// <summary>Applies softmax to all of the elements of the matrix.</summary>
/// <remarks>
/// Applies softmax to all of the elements of the matrix. The return
/// matrix will have all of its elements sum to 1. If your matrix is
/// not already a vector, be sure this is what you actually want.
/// </remarks>
public static SimpleMatrix Softmax(SimpleMatrix input)
{
SimpleMatrix output = new SimpleMatrix(input);
for (int i = 0; i < output.NumRows(); ++i)
{
for (int j = 0; j < output.NumCols(); ++j)
{
output.Set(i, j, Math.Exp(output.Get(i, j)));
}
}
double sum = output.ElementSum();
// will be safe, since exp should never return 0
return(output.Scale(1.0 / sum));
}
private static SimpleMatrix EncodeDistance(int d)
{
SimpleMatrix m = new SimpleMatrix(11, 1);
if (d < 5)
{
m.Set(d, 1);
}
else
{
if (d < 8)
{
m.Set(5, 1);
}
else
{
if (d < 16)
{
m.Set(6, 1);
}
else
{
if (d < 32)
{
m.Set(7, 1);
}
else
{
if (d < 64)
{
m.Set(8, 1);
}
else
{
m.Set(9, 1);
}
}
}
}
}
m.Set(10, Math.Min(d, 64) / 64.0);
return(m);
}
/// <summary>
/// Concatenates several column vectors into one large column
/// vector, adds a 1.0 at the end as a bias term
/// </summary>
public static SimpleMatrix ConcatenateWithBias(params SimpleMatrix[] vectors)
{
int size = 0;
foreach (SimpleMatrix vector in vectors)
{
size += vector.NumRows();
}
// one extra for the bias
size++;
SimpleMatrix result = new SimpleMatrix(size, 1);
int index = 0;
foreach (SimpleMatrix vector_1 in vectors)
{
result.InsertIntoThis(index, 0, vector_1);
index += vector_1.NumRows();
}
result.Set(index, 0, 1.0);
return(result);
}
public static void VectorToParams(double[] theta, params IEnumerator <SimpleMatrix>[] matrices)
{
int index = 0;
foreach (IEnumerator <SimpleMatrix> matrixIterator in matrices)
{
while (matrixIterator.MoveNext())
{
SimpleMatrix matrix = matrixIterator.Current;
int numElements = matrix.GetNumElements();
for (int i = 0; i < numElements; ++i)
{
matrix.Set(i, theta[index]);
++index;
}
}
}
if (index != theta.Length)
{
throw new AssertionError("Did not entirely use the theta vector");
}
}
/// <summary>
/// Returns a column vector where each entry is the nth bilinear
/// product of the nth slices of the two tensors.
/// </summary>
public virtual SimpleMatrix BilinearProducts(SimpleMatrix @in)
{
if (@in.NumCols() != 1)
{
throw new AssertionError("Expected a column vector");
}
if (@in.NumRows() != numCols)
{
throw new AssertionError("Number of rows in the input does not match number of columns in tensor");
}
if (numRows != numCols)
{
throw new AssertionError("Can only perform this operation on a SimpleTensor with square slices");
}
SimpleMatrix inT = @in.Transpose();
SimpleMatrix @out = new SimpleMatrix(numSlices, 1);
for (int slice = 0; slice < numSlices; ++slice)
{
double result = inT.Mult(slices[slice]).Mult(@in).Get(0);
@out.Set(slice, result);
}
return(@out);
}
private void BackpropDerivativesAndError(Tree tree, TwoDimensionalMap <string, string, SimpleMatrix> binaryTD, TwoDimensionalMap <string, string, SimpleMatrix> binaryCD, TwoDimensionalMap <string, string, SimpleTensor> binaryTensorTD, IDictionary
<string, SimpleMatrix> unaryCD, IDictionary <string, SimpleMatrix> wordVectorD, SimpleMatrix deltaUp)
{
if (tree.IsLeaf())
{
return;
}
SimpleMatrix currentVector = RNNCoreAnnotations.GetNodeVector(tree);
string category = tree.Label().Value();
category = model.BasicCategory(category);
// Build a vector that looks like 0,0,1,0,0 with an indicator for the correct class
SimpleMatrix goldLabel = new SimpleMatrix(model.numClasses, 1);
int goldClass = RNNCoreAnnotations.GetGoldClass(tree);
if (goldClass >= 0)
{
goldLabel.Set(goldClass, 1.0);
}
double nodeWeight = model.op.trainOptions.GetClassWeight(goldClass);
SimpleMatrix predictions = RNNCoreAnnotations.GetPredictions(tree);
// If this is an unlabeled class, set deltaClass to 0. We could
// make this more efficient by eliminating various of the below
// calculations, but this would be the easiest way to handle the
// unlabeled class
SimpleMatrix deltaClass = goldClass >= 0 ? predictions.Minus(goldLabel).Scale(nodeWeight) : new SimpleMatrix(predictions.NumRows(), predictions.NumCols());
SimpleMatrix localCD = deltaClass.Mult(NeuralUtils.ConcatenateWithBias(currentVector).Transpose());
double error = -(NeuralUtils.ElementwiseApplyLog(predictions).ElementMult(goldLabel).ElementSum());
error = error * nodeWeight;
RNNCoreAnnotations.SetPredictionError(tree, error);
if (tree.IsPreTerminal())
{
// below us is a word vector
unaryCD[category] = unaryCD[category].Plus(localCD);
string word = tree.Children()[0].Label().Value();
word = model.GetVocabWord(word);
//SimpleMatrix currentVectorDerivative = NeuralUtils.elementwiseApplyTanhDerivative(currentVector);
//SimpleMatrix deltaFromClass = model.getUnaryClassification(category).transpose().mult(deltaClass);
//SimpleMatrix deltaFull = deltaFromClass.extractMatrix(0, model.op.numHid, 0, 1).plus(deltaUp);
//SimpleMatrix wordDerivative = deltaFull.elementMult(currentVectorDerivative);
//wordVectorD.put(word, wordVectorD.get(word).plus(wordDerivative));
SimpleMatrix currentVectorDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(currentVector);
SimpleMatrix deltaFromClass = model.GetUnaryClassification(category).Transpose().Mult(deltaClass);
deltaFromClass = deltaFromClass.ExtractMatrix(0, model.op.numHid, 0, 1).ElementMult(currentVectorDerivative);
SimpleMatrix deltaFull = deltaFromClass.Plus(deltaUp);
SimpleMatrix oldWordVectorD = wordVectorD[word];
if (oldWordVectorD == null)
{
wordVectorD[word] = deltaFull;
}
else
{
wordVectorD[word] = oldWordVectorD.Plus(deltaFull);
}
}
else
{
// Otherwise, this must be a binary node
string leftCategory = model.BasicCategory(tree.Children()[0].Label().Value());
string rightCategory = model.BasicCategory(tree.Children()[1].Label().Value());
if (model.op.combineClassification)
{
unaryCD[string.Empty] = unaryCD[string.Empty].Plus(localCD);
}
else
{
binaryCD.Put(leftCategory, rightCategory, binaryCD.Get(leftCategory, rightCategory).Plus(localCD));
}
SimpleMatrix currentVectorDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(currentVector);
SimpleMatrix deltaFromClass = model.GetBinaryClassification(leftCategory, rightCategory).Transpose().Mult(deltaClass);
deltaFromClass = deltaFromClass.ExtractMatrix(0, model.op.numHid, 0, 1).ElementMult(currentVectorDerivative);
SimpleMatrix deltaFull = deltaFromClass.Plus(deltaUp);
SimpleMatrix leftVector = RNNCoreAnnotations.GetNodeVector(tree.Children()[0]);
SimpleMatrix rightVector = RNNCoreAnnotations.GetNodeVector(tree.Children()[1]);
SimpleMatrix childrenVector = NeuralUtils.ConcatenateWithBias(leftVector, rightVector);
SimpleMatrix W_df = deltaFull.Mult(childrenVector.Transpose());
binaryTD.Put(leftCategory, rightCategory, binaryTD.Get(leftCategory, rightCategory).Plus(W_df));
SimpleMatrix deltaDown;
if (model.op.useTensors)
{
SimpleTensor Wt_df = GetTensorGradient(deltaFull, leftVector, rightVector);
binaryTensorTD.Put(leftCategory, rightCategory, binaryTensorTD.Get(leftCategory, rightCategory).Plus(Wt_df));
deltaDown = ComputeTensorDeltaDown(deltaFull, leftVector, rightVector, model.GetBinaryTransform(leftCategory, rightCategory), model.GetBinaryTensor(leftCategory, rightCategory));
}
else
{
deltaDown = model.GetBinaryTransform(leftCategory, rightCategory).Transpose().Mult(deltaFull);
}
SimpleMatrix leftDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(leftVector);
SimpleMatrix rightDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(rightVector);
SimpleMatrix leftDeltaDown = deltaDown.ExtractMatrix(0, deltaFull.NumRows(), 0, 1);
SimpleMatrix rightDeltaDown = deltaDown.ExtractMatrix(deltaFull.NumRows(), deltaFull.NumRows() * 2, 0, 1);
BackpropDerivativesAndError(tree.Children()[0], binaryTD, binaryCD, binaryTensorTD, unaryCD, wordVectorD, leftDerivative.ElementMult(leftDeltaDown));
BackpropDerivativesAndError(tree.Children()[1], binaryTD, binaryCD, binaryTensorTD, unaryCD, wordVectorD, rightDerivative.ElementMult(rightDeltaDown));
}
}
public MatrixDisplayControl()
{
InitializeComponent();
Matrix = new SimpleMatrix();
Matrix.Set(Microsoft.Xna.Framework.Matrix.Identity);
}
