private static SimpleTensor GetTensorGradient(SimpleMatrix deltaFull, SimpleMatrix leftVector, SimpleMatrix rightVector)
{
int size = deltaFull.GetNumElements();
SimpleTensor Wt_df = new SimpleTensor(size * 2, size * 2, size);
// TODO: combine this concatenation with computeTensorDeltaDown?
SimpleMatrix fullVector = NeuralUtils.Concatenate(leftVector, rightVector);
for (int slice = 0; slice < size; ++slice)
{
Wt_df.SetSlice(slice, fullVector.Scale(deltaFull.Get(slice)).Mult(fullVector.Transpose()));
}
return(Wt_df);
}
private static SimpleMatrix ComputeTensorDeltaDown(SimpleMatrix deltaFull, SimpleMatrix leftVector, SimpleMatrix rightVector, SimpleMatrix W, SimpleTensor Wt)
{
SimpleMatrix WTDelta = W.Transpose().Mult(deltaFull);
SimpleMatrix WTDeltaNoBias = WTDelta.ExtractMatrix(0, deltaFull.NumRows() * 2, 0, 1);
int size = deltaFull.GetNumElements();
SimpleMatrix deltaTensor = new SimpleMatrix(size * 2, 1);
SimpleMatrix fullVector = NeuralUtils.Concatenate(leftVector, rightVector);
for (int slice = 0; slice < size; ++slice)
{
SimpleMatrix scaledFullVector = fullVector.Scale(deltaFull.Get(slice));
deltaTensor = deltaTensor.Plus(Wt.GetSlice(slice).Plus(Wt.GetSlice(slice).Transpose()).Mult(scaledFullVector));
}
return(deltaTensor.Plus(WTDeltaNoBias));
}
public virtual void TestCosine()
{
double[][] values = new double[][] { new double[5] };
values[0] = new double[] { 0.1, 0.2, 0.3, 0.4, 0.5 };
SimpleMatrix vector1 = new SimpleMatrix(values);
values[0] = new double[] { 0.5, 0.4, 0.3, 0.2, 0.1 };
SimpleMatrix vector2 = new SimpleMatrix(values);
NUnit.Framework.Assert.AreEqual(NeuralUtils.Dot(vector1, vector2), 1e-5, 0.35000000000000003);
NUnit.Framework.Assert.AreEqual(NeuralUtils.Cosine(vector1, vector2), 1e-5, 0.6363636363636364);
vector1 = vector1.Transpose();
vector2 = vector2.Transpose();
NUnit.Framework.Assert.AreEqual(NeuralUtils.Dot(vector1, vector2), 1e-5, 0.35000000000000003);
NUnit.Framework.Assert.AreEqual(NeuralUtils.Cosine(vector1, vector2), 1e-5, 0.6363636363636364);
}
/// <summary>Compute dot product between two vectors.</summary>
public static double Dot(SimpleMatrix vector1, SimpleMatrix vector2)
{
if (vector1.NumRows() == 1)
{
// vector1: row vector, assume that vector2 is a row vector too
return(vector1.Mult(vector2.Transpose()).Get(0));
}
else
{
if (vector1.NumCols() == 1)
{
// vector1: col vector, assume that vector2 is also a column vector.
return(vector1.Transpose().Mult(vector2).Get(0));
}
else
{
throw new AssertionError("Error in neural.Utils.dot: vector1 is a matrix " + vector1.NumRows() + " x " + vector1.NumCols());
}
}
}
/// <summary>
/// Returns a column vector where each entry is the nth bilinear
/// product of the nth slices of the two tensors.
/// </summary>
public virtual SimpleMatrix BilinearProducts(SimpleMatrix @in)
{
if (@in.NumCols() != 1)
{
throw new AssertionError("Expected a column vector");
}
if (@in.NumRows() != numCols)
{
throw new AssertionError("Number of rows in the input does not match number of columns in tensor");
}
if (numRows != numCols)
{
throw new AssertionError("Can only perform this operation on a SimpleTensor with square slices");
}
SimpleMatrix inT = @in.Transpose();
SimpleMatrix @out = new SimpleMatrix(numSlices, 1);
for (int slice = 0; slice < numSlices; ++slice)
{
double result = inT.Mult(slices[slice]).Mult(@in).Get(0);
@out.Set(slice, result);
}
return(@out);
}
private void BackpropDerivativesAndError(Tree tree, TwoDimensionalMap <string, string, SimpleMatrix> binaryTD, TwoDimensionalMap <string, string, SimpleMatrix> binaryCD, TwoDimensionalMap <string, string, SimpleTensor> binaryTensorTD, IDictionary
<string, SimpleMatrix> unaryCD, IDictionary <string, SimpleMatrix> wordVectorD, SimpleMatrix deltaUp)
{
if (tree.IsLeaf())
{
return;
}
SimpleMatrix currentVector = RNNCoreAnnotations.GetNodeVector(tree);
string category = tree.Label().Value();
category = model.BasicCategory(category);
// Build a vector that looks like 0,0,1,0,0 with an indicator for the correct class
SimpleMatrix goldLabel = new SimpleMatrix(model.numClasses, 1);
int goldClass = RNNCoreAnnotations.GetGoldClass(tree);
if (goldClass >= 0)
{
goldLabel.Set(goldClass, 1.0);
}
double nodeWeight = model.op.trainOptions.GetClassWeight(goldClass);
SimpleMatrix predictions = RNNCoreAnnotations.GetPredictions(tree);
// If this is an unlabeled class, set deltaClass to 0. We could
// make this more efficient by eliminating various of the below
// calculations, but this would be the easiest way to handle the
// unlabeled class
SimpleMatrix deltaClass = goldClass >= 0 ? predictions.Minus(goldLabel).Scale(nodeWeight) : new SimpleMatrix(predictions.NumRows(), predictions.NumCols());
SimpleMatrix localCD = deltaClass.Mult(NeuralUtils.ConcatenateWithBias(currentVector).Transpose());
double error = -(NeuralUtils.ElementwiseApplyLog(predictions).ElementMult(goldLabel).ElementSum());
error = error * nodeWeight;
RNNCoreAnnotations.SetPredictionError(tree, error);
if (tree.IsPreTerminal())
{
// below us is a word vector
unaryCD[category] = unaryCD[category].Plus(localCD);
string word = tree.Children()[0].Label().Value();
word = model.GetVocabWord(word);
//SimpleMatrix currentVectorDerivative = NeuralUtils.elementwiseApplyTanhDerivative(currentVector);
//SimpleMatrix deltaFromClass = model.getUnaryClassification(category).transpose().mult(deltaClass);
//SimpleMatrix deltaFull = deltaFromClass.extractMatrix(0, model.op.numHid, 0, 1).plus(deltaUp);
//SimpleMatrix wordDerivative = deltaFull.elementMult(currentVectorDerivative);
//wordVectorD.put(word, wordVectorD.get(word).plus(wordDerivative));
SimpleMatrix currentVectorDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(currentVector);
SimpleMatrix deltaFromClass = model.GetUnaryClassification(category).Transpose().Mult(deltaClass);
deltaFromClass = deltaFromClass.ExtractMatrix(0, model.op.numHid, 0, 1).ElementMult(currentVectorDerivative);
SimpleMatrix deltaFull = deltaFromClass.Plus(deltaUp);
SimpleMatrix oldWordVectorD = wordVectorD[word];
if (oldWordVectorD == null)
{
wordVectorD[word] = deltaFull;
}
else
{
wordVectorD[word] = oldWordVectorD.Plus(deltaFull);
}
}
else
{
// Otherwise, this must be a binary node
string leftCategory = model.BasicCategory(tree.Children()[0].Label().Value());
string rightCategory = model.BasicCategory(tree.Children()[1].Label().Value());
if (model.op.combineClassification)
{
unaryCD[string.Empty] = unaryCD[string.Empty].Plus(localCD);
}
else
{
binaryCD.Put(leftCategory, rightCategory, binaryCD.Get(leftCategory, rightCategory).Plus(localCD));
}
SimpleMatrix currentVectorDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(currentVector);
SimpleMatrix deltaFromClass = model.GetBinaryClassification(leftCategory, rightCategory).Transpose().Mult(deltaClass);
deltaFromClass = deltaFromClass.ExtractMatrix(0, model.op.numHid, 0, 1).ElementMult(currentVectorDerivative);
SimpleMatrix deltaFull = deltaFromClass.Plus(deltaUp);
SimpleMatrix leftVector = RNNCoreAnnotations.GetNodeVector(tree.Children()[0]);
SimpleMatrix rightVector = RNNCoreAnnotations.GetNodeVector(tree.Children()[1]);
SimpleMatrix childrenVector = NeuralUtils.ConcatenateWithBias(leftVector, rightVector);
SimpleMatrix W_df = deltaFull.Mult(childrenVector.Transpose());
binaryTD.Put(leftCategory, rightCategory, binaryTD.Get(leftCategory, rightCategory).Plus(W_df));
SimpleMatrix deltaDown;
if (model.op.useTensors)
{
SimpleTensor Wt_df = GetTensorGradient(deltaFull, leftVector, rightVector);
binaryTensorTD.Put(leftCategory, rightCategory, binaryTensorTD.Get(leftCategory, rightCategory).Plus(Wt_df));
deltaDown = ComputeTensorDeltaDown(deltaFull, leftVector, rightVector, model.GetBinaryTransform(leftCategory, rightCategory), model.GetBinaryTensor(leftCategory, rightCategory));
}
else
{
deltaDown = model.GetBinaryTransform(leftCategory, rightCategory).Transpose().Mult(deltaFull);
}
SimpleMatrix leftDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(leftVector);
SimpleMatrix rightDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(rightVector);
SimpleMatrix leftDeltaDown = deltaDown.ExtractMatrix(0, deltaFull.NumRows(), 0, 1);
SimpleMatrix rightDeltaDown = deltaDown.ExtractMatrix(deltaFull.NumRows(), deltaFull.NumRows() * 2, 0, 1);
BackpropDerivativesAndError(tree.Children()[0], binaryTD, binaryCD, binaryTensorTD, unaryCD, wordVectorD, leftDerivative.ElementMult(leftDeltaDown));
BackpropDerivativesAndError(tree.Children()[1], binaryTD, binaryCD, binaryTensorTD, unaryCD, wordVectorD, rightDerivative.ElementMult(rightDeltaDown));
}
}
public virtual void BackpropDerivative(Tree tree, IList <string> words, IdentityHashMap <Tree, SimpleMatrix> nodeVectors, TwoDimensionalMap <string, string, SimpleMatrix> binaryW_dfs, IDictionary <string, SimpleMatrix> unaryW_dfs, TwoDimensionalMap
<string, string, SimpleMatrix> binaryScoreDerivatives, IDictionary <string, SimpleMatrix> unaryScoreDerivatives, IDictionary <string, SimpleMatrix> wordVectorDerivatives, SimpleMatrix deltaUp)
{
if (tree.IsLeaf())
{
return;
}
if (tree.IsPreTerminal())
{
if (op.trainOptions.trainWordVectors)
{
string word = tree.Children()[0].Label().Value();
word = dvModel.GetVocabWord(word);
// SimpleMatrix currentVector = nodeVectors.get(tree);
// SimpleMatrix currentVectorDerivative = nonlinearityVectorToDerivative(currentVector);
// SimpleMatrix derivative = deltaUp.elementMult(currentVectorDerivative);
SimpleMatrix derivative = deltaUp;
wordVectorDerivatives[word] = wordVectorDerivatives[word].Plus(derivative);
}
return;
}
SimpleMatrix currentVector = nodeVectors[tree];
SimpleMatrix currentVectorDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(currentVector);
SimpleMatrix scoreW = dvModel.GetScoreWForNode(tree);
currentVectorDerivative = currentVectorDerivative.ElementMult(scoreW.Transpose());
// the delta that is used at the current nodes
SimpleMatrix deltaCurrent = deltaUp.Plus(currentVectorDerivative);
SimpleMatrix W = dvModel.GetWForNode(tree);
SimpleMatrix WTdelta = W.Transpose().Mult(deltaCurrent);
if (tree.Children().Length == 2)
{
//TODO: RS: Change to the nice "getWForNode" setup?
string leftLabel = dvModel.BasicCategory(tree.Children()[0].Label().Value());
string rightLabel = dvModel.BasicCategory(tree.Children()[1].Label().Value());
binaryScoreDerivatives.Put(leftLabel, rightLabel, binaryScoreDerivatives.Get(leftLabel, rightLabel).Plus(currentVector.Transpose()));
SimpleMatrix leftVector = nodeVectors[tree.Children()[0]];
SimpleMatrix rightVector = nodeVectors[tree.Children()[1]];
SimpleMatrix childrenVector = NeuralUtils.ConcatenateWithBias(leftVector, rightVector);
if (op.trainOptions.useContextWords)
{
childrenVector = ConcatenateContextWords(childrenVector, tree.GetSpan(), words);
}
SimpleMatrix W_df = deltaCurrent.Mult(childrenVector.Transpose());
binaryW_dfs.Put(leftLabel, rightLabel, binaryW_dfs.Get(leftLabel, rightLabel).Plus(W_df));
// and then recurse
SimpleMatrix leftDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(leftVector);
SimpleMatrix rightDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(rightVector);
SimpleMatrix leftWTDelta = WTdelta.ExtractMatrix(0, deltaCurrent.NumRows(), 0, 1);
SimpleMatrix rightWTDelta = WTdelta.ExtractMatrix(deltaCurrent.NumRows(), deltaCurrent.NumRows() * 2, 0, 1);
BackpropDerivative(tree.Children()[0], words, nodeVectors, binaryW_dfs, unaryW_dfs, binaryScoreDerivatives, unaryScoreDerivatives, wordVectorDerivatives, leftDerivative.ElementMult(leftWTDelta));
BackpropDerivative(tree.Children()[1], words, nodeVectors, binaryW_dfs, unaryW_dfs, binaryScoreDerivatives, unaryScoreDerivatives, wordVectorDerivatives, rightDerivative.ElementMult(rightWTDelta));
}
else
{
if (tree.Children().Length == 1)
{
string childLabel = dvModel.BasicCategory(tree.Children()[0].Label().Value());
unaryScoreDerivatives[childLabel] = unaryScoreDerivatives[childLabel].Plus(currentVector.Transpose());
SimpleMatrix childVector = nodeVectors[tree.Children()[0]];
SimpleMatrix childVectorWithBias = NeuralUtils.ConcatenateWithBias(childVector);
if (op.trainOptions.useContextWords)
{
childVectorWithBias = ConcatenateContextWords(childVectorWithBias, tree.GetSpan(), words);
}
SimpleMatrix W_df = deltaCurrent.Mult(childVectorWithBias.Transpose());
// System.out.println("unary backprop derivative for " + childLabel);
// System.out.println("Old transform:");
// System.out.println(unaryW_dfs.get(childLabel));
// System.out.println(" Delta:");
// System.out.println(W_df.scale(scale));
unaryW_dfs[childLabel] = unaryW_dfs[childLabel].Plus(W_df);
// and then recurse
SimpleMatrix childDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(childVector);
//SimpleMatrix childDerivative = childVector;
SimpleMatrix childWTDelta = WTdelta.ExtractMatrix(0, deltaCurrent.NumRows(), 0, 1);
BackpropDerivative(tree.Children()[0], words, nodeVectors, binaryW_dfs, unaryW_dfs, binaryScoreDerivatives, unaryScoreDerivatives, wordVectorDerivatives, childDerivative.ElementMult(childWTDelta));
}
}
}
static void buildCurvatureInfo()
{
if (inside_submesh_set[current_vertex])
{
ArrayList adjanced_vertices_idx = new ArrayList();
for (int i = 0; i < triangles.Length;)
{
int i1 = triangles[i++];
int i2 = triangles[i++];
int i3 = triangles[i++];
if (is_common_vertex(i1, current_vertex))
{
if (!adjanced_vertices_idx.Contains(i2))
{
adjanced_vertices_idx.Add(i2);
}
if (!adjanced_vertices_idx.Contains(i3))
{
adjanced_vertices_idx.Add(i3);
}
}
else if (is_common_vertex(i2, current_vertex))
{
if (!adjanced_vertices_idx.Contains(i1))
{
adjanced_vertices_idx.Add(i1);
}
if (!adjanced_vertices_idx.Contains(i3))
{
adjanced_vertices_idx.Add(i3);
}
}
else if (is_common_vertex(i3, current_vertex))
{
if (!adjanced_vertices_idx.Contains(i1))
{
adjanced_vertices_idx.Add(i1);
}
if (!adjanced_vertices_idx.Contains(i2))
{
adjanced_vertices_idx.Add(i2);
}
}
}
#if RECURSIVE_VERTICES_FLAG
ArrayList adjanced_vertices_idx2 = new ArrayList();
for (int k = 0; k < adjanced_vertices_idx.Count; k++)
{
for (int i = 0; i < triangles.Length;)
{
int i1 = triangles[i++];
int i2 = triangles[i++];
int i3 = triangles[i++];
if (is_common_vertex(i1, (int)adjanced_vertices_idx[k]))
{
if (!adjanced_vertices_idx2.Contains(i2))
{
adjanced_vertices_idx2.Add(i2);
}
if (!adjanced_vertices_idx2.Contains(i3))
{
adjanced_vertices_idx2.Add(i3);
}
}
else if (is_common_vertex(i2, (int)adjanced_vertices_idx[k]))
{
if (!adjanced_vertices_idx2.Contains(i1))
{
adjanced_vertices_idx2.Add(i1);
}
if (!adjanced_vertices_idx2.Contains(i3))
{
adjanced_vertices_idx2.Add(i3);
}
}
else if (is_common_vertex(i3, (int)adjanced_vertices_idx[k]))
{
if (!adjanced_vertices_idx2.Contains(i1))
{
adjanced_vertices_idx2.Add(i1);
}
if (!adjanced_vertices_idx2.Contains(i2))
{
adjanced_vertices_idx2.Add(i2);
}
}
}
}
adjanced_vertices_idx = adjanced_vertices_idx2;
#endif
Vector3 norm = normals[current_vertex];
Vector3 tangent = new Vector3(tangents[current_vertex].x, tangents[current_vertex].y, tangents[current_vertex].z);
Vector3 binorm = Vector3.Cross(norm, tangent) * tangents[current_vertex].w;
Matrix4x4 tR = Matrix4x4.identity;
tR.SetRow(0, new Vector4(tangent.x, tangent.y, tangent.z, 0));
tR.SetRow(1, new Vector4(binorm.x, binorm.y, binorm.z, 0));
tR.SetRow(2, new Vector4(norm.x, norm.y, norm.z, 0));
Vector3[] adjanced_vertices = new Vector3[adjanced_vertices_idx.Count + 1];
adjanced_vertices[0] = Vector3.zero;
for (int i = 1; i <= adjanced_vertices_idx.Count; i++)
{
adjanced_vertices[i] = tR.MultiplyPoint3x4((vertices[(int)adjanced_vertices_idx[i - 1]] - vertices[current_vertex]));
}
float scl_u = 1;
float scl_v = 1;
for (int i = 0; i < triangles.Length;)
{
int i1 = triangles[i++];
int i2 = triangles[i++];
int i3 = triangles[i++];
bool process_flag = false;
if (i1 == current_vertex)
{
process_flag = true;
}
else if (i2 == current_vertex)
{
i2 = i1;
process_flag = true;
}
else if (i3 == current_vertex)
{
i3 = i1;
process_flag = true;
}
if (process_flag)
{
Vector2 scl = get_UV2ObjectScale(current_vertex, i2, i3);
scl_u = scl.x;
scl_v = scl.y;
break;
}
}
SimpleMatrix M = new SimpleMatrix(adjanced_vertices.Length, 2);
SimpleMatrix b = new SimpleMatrix(adjanced_vertices.Length, 1);
for (int i = 0; i < adjanced_vertices.Length; i++)
{
M[i, 0] = adjanced_vertices[i].x * adjanced_vertices[i].x;
M[i, 1] = adjanced_vertices[i].y * adjanced_vertices[i].y;
b[i, 0] = -adjanced_vertices[i].z;
}
SimpleMatrix M_t = SimpleMatrix.Transpose(M);
SimpleMatrix M_inv = (M_t * M).Invert();
SimpleMatrix coeffs = (M_inv * M_t) * b;
curvature[current_vertex].x = (float)coeffs[0, 0];
curvature[current_vertex].y = (float)coeffs[1, 0];
// curvature[current_vertex].x=Mathf.Abs(curvature[current_vertex].x)>128 ? Mathf.Sign(curvature[current_vertex].x)*128 : curvature[current_vertex].x;
// curvature[current_vertex].y=Mathf.Abs(curvature[current_vertex].y)>128 ? Mathf.Sign(curvature[current_vertex].y)*128 : curvature[current_vertex].y;
// scl_u=scl_u > 255 ? 255 : scl_u;
// scl_v=scl_v > 255 ? 255 : scl_v;
uvs_scl[current_vertex].x = scl_u;
uvs_scl[current_vertex].y = scl_v;
}
current_vertex++;
if (current_vertex == vertices.Length)
{
for (int i = 0; i < vertices.Length; i++)
{
if (inside_submesh_set[i])
{
// Debug.Log(uvs_scl[i] +" "+curvature[i]);
// Debug.Log((Mathf.Round(uvs_scl[i].x*100)/100)+", "+(Mathf.Round(uvs_scl[i].y*100)/100)+" "+(Mathf.Round(curvature[i].x*100)/100)+","+(Mathf.Round(curvature[i].y*100)/100));
float CurvU = Mathf.Clamp(curvature[i].x, -19.9f, 19.9f);
float CurvV = Mathf.Clamp(curvature[i].y, -19.9f, 19.9f);
CurvU = CurvU / 20.0f + 0.5f; // curvature - stored in frac term
CurvV = CurvV / 20.0f + 0.5f;
float SclU = Mathf.Floor(uvs_scl[i].x * 100); // scale is always positive
float SclV = Mathf.Floor(uvs_scl[i].y * 100);
uvs4[i].x = SclU + CurvU;
uvs4[i].y = SclV + CurvV;
// SclU=Mathf.Floor(uvs4[i].x);
// CurvU=uvs4[i].x-SclU;
// SclU/=100;
// CurvU=CurvU*20-10;
//
// SclV=Mathf.Floor(uvs4[i].y);
// CurvV=uvs4[i].y-SclV;
// SclV/=100;
// CurvV=CurvV*20-10;
//
// Debug.Log((Mathf.Round(SclU*100)/100)+", "+(Mathf.Round(SclV*100)/100)+" "+(Mathf.Round(CurvU*100)/100)+","+(Mathf.Round(CurvV*100)/100));
}
}
//mesh.uv3=uvs3; // reserved for dyn lightmaps, we can't use it
// so, pack it into float2
mesh.uv4 = uvs4;
is_being_rendered = false;
//Repaint();
}
}