internal virtual SimpleMatrix RandomTransformMatrix() { SimpleMatrix binary = new SimpleMatrix(numHid, numHid * 2 + 1); // bias column values are initialized zero binary.InsertIntoThis(0, 0, RandomTransformBlock()); binary.InsertIntoThis(0, numHid, RandomTransformBlock()); return(binary.Scale(op.trainOptions.scalingForInit)); }
/// <summary>Creates a random context matrix.</summary> /// <remarks> /// Creates a random context matrix. This will be numRows x /// 2*numCols big. These can be appended to the end of either a /// unary or binary transform matrix to get the transform matrix /// which uses context words. /// </remarks> private SimpleMatrix RandomContextMatrix() { SimpleMatrix matrix = new SimpleMatrix(numRows, numCols * 2); matrix.InsertIntoThis(0, 0, identity.Scale(op.trainOptions.scalingForInit * 0.1)); matrix.InsertIntoThis(0, numCols, identity.Scale(op.trainOptions.scalingForInit * 0.1)); matrix = matrix.Plus(SimpleMatrix.Random(numRows, numCols * 2, -1.0 / Math.Sqrt((double)numCols * 100.0), 1.0 / Math.Sqrt((double)numCols * 100.0), rand)); return(matrix); }
/// <summary>Returns matrices of the right size for either binary or unary (terminal) classification</summary> internal virtual SimpleMatrix RandomClassificationMatrix() { SimpleMatrix score = new SimpleMatrix(numClasses, numHid + 1); double range = 1.0 / (Math.Sqrt((double)numHid)); score.InsertIntoThis(0, 0, SimpleMatrix.Random(numClasses, numHid, -range, range, rand)); // bias column goes from 0 to 1 initially score.InsertIntoThis(0, numHid, SimpleMatrix.Random(numClasses, 1, 0.0, 1.0, rand)); return(score.Scale(op.trainOptions.scalingForInit)); }
private static double ScaleAndRegularize(IDictionary <string, SimpleMatrix> derivatives, IDictionary <string, SimpleMatrix> currentMatrices, double scale, double regCost, bool activeMatricesOnly, bool dropBiasColumn) { double cost = 0.0; // the regularization cost foreach (KeyValuePair <string, SimpleMatrix> entry in currentMatrices) { SimpleMatrix D = derivatives[entry.Key]; if (activeMatricesOnly && D == null) { // Fill in an emptpy matrix so the length of theta can match. // TODO: might want to allow for sparse parameter vectors derivatives[entry.Key] = new SimpleMatrix(entry.Value.NumRows(), entry.Value.NumCols()); continue; } SimpleMatrix regMatrix = entry.Value; if (dropBiasColumn) { regMatrix = new SimpleMatrix(regMatrix); regMatrix.InsertIntoThis(0, regMatrix.NumCols() - 1, new SimpleMatrix(regMatrix.NumRows(), 1)); } D = D.Scale(scale).Plus(regMatrix.Scale(regCost)); derivatives[entry.Key] = D; cost += regMatrix.ElementMult(regMatrix).ElementSum() * regCost / 2.0; } return(cost); }
public virtual void AddRandomBinaryMatrix(string leftBasic, string rightBasic) { if (binaryTransform.Get(leftBasic, rightBasic) != null) { return; } ++numBinaryMatrices; // scoring matrix SimpleMatrix score = SimpleMatrix.Random(1, numCols, -1.0 / Math.Sqrt((double)numCols), 1.0 / Math.Sqrt((double)numCols), rand); binaryScore.Put(leftBasic, rightBasic, score.Scale(op.trainOptions.scalingForInit)); SimpleMatrix binary; if (op.trainOptions.useContextWords) { binary = new SimpleMatrix(numRows, numCols * 4 + 1); // leave room for bias term binary.InsertIntoThis(0, numCols * 2 + 1, RandomContextMatrix()); } else { binary = new SimpleMatrix(numRows, numCols * 2 + 1); } SimpleMatrix left = RandomTransformMatrix(); SimpleMatrix right = RandomTransformMatrix(); binary.InsertIntoThis(0, 0, left); binary.InsertIntoThis(0, numCols, right); binaryTransform.Put(leftBasic, rightBasic, binary.Scale(op.trainOptions.scalingForInit)); }
public virtual void AddRandomUnaryMatrix(string childBasic) { if (unaryTransform[childBasic] != null) { return; } ++numUnaryMatrices; // scoring matrix SimpleMatrix score = SimpleMatrix.Random(1, numCols, -1.0 / Math.Sqrt((double)numCols), 1.0 / Math.Sqrt((double)numCols), rand); unaryScore[childBasic] = score.Scale(op.trainOptions.scalingForInit); SimpleMatrix transform; if (op.trainOptions.useContextWords) { transform = new SimpleMatrix(numRows, numCols * 3 + 1); // leave room for bias term transform.InsertIntoThis(0, numCols + 1, RandomContextMatrix()); } else { transform = new SimpleMatrix(numRows, numCols + 1); } SimpleMatrix unary = RandomTransformMatrix(); transform.InsertIntoThis(0, 0, unary); unaryTransform[childBasic] = transform.Scale(op.trainOptions.scalingForInit); }
/// <summary>Concatenates several column vectors into one large column vector</summary> public static SimpleMatrix Concatenate(params SimpleMatrix[] vectors) { int size = 0; foreach (SimpleMatrix vector in vectors) { size += vector.NumRows(); } SimpleMatrix result = new SimpleMatrix(size, 1); int index = 0; foreach (SimpleMatrix vector_1 in vectors) { result.InsertIntoThis(index, 0, vector_1); index += vector_1.NumRows(); } return(result); }
private static double ScaleAndRegularize(TwoDimensionalMap <string, string, SimpleMatrix> derivatives, TwoDimensionalMap <string, string, SimpleMatrix> currentMatrices, double scale, double regCost, bool dropBiasColumn) { double cost = 0.0; // the regularization cost foreach (TwoDimensionalMap.Entry <string, string, SimpleMatrix> entry in currentMatrices) { SimpleMatrix D = derivatives.Get(entry.GetFirstKey(), entry.GetSecondKey()); SimpleMatrix regMatrix = entry.GetValue(); if (dropBiasColumn) { regMatrix = new SimpleMatrix(regMatrix); regMatrix.InsertIntoThis(0, regMatrix.NumCols() - 1, new SimpleMatrix(regMatrix.NumRows(), 1)); } D = D.Scale(scale).Plus(regMatrix.Scale(regCost)); derivatives.Put(entry.GetFirstKey(), entry.GetSecondKey(), D); cost += regMatrix.ElementMult(regMatrix).ElementSum() * regCost / 2.0; } return(cost); }
/// <summary> /// Concatenates several column vectors into one large column /// vector, adds a 1.0 at the end as a bias term /// </summary> public static SimpleMatrix ConcatenateWithBias(params SimpleMatrix[] vectors) { int size = 0; foreach (SimpleMatrix vector in vectors) { size += vector.NumRows(); } // one extra for the bias size++; SimpleMatrix result = new SimpleMatrix(size, 1); int index = 0; foreach (SimpleMatrix vector_1 in vectors) { result.InsertIntoThis(index, 0, vector_1); index += vector_1.NumRows(); } result.Set(index, 0, 1.0); return(result); }