public virtual void AddRandomBinaryMatrix(string leftBasic, string rightBasic) { if (binaryTransform.Get(leftBasic, rightBasic) != null) { return; } ++numBinaryMatrices; // scoring matrix SimpleMatrix score = SimpleMatrix.Random(1, numCols, -1.0 / Math.Sqrt((double)numCols), 1.0 / Math.Sqrt((double)numCols), rand); binaryScore.Put(leftBasic, rightBasic, score.Scale(op.trainOptions.scalingForInit)); SimpleMatrix binary; if (op.trainOptions.useContextWords) { binary = new SimpleMatrix(numRows, numCols * 4 + 1); // leave room for bias term binary.InsertIntoThis(0, numCols * 2 + 1, RandomContextMatrix()); } else { binary = new SimpleMatrix(numRows, numCols * 2 + 1); } SimpleMatrix left = RandomTransformMatrix(); SimpleMatrix right = RandomTransformMatrix(); binary.InsertIntoThis(0, 0, left); binary.InsertIntoThis(0, numCols, right); binaryTransform.Put(leftBasic, rightBasic, binary.Scale(op.trainOptions.scalingForInit)); }
public virtual void AddRandomUnaryMatrix(string childBasic) { if (unaryTransform[childBasic] != null) { return; } ++numUnaryMatrices; // scoring matrix SimpleMatrix score = SimpleMatrix.Random(1, numCols, -1.0 / Math.Sqrt((double)numCols), 1.0 / Math.Sqrt((double)numCols), rand); unaryScore[childBasic] = score.Scale(op.trainOptions.scalingForInit); SimpleMatrix transform; if (op.trainOptions.useContextWords) { transform = new SimpleMatrix(numRows, numCols * 3 + 1); // leave room for bias term transform.InsertIntoThis(0, numCols + 1, RandomContextMatrix()); } else { transform = new SimpleMatrix(numRows, numCols + 1); } SimpleMatrix unary = RandomTransformMatrix(); transform.InsertIntoThis(0, 0, unary); unaryTransform[childBasic] = transform.Scale(op.trainOptions.scalingForInit); }
/// <summary>Creates a random context matrix.</summary> /// <remarks> /// Creates a random context matrix. This will be numRows x /// 2*numCols big. These can be appended to the end of either a /// unary or binary transform matrix to get the transform matrix /// which uses context words. /// </remarks> private SimpleMatrix RandomContextMatrix() { SimpleMatrix matrix = new SimpleMatrix(numRows, numCols * 2); matrix.InsertIntoThis(0, 0, identity.Scale(op.trainOptions.scalingForInit * 0.1)); matrix.InsertIntoThis(0, numCols, identity.Scale(op.trainOptions.scalingForInit * 0.1)); matrix = matrix.Plus(SimpleMatrix.Random(numRows, numCols * 2, -1.0 / Math.Sqrt((double)numCols * 100.0), 1.0 / Math.Sqrt((double)numCols * 100.0), rand)); return(matrix); }
/// <summary> /// Returns a randomly initialized tensor with values draft from the /// uniform distribution between minValue and maxValue. /// </summary> public static Edu.Stanford.Nlp.Neural.SimpleTensor Random(int numRows, int numCols, int numSlices, double minValue, double maxValue, Java.Util.Random rand) { Edu.Stanford.Nlp.Neural.SimpleTensor tensor = new Edu.Stanford.Nlp.Neural.SimpleTensor(numRows, numCols, numSlices); for (int i = 0; i < numSlices; ++i) { tensor.slices[i] = SimpleMatrix.Random(numRows, numCols, minValue, maxValue, rand); } return(tensor); }
/// <summary>Returns matrices of the right size for either binary or unary (terminal) classification</summary> internal virtual SimpleMatrix RandomClassificationMatrix() { SimpleMatrix score = new SimpleMatrix(numClasses, numHid + 1); double range = 1.0 / (Math.Sqrt((double)numHid)); score.InsertIntoThis(0, 0, SimpleMatrix.Random(numClasses, numHid, -range, range, rand)); // bias column goes from 0 to 1 initially score.InsertIntoThis(0, numHid, SimpleMatrix.Random(numClasses, 1, 0.0, 1.0, rand)); return(score.Scale(op.trainOptions.scalingForInit)); }
/// <summary> /// Create a random transform matrix based on the initialization /// parameters. /// </summary> /// <remarks> /// Create a random transform matrix based on the initialization /// parameters. This will be numRows x numCols big. These can be /// plugged into either unary or binary transform matrices. /// </remarks> private SimpleMatrix RandomTransformMatrix() { SimpleMatrix matrix; switch (op.trainOptions.transformMatrixType) { case TrainOptions.TransformMatrixType.Diagonal: { matrix = SimpleMatrix.Random(numRows, numCols, -1.0 / Math.Sqrt((double)numCols * 100.0), 1.0 / Math.Sqrt((double)numCols * 100.0), rand).Plus(identity); break; } case TrainOptions.TransformMatrixType.Random: { matrix = SimpleMatrix.Random(numRows, numCols, -1.0 / Math.Sqrt((double)numCols), 1.0 / Math.Sqrt((double)numCols), rand); break; } case TrainOptions.TransformMatrixType.OffDiagonal: { matrix = SimpleMatrix.Random(numRows, numCols, -1.0 / Math.Sqrt((double)numCols * 100.0), 1.0 / Math.Sqrt((double)numCols * 100.0), rand).Plus(identity); for (int i = 0; i < numCols; ++i) { int x = rand.NextInt(numCols); int y = rand.NextInt(numCols); int scale = rand.NextInt(3) - 1; // -1, 0, or 1 matrix.Set(x, y, matrix.Get(x, y) + scale); } break; } case TrainOptions.TransformMatrixType.RandomZeros: { matrix = SimpleMatrix.Random(numRows, numCols, -1.0 / Math.Sqrt((double)numCols * 100.0), 1.0 / Math.Sqrt((double)numCols * 100.0), rand).Plus(identity); for (int i_1 = 0; i_1 < numCols; ++i_1) { int x = rand.NextInt(numCols); int y = rand.NextInt(numCols); matrix.Set(x, y, 0.0); } break; } default: { throw new ArgumentException("Unexpected matrix initialization type " + op.trainOptions.transformMatrixType); } } return(matrix); }
internal virtual SimpleMatrix RandomTransformBlock() { double range = 1.0 / (Math.Sqrt((double)numHid) * 2.0); return(SimpleMatrix.Random(numHid, numHid, -range, range, rand).Plus(identity)); }
public virtual void ReadWordVectors() { SimpleMatrix unknownNumberVector = null; SimpleMatrix unknownCapsVector = null; SimpleMatrix unknownChineseYearVector = null; SimpleMatrix unknownChineseNumberVector = null; SimpleMatrix unknownChinesePercentVector = null; wordVectors = Generics.NewTreeMap(); int numberCount = 0; int capsCount = 0; int chineseYearCount = 0; int chineseNumberCount = 0; int chinesePercentCount = 0; //Map<String, SimpleMatrix> rawWordVectors = NeuralUtils.readRawWordVectors(op.lexOptions.wordVectorFile, op.lexOptions.numHid); Embedding rawWordVectors = new Embedding(op.lexOptions.wordVectorFile, op.lexOptions.numHid); foreach (string word in rawWordVectors.KeySet()) { SimpleMatrix vector = rawWordVectors.Get(word); if (op.wordFunction != null) { word = op.wordFunction.Apply(word); } wordVectors[word] = vector; if (op.lexOptions.numHid <= 0) { op.lexOptions.numHid = vector.GetNumElements(); } // TODO: factor out all of these identical blobs if (op.trainOptions.unknownNumberVector && (NumberPattern.Matcher(word).Matches() || DgPattern.Matcher(word).Matches())) { ++numberCount; if (unknownNumberVector == null) { unknownNumberVector = new SimpleMatrix(vector); } else { unknownNumberVector = unknownNumberVector.Plus(vector); } } if (op.trainOptions.unknownCapsVector && CapsPattern.Matcher(word).Matches()) { ++capsCount; if (unknownCapsVector == null) { unknownCapsVector = new SimpleMatrix(vector); } else { unknownCapsVector = unknownCapsVector.Plus(vector); } } if (op.trainOptions.unknownChineseYearVector && ChineseYearPattern.Matcher(word).Matches()) { ++chineseYearCount; if (unknownChineseYearVector == null) { unknownChineseYearVector = new SimpleMatrix(vector); } else { unknownChineseYearVector = unknownChineseYearVector.Plus(vector); } } if (op.trainOptions.unknownChineseNumberVector && (ChineseNumberPattern.Matcher(word).Matches() || DgPattern.Matcher(word).Matches())) { ++chineseNumberCount; if (unknownChineseNumberVector == null) { unknownChineseNumberVector = new SimpleMatrix(vector); } else { unknownChineseNumberVector = unknownChineseNumberVector.Plus(vector); } } if (op.trainOptions.unknownChinesePercentVector && ChinesePercentPattern.Matcher(word).Matches()) { ++chinesePercentCount; if (unknownChinesePercentVector == null) { unknownChinesePercentVector = new SimpleMatrix(vector); } else { unknownChinesePercentVector = unknownChinesePercentVector.Plus(vector); } } } string unkWord = op.trainOptions.unkWord; if (op.wordFunction != null) { unkWord = op.wordFunction.Apply(unkWord); } SimpleMatrix unknownWordVector = wordVectors[unkWord]; wordVectors[UnknownWord] = unknownWordVector; if (unknownWordVector == null) { throw new Exception("Unknown word vector not specified in the word vector file"); } if (op.trainOptions.unknownNumberVector) { if (numberCount > 0) { unknownNumberVector = unknownNumberVector.Divide(numberCount); } else { unknownNumberVector = new SimpleMatrix(unknownWordVector); } wordVectors[UnknownNumber] = unknownNumberVector; } if (op.trainOptions.unknownCapsVector) { if (capsCount > 0) { unknownCapsVector = unknownCapsVector.Divide(capsCount); } else { unknownCapsVector = new SimpleMatrix(unknownWordVector); } wordVectors[UnknownCaps] = unknownCapsVector; } if (op.trainOptions.unknownChineseYearVector) { log.Info("Matched " + chineseYearCount + " chinese year vectors"); if (chineseYearCount > 0) { unknownChineseYearVector = unknownChineseYearVector.Divide(chineseYearCount); } else { unknownChineseYearVector = new SimpleMatrix(unknownWordVector); } wordVectors[UnknownChineseYear] = unknownChineseYearVector; } if (op.trainOptions.unknownChineseNumberVector) { log.Info("Matched " + chineseNumberCount + " chinese number vectors"); if (chineseNumberCount > 0) { unknownChineseNumberVector = unknownChineseNumberVector.Divide(chineseNumberCount); } else { unknownChineseNumberVector = new SimpleMatrix(unknownWordVector); } wordVectors[UnknownChineseNumber] = unknownChineseNumberVector; } if (op.trainOptions.unknownChinesePercentVector) { log.Info("Matched " + chinesePercentCount + " chinese percent vectors"); if (chinesePercentCount > 0) { unknownChinesePercentVector = unknownChinesePercentVector.Divide(chinesePercentCount); } else { unknownChinesePercentVector = new SimpleMatrix(unknownWordVector); } wordVectors[UnknownChinesePercent] = unknownChinesePercentVector; } if (op.trainOptions.useContextWords) { SimpleMatrix start = SimpleMatrix.Random(op.lexOptions.numHid, 1, -0.5, 0.5, rand); SimpleMatrix end = SimpleMatrix.Random(op.lexOptions.numHid, 1, -0.5, 0.5, rand); wordVectors[StartWord] = start; wordVectors[EndWord] = end; } }
/// <exception cref="System.IO.IOException"/> public static void Main(string[] args) { string basePath = "/user/socherr/scr/projects/semComp/RNTN/src/params/"; int numSlices = 25; bool useEscapedParens = false; for (int argIndex = 0; argIndex < args.Length;) { if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-slices")) { numSlices = System.Convert.ToInt32(args[argIndex + 1]); argIndex += 2; } else { if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-path")) { basePath = args[argIndex + 1]; argIndex += 2; } else { if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-useEscapedParens")) { useEscapedParens = true; argIndex += 1; } else { log.Info("Unknown argument " + args[argIndex]); System.Environment.Exit(2); } } } } SimpleMatrix[] slices = new SimpleMatrix[numSlices]; for (int i = 0; i < numSlices; ++i) { slices[i] = LoadMatrix(basePath + "bin/Wt_" + (i + 1) + ".bin", basePath + "Wt_" + (i + 1) + ".txt"); } SimpleTensor tensor = new SimpleTensor(slices); log.Info("W tensor size: " + tensor.NumRows() + "x" + tensor.NumCols() + "x" + tensor.NumSlices()); SimpleMatrix W = LoadMatrix(basePath + "bin/W.bin", basePath + "W.txt"); log.Info("W matrix size: " + W.NumRows() + "x" + W.NumCols()); SimpleMatrix Wcat = LoadMatrix(basePath + "bin/Wcat.bin", basePath + "Wcat.txt"); log.Info("W cat size: " + Wcat.NumRows() + "x" + Wcat.NumCols()); SimpleMatrix combinedWV = LoadMatrix(basePath + "bin/Wv.bin", basePath + "Wv.txt"); log.Info("Word matrix size: " + combinedWV.NumRows() + "x" + combinedWV.NumCols()); File vocabFile = new File(basePath + "vocab_1.txt"); if (!vocabFile.Exists()) { vocabFile = new File(basePath + "words.txt"); } IList <string> lines = Generics.NewArrayList(); foreach (string line in IOUtils.ReadLines(vocabFile)) { lines.Add(line.Trim()); } log.Info("Lines in vocab file: " + lines.Count); IDictionary <string, SimpleMatrix> wordVectors = Generics.NewTreeMap(); for (int i_1 = 0; i_1 < lines.Count && i_1 < combinedWV.NumCols(); ++i_1) { string[] pieces = lines[i_1].Split(" +"); if (pieces.Length == 0 || pieces.Length > 1) { continue; } wordVectors[pieces[0]] = combinedWV.ExtractMatrix(0, numSlices, i_1, i_1 + 1); if (pieces[0].Equals("UNK")) { wordVectors[SentimentModel.UnknownWord] = wordVectors["UNK"]; } } // If there is no ",", we first try to look for an HTML escaping, // then fall back to "." as better than just a random word vector. // Same for "``" and ";" CopyWordVector(wordVectors, ",", ","); CopyWordVector(wordVectors, ".", ","); CopyWordVector(wordVectors, ";", ";"); CopyWordVector(wordVectors, ".", ";"); CopyWordVector(wordVectors, "``", "``"); CopyWordVector(wordVectors, "''", "``"); if (useEscapedParens) { ReplaceWordVector(wordVectors, "(", "-LRB-"); ReplaceWordVector(wordVectors, ")", "-RRB-"); } RNNOptions op = new RNNOptions(); op.numHid = numSlices; op.lowercaseWordVectors = false; if (Wcat.NumRows() == 2) { op.classNames = new string[] { "Negative", "Positive" }; op.equivalenceClasses = new int[][] { new int[] { 0 }, new int[] { 1 } }; // TODO: set to null once old models are updated op.numClasses = 2; } if (!wordVectors.Contains(SentimentModel.UnknownWord)) { wordVectors[SentimentModel.UnknownWord] = SimpleMatrix.Random(numSlices, 1, -0.00001, 0.00001, new Random()); } SentimentModel model = SentimentModel.ModelFromMatrices(W, Wcat, tensor, wordVectors, op); model.SaveSerialized("matlab.ser.gz"); }