private static SimpleMatrix ComputeTensorDeltaDown(SimpleMatrix deltaFull, SimpleMatrix leftVector, SimpleMatrix rightVector, SimpleMatrix W, SimpleTensor Wt)
        {
            SimpleMatrix WTDelta       = W.Transpose().Mult(deltaFull);
            SimpleMatrix WTDeltaNoBias = WTDelta.ExtractMatrix(0, deltaFull.NumRows() * 2, 0, 1);
            int          size          = deltaFull.GetNumElements();
            SimpleMatrix deltaTensor   = new SimpleMatrix(size * 2, 1);
            SimpleMatrix fullVector    = NeuralUtils.Concatenate(leftVector, rightVector);

            for (int slice = 0; slice < size; ++slice)
            {
                SimpleMatrix scaledFullVector = fullVector.Scale(deltaFull.Get(slice));
                deltaTensor = deltaTensor.Plus(Wt.GetSlice(slice).Plus(Wt.GetSlice(slice).Transpose()).Mult(scaledFullVector));
            }
            return(deltaTensor.Plus(WTDeltaNoBias));
        }
コード例 #2
0
        public static IDictionary <string, SimpleMatrix> AverageUnaryMatrices(IList <IDictionary <string, SimpleMatrix> > maps)
        {
            IDictionary <string, SimpleMatrix> averages = Generics.NewTreeMap();

            foreach (string name in GetUnaryMatrixNames(maps))
            {
                int          count  = 0;
                SimpleMatrix matrix = null;
                foreach (IDictionary <string, SimpleMatrix> map in maps)
                {
                    if (!map.Contains(name))
                    {
                        continue;
                    }
                    SimpleMatrix original = map[name];
                    ++count;
                    if (matrix == null)
                    {
                        matrix = original;
                    }
                    else
                    {
                        matrix = matrix.Plus(original);
                    }
                }
                matrix         = matrix.Divide(count);
                averages[name] = matrix;
            }
            return(averages);
        }
コード例 #3
0
        public static TwoDimensionalMap <string, string, SimpleMatrix> AverageBinaryMatrices(IList <TwoDimensionalMap <string, string, SimpleMatrix> > maps)
        {
            TwoDimensionalMap <string, string, SimpleMatrix> averages = TwoDimensionalMap.TreeMap();

            foreach (Pair <string, string> binary in GetBinaryMatrixNames(maps))
            {
                int          count  = 0;
                SimpleMatrix matrix = null;
                foreach (TwoDimensionalMap <string, string, SimpleMatrix> map in maps)
                {
                    if (!map.Contains(binary.First(), binary.Second()))
                    {
                        continue;
                    }
                    SimpleMatrix original = map.Get(binary.First(), binary.Second());
                    ++count;
                    if (matrix == null)
                    {
                        matrix = original;
                    }
                    else
                    {
                        matrix = matrix.Plus(original);
                    }
                }
                matrix = matrix.Divide(count);
                averages.Put(binary.First(), binary.Second(), matrix);
            }
            return(averages);
        }
コード例 #4
0
        /// <summary>Creates a random context matrix.</summary>
        /// <remarks>
        /// Creates a random context matrix.  This will be numRows x
        /// 2*numCols big.  These can be appended to the end of either a
        /// unary or binary transform matrix to get the transform matrix
        /// which uses context words.
        /// </remarks>
        private SimpleMatrix RandomContextMatrix()
        {
            SimpleMatrix matrix = new SimpleMatrix(numRows, numCols * 2);

            matrix.InsertIntoThis(0, 0, identity.Scale(op.trainOptions.scalingForInit * 0.1));
            matrix.InsertIntoThis(0, numCols, identity.Scale(op.trainOptions.scalingForInit * 0.1));
            matrix = matrix.Plus(SimpleMatrix.Random(numRows, numCols * 2, -1.0 / Math.Sqrt((double)numCols * 100.0), 1.0 / Math.Sqrt((double)numCols * 100.0), rand));
            return(matrix);
        }
コード例 #5
0
        private SimpleMatrix GetAverageEmbedding(IList <CoreLabel> words)
        {
            SimpleMatrix emb = new SimpleMatrix(staticWordEmbeddings.GetEmbeddingSize(), 1);

            foreach (CoreLabel word in words)
            {
                emb = emb.Plus(GetStaticWordEmbedding(word.Word()));
            }
            return(emb.Divide(Math.Max(1, words.Count)));
        }
コード例 #6
0
        public virtual double GetPairwiseScore(SimpleMatrix antecedentEmbedding, SimpleMatrix anaphorEmbedding, SimpleMatrix pairFeatures)
        {
            SimpleMatrix firstLayerOutput = NeuralUtils.ElementwiseApplyReLU(antecedentEmbedding.Plus(anaphorEmbedding).Plus(pairFeaturesMatrix.Mult(pairFeatures)).Plus(pairwiseFirstLayerBias));

            return(Score(firstLayerOutput, pairwiseModel));
        }
        private void BackpropDerivativesAndError(Tree tree, TwoDimensionalMap <string, string, SimpleMatrix> binaryTD, TwoDimensionalMap <string, string, SimpleMatrix> binaryCD, TwoDimensionalMap <string, string, SimpleTensor> binaryTensorTD, IDictionary
                                                 <string, SimpleMatrix> unaryCD, IDictionary <string, SimpleMatrix> wordVectorD, SimpleMatrix deltaUp)
        {
            if (tree.IsLeaf())
            {
                return;
            }
            SimpleMatrix currentVector = RNNCoreAnnotations.GetNodeVector(tree);
            string       category      = tree.Label().Value();

            category = model.BasicCategory(category);
            // Build a vector that looks like 0,0,1,0,0 with an indicator for the correct class
            SimpleMatrix goldLabel = new SimpleMatrix(model.numClasses, 1);
            int          goldClass = RNNCoreAnnotations.GetGoldClass(tree);

            if (goldClass >= 0)
            {
                goldLabel.Set(goldClass, 1.0);
            }
            double       nodeWeight  = model.op.trainOptions.GetClassWeight(goldClass);
            SimpleMatrix predictions = RNNCoreAnnotations.GetPredictions(tree);
            // If this is an unlabeled class, set deltaClass to 0.  We could
            // make this more efficient by eliminating various of the below
            // calculations, but this would be the easiest way to handle the
            // unlabeled class
            SimpleMatrix deltaClass = goldClass >= 0 ? predictions.Minus(goldLabel).Scale(nodeWeight) : new SimpleMatrix(predictions.NumRows(), predictions.NumCols());
            SimpleMatrix localCD    = deltaClass.Mult(NeuralUtils.ConcatenateWithBias(currentVector).Transpose());
            double       error      = -(NeuralUtils.ElementwiseApplyLog(predictions).ElementMult(goldLabel).ElementSum());

            error = error * nodeWeight;
            RNNCoreAnnotations.SetPredictionError(tree, error);
            if (tree.IsPreTerminal())
            {
                // below us is a word vector
                unaryCD[category] = unaryCD[category].Plus(localCD);
                string word = tree.Children()[0].Label().Value();
                word = model.GetVocabWord(word);
                //SimpleMatrix currentVectorDerivative = NeuralUtils.elementwiseApplyTanhDerivative(currentVector);
                //SimpleMatrix deltaFromClass = model.getUnaryClassification(category).transpose().mult(deltaClass);
                //SimpleMatrix deltaFull = deltaFromClass.extractMatrix(0, model.op.numHid, 0, 1).plus(deltaUp);
                //SimpleMatrix wordDerivative = deltaFull.elementMult(currentVectorDerivative);
                //wordVectorD.put(word, wordVectorD.get(word).plus(wordDerivative));
                SimpleMatrix currentVectorDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(currentVector);
                SimpleMatrix deltaFromClass          = model.GetUnaryClassification(category).Transpose().Mult(deltaClass);
                deltaFromClass = deltaFromClass.ExtractMatrix(0, model.op.numHid, 0, 1).ElementMult(currentVectorDerivative);
                SimpleMatrix deltaFull      = deltaFromClass.Plus(deltaUp);
                SimpleMatrix oldWordVectorD = wordVectorD[word];
                if (oldWordVectorD == null)
                {
                    wordVectorD[word] = deltaFull;
                }
                else
                {
                    wordVectorD[word] = oldWordVectorD.Plus(deltaFull);
                }
            }
            else
            {
                // Otherwise, this must be a binary node
                string leftCategory  = model.BasicCategory(tree.Children()[0].Label().Value());
                string rightCategory = model.BasicCategory(tree.Children()[1].Label().Value());
                if (model.op.combineClassification)
                {
                    unaryCD[string.Empty] = unaryCD[string.Empty].Plus(localCD);
                }
                else
                {
                    binaryCD.Put(leftCategory, rightCategory, binaryCD.Get(leftCategory, rightCategory).Plus(localCD));
                }
                SimpleMatrix currentVectorDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(currentVector);
                SimpleMatrix deltaFromClass          = model.GetBinaryClassification(leftCategory, rightCategory).Transpose().Mult(deltaClass);
                deltaFromClass = deltaFromClass.ExtractMatrix(0, model.op.numHid, 0, 1).ElementMult(currentVectorDerivative);
                SimpleMatrix deltaFull      = deltaFromClass.Plus(deltaUp);
                SimpleMatrix leftVector     = RNNCoreAnnotations.GetNodeVector(tree.Children()[0]);
                SimpleMatrix rightVector    = RNNCoreAnnotations.GetNodeVector(tree.Children()[1]);
                SimpleMatrix childrenVector = NeuralUtils.ConcatenateWithBias(leftVector, rightVector);
                SimpleMatrix W_df           = deltaFull.Mult(childrenVector.Transpose());
                binaryTD.Put(leftCategory, rightCategory, binaryTD.Get(leftCategory, rightCategory).Plus(W_df));
                SimpleMatrix deltaDown;
                if (model.op.useTensors)
                {
                    SimpleTensor Wt_df = GetTensorGradient(deltaFull, leftVector, rightVector);
                    binaryTensorTD.Put(leftCategory, rightCategory, binaryTensorTD.Get(leftCategory, rightCategory).Plus(Wt_df));
                    deltaDown = ComputeTensorDeltaDown(deltaFull, leftVector, rightVector, model.GetBinaryTransform(leftCategory, rightCategory), model.GetBinaryTensor(leftCategory, rightCategory));
                }
                else
                {
                    deltaDown = model.GetBinaryTransform(leftCategory, rightCategory).Transpose().Mult(deltaFull);
                }
                SimpleMatrix leftDerivative  = NeuralUtils.ElementwiseApplyTanhDerivative(leftVector);
                SimpleMatrix rightDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(rightVector);
                SimpleMatrix leftDeltaDown   = deltaDown.ExtractMatrix(0, deltaFull.NumRows(), 0, 1);
                SimpleMatrix rightDeltaDown  = deltaDown.ExtractMatrix(deltaFull.NumRows(), deltaFull.NumRows() * 2, 0, 1);
                BackpropDerivativesAndError(tree.Children()[0], binaryTD, binaryCD, binaryTensorTD, unaryCD, wordVectorD, leftDerivative.ElementMult(leftDeltaDown));
                BackpropDerivativesAndError(tree.Children()[1], binaryTD, binaryCD, binaryTensorTD, unaryCD, wordVectorD, rightDerivative.ElementMult(rightDeltaDown));
            }
        }
コード例 #8
0
        public virtual void BackpropDerivative(Tree tree, IList <string> words, IdentityHashMap <Tree, SimpleMatrix> nodeVectors, TwoDimensionalMap <string, string, SimpleMatrix> binaryW_dfs, IDictionary <string, SimpleMatrix> unaryW_dfs, TwoDimensionalMap
                                               <string, string, SimpleMatrix> binaryScoreDerivatives, IDictionary <string, SimpleMatrix> unaryScoreDerivatives, IDictionary <string, SimpleMatrix> wordVectorDerivatives, SimpleMatrix deltaUp)
        {
            if (tree.IsLeaf())
            {
                return;
            }
            if (tree.IsPreTerminal())
            {
                if (op.trainOptions.trainWordVectors)
                {
                    string word = tree.Children()[0].Label().Value();
                    word = dvModel.GetVocabWord(word);
                    //        SimpleMatrix currentVector = nodeVectors.get(tree);
                    //        SimpleMatrix currentVectorDerivative = nonlinearityVectorToDerivative(currentVector);
                    //        SimpleMatrix derivative = deltaUp.elementMult(currentVectorDerivative);
                    SimpleMatrix derivative = deltaUp;
                    wordVectorDerivatives[word] = wordVectorDerivatives[word].Plus(derivative);
                }
                return;
            }
            SimpleMatrix currentVector           = nodeVectors[tree];
            SimpleMatrix currentVectorDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(currentVector);
            SimpleMatrix scoreW = dvModel.GetScoreWForNode(tree);

            currentVectorDerivative = currentVectorDerivative.ElementMult(scoreW.Transpose());
            // the delta that is used at the current nodes
            SimpleMatrix deltaCurrent = deltaUp.Plus(currentVectorDerivative);
            SimpleMatrix W            = dvModel.GetWForNode(tree);
            SimpleMatrix WTdelta      = W.Transpose().Mult(deltaCurrent);

            if (tree.Children().Length == 2)
            {
                //TODO: RS: Change to the nice "getWForNode" setup?
                string leftLabel  = dvModel.BasicCategory(tree.Children()[0].Label().Value());
                string rightLabel = dvModel.BasicCategory(tree.Children()[1].Label().Value());
                binaryScoreDerivatives.Put(leftLabel, rightLabel, binaryScoreDerivatives.Get(leftLabel, rightLabel).Plus(currentVector.Transpose()));
                SimpleMatrix leftVector     = nodeVectors[tree.Children()[0]];
                SimpleMatrix rightVector    = nodeVectors[tree.Children()[1]];
                SimpleMatrix childrenVector = NeuralUtils.ConcatenateWithBias(leftVector, rightVector);
                if (op.trainOptions.useContextWords)
                {
                    childrenVector = ConcatenateContextWords(childrenVector, tree.GetSpan(), words);
                }
                SimpleMatrix W_df = deltaCurrent.Mult(childrenVector.Transpose());
                binaryW_dfs.Put(leftLabel, rightLabel, binaryW_dfs.Get(leftLabel, rightLabel).Plus(W_df));
                // and then recurse
                SimpleMatrix leftDerivative  = NeuralUtils.ElementwiseApplyTanhDerivative(leftVector);
                SimpleMatrix rightDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(rightVector);
                SimpleMatrix leftWTDelta     = WTdelta.ExtractMatrix(0, deltaCurrent.NumRows(), 0, 1);
                SimpleMatrix rightWTDelta    = WTdelta.ExtractMatrix(deltaCurrent.NumRows(), deltaCurrent.NumRows() * 2, 0, 1);
                BackpropDerivative(tree.Children()[0], words, nodeVectors, binaryW_dfs, unaryW_dfs, binaryScoreDerivatives, unaryScoreDerivatives, wordVectorDerivatives, leftDerivative.ElementMult(leftWTDelta));
                BackpropDerivative(tree.Children()[1], words, nodeVectors, binaryW_dfs, unaryW_dfs, binaryScoreDerivatives, unaryScoreDerivatives, wordVectorDerivatives, rightDerivative.ElementMult(rightWTDelta));
            }
            else
            {
                if (tree.Children().Length == 1)
                {
                    string childLabel = dvModel.BasicCategory(tree.Children()[0].Label().Value());
                    unaryScoreDerivatives[childLabel] = unaryScoreDerivatives[childLabel].Plus(currentVector.Transpose());
                    SimpleMatrix childVector         = nodeVectors[tree.Children()[0]];
                    SimpleMatrix childVectorWithBias = NeuralUtils.ConcatenateWithBias(childVector);
                    if (op.trainOptions.useContextWords)
                    {
                        childVectorWithBias = ConcatenateContextWords(childVectorWithBias, tree.GetSpan(), words);
                    }
                    SimpleMatrix W_df = deltaCurrent.Mult(childVectorWithBias.Transpose());
                    // System.out.println("unary backprop derivative for " + childLabel);
                    // System.out.println("Old transform:");
                    // System.out.println(unaryW_dfs.get(childLabel));
                    // System.out.println(" Delta:");
                    // System.out.println(W_df.scale(scale));
                    unaryW_dfs[childLabel] = unaryW_dfs[childLabel].Plus(W_df);
                    // and then recurse
                    SimpleMatrix childDerivative = NeuralUtils.ElementwiseApplyTanhDerivative(childVector);
                    //SimpleMatrix childDerivative = childVector;
                    SimpleMatrix childWTDelta = WTdelta.ExtractMatrix(0, deltaCurrent.NumRows(), 0, 1);
                    BackpropDerivative(tree.Children()[0], words, nodeVectors, binaryW_dfs, unaryW_dfs, binaryScoreDerivatives, unaryScoreDerivatives, wordVectorDerivatives, childDerivative.ElementMult(childWTDelta));
                }
            }
        }