/// <summary>
        /// Slowest possible way to calculate a derivative for a model: exhaustive definitional calculation, using the super
        /// slow logLikelihood function from this test suite.
        /// </summary>
        /// <param name="model">the model the get the derivative for</param>
        /// <param name="weights">the weights to get the derivative at</param>
        /// <returns>the derivative of the log likelihood with respect to the weights</returns>
        private ConcatVector DefinitionOfDerivative(GraphicalModel model, ConcatVector weights)
        {
            double       epsilon      = 1.0e-7;
            ConcatVector goldGradient = new ConcatVector(ConcatVecComponents);

            for (int i = 0; i < ConcatVecComponents; i++)
            {
                double[] component = new double[ConcatVecComponentLength];
                for (int j = 0; j < ConcatVecComponentLength; j++)
                {
                    // Create a unit vector pointing in the direction of this element of this component
                    ConcatVector unitVectorIJ = new ConcatVector(ConcatVecComponents);
                    unitVectorIJ.SetSparseComponent(i, j, 1.0);
                    // Create a +eps weight vector
                    ConcatVector weightsPlusEpsilon = weights.DeepClone();
                    weightsPlusEpsilon.AddVectorInPlace(unitVectorIJ, epsilon);
                    // Create a -eps weight vector
                    ConcatVector weightsMinusEpsilon = weights.DeepClone();
                    weightsMinusEpsilon.AddVectorInPlace(unitVectorIJ, -epsilon);
                    // Use the definition (f(x+eps) - f(x-eps))/(2*eps)
                    component[j] = (LogLikelihood(model, weightsPlusEpsilon) - LogLikelihood(model, weightsMinusEpsilon)) / (2 * epsilon);
                    // If we encounter an impossible assignment, logLikelihood will return negative infinity, which will
                    // screw with the definitional calculation
                    if (double.IsNaN(component[j]))
                    {
                        component[j] = 0.0;
                    }
                }
                goldGradient.SetDenseComponent(i, component);
            }
            return(goldGradient);
        }
        /// <summary>The slowest, but obviously correct way to get log likelihood.</summary>
        /// <remarks>
        /// The slowest, but obviously correct way to get log likelihood. We've already tested the partition function in
        /// the CliqueTreeTest, but in the interest of making things as different as possible to catch any lurking bugs or
        /// numerical issues, we use the brute force approach here.
        /// </remarks>
        /// <param name="model">the model to get the log-likelihood of, assumes labels for assignments</param>
        /// <param name="weights">the weights to get the log-likelihood at</param>
        /// <returns>the log-likelihood</returns>
        private double LogLikelihood(GraphicalModel model, ConcatVector weights)
        {
            ICollection <TableFactor> tableFactors = model.factors.Stream().Map(null).Collect(Collectors.ToSet());

            System.Diagnostics.Debug.Assert((tableFactors.Count == model.factors.Count));
            // this is the super slow but obviously correct way to get global marginals
            TableFactor bruteForce = null;

            foreach (TableFactor factor in tableFactors)
            {
                if (bruteForce == null)
                {
                    bruteForce = factor;
                }
                else
                {
                    bruteForce = bruteForce.Multiply(factor);
                }
            }
            System.Diagnostics.Debug.Assert((bruteForce != null));
            // observe out all variables that have been registered
            TableFactor observed = bruteForce;

            foreach (int n in bruteForce.neighborIndices)
            {
                if (model.GetVariableMetaDataByReference(n).Contains(CliqueTree.VariableObservedValue))
                {
                    int value = System.Convert.ToInt32(model.GetVariableMetaDataByReference(n)[CliqueTree.VariableObservedValue]);
                    if (observed.neighborIndices.Length > 1)
                    {
                        observed = observed.Observe(n, value);
                    }
                    else
                    {
                        // If we've observed everything, then just quit
                        return(0.0);
                    }
                }
            }
            bruteForce = observed;
            // Now we can get a partition function
            double partitionFunction = bruteForce.ValueSum();

            // For now, we'll assume that all the variables are given for training. EM is another problem altogether
            int[] assignment = new int[bruteForce.neighborIndices.Length];
            for (int i = 0; i < assignment.Length; i++)
            {
                System.Diagnostics.Debug.Assert((!model.GetVariableMetaDataByReference(bruteForce.neighborIndices[i]).Contains(CliqueTree.VariableObservedValue)));
                assignment[i] = System.Convert.ToInt32(model.GetVariableMetaDataByReference(bruteForce.neighborIndices[i])[LogLikelihoodDifferentiableFunction.VariableTrainingValue]);
            }
            if (bruteForce.GetAssignmentValue(assignment) == 0 || partitionFunction == 0)
            {
                return(double.NegativeInfinity);
            }
            return(Math.Log(bruteForce.GetAssignmentValue(assignment)) - Math.Log(partitionFunction));
        }
        public virtual void TestCalculateMarginals(GraphicalModel model, ConcatVector weights)
        {
            CliqueTree inference = new CliqueTree(model, weights);

            // This is the basic check that inference works when you first construct the model
            CheckMarginalsAgainstBruteForce((GraphicalModel)model, (ConcatVector)weights, inference);
            // Now we go through several random mutations to the model, and check that everything is still consistent
            Random r = new Random();

            for (int i = 0; i < 10; i++)
            {
                RandomlyMutateGraphicalModel((GraphicalModel)model, r);
                CheckMarginalsAgainstBruteForce((GraphicalModel)model, (ConcatVector)weights, inference);
            }
        }
 /// <summary>
 /// This lets the system allocate work to threads evenly, which reduces the amount of blocking and can improve
 /// runtimes by 20% or more.
 /// </summary>
 /// <param name="datum">the datum to estimate work for</param>
 /// <returns>a work estimate, on a relative scale of single cpu wall time, for getting the gradient and log-likelihood</returns>
 private int EstimateRelativeRuntime(T datum)
 {
     if (datum is GraphicalModel)
     {
         int            cost  = 0;
         GraphicalModel model = (GraphicalModel)datum;
         foreach (GraphicalModel.Factor f in model.factors)
         {
             cost += f.featuresTable.CombinatorialNeighborStatesCount();
         }
         return(cost);
     }
     else
     {
         return(1);
     }
 }
 public override GraphicalModel[] Generate(SourceOfRandomness sourceOfRandomness, IGenerationStatus generationStatus)
 {
     GraphicalModel[] dataset = new GraphicalModel[sourceOfRandomness.NextInt(1, 10)];
     for (int i = 0; i < dataset.Length; i++)
     {
         dataset[i] = modelGenerator.Generate(sourceOfRandomness, generationStatus);
         foreach (GraphicalModel.Factor f in dataset[i].factors)
         {
             for (int j = 0; j < f.neigborIndices.Length; j++)
             {
                 int n   = f.neigborIndices[j];
                 int dim = f.featuresTable.GetDimensions()[j];
                 dataset[i].GetVariableMetaDataByReference(n)[LogLikelihoodDifferentiableFunction.VariableTrainingValue] = string.Empty + sourceOfRandomness.NextInt(dim);
             }
         }
     }
     return(dataset);
 }
Example #6
0
 public static void Annotate(GraphicalModel model, IList <string> tags, ConcatVectorNamespace @namespace, IDictionary <string, double[]> embeddings)
 {
     for (int i = 0; i < model.variableMetaData.Count; i++)
     {
         IDictionary <string, string> metadata = model.GetVariableMetaDataByReference(i);
         string token = metadata["TOKEN"];
         string pos   = metadata["POS"];
         string chunk = metadata["CHUNK"];
         IDictionary <string, string> leftMetadata = null;
         if (i > 0)
         {
             leftMetadata = model.GetVariableMetaDataByReference(i - 1);
         }
         string leftToken = (leftMetadata == null) ? "^" : leftMetadata["TOKEN"];
         string leftPos   = (leftMetadata == null) ? "^" : leftMetadata["POS"];
         string leftChunk = (leftMetadata == null) ? "^" : leftMetadata["CHUNK"];
         IDictionary <string, string> rightMetadata = null;
         if (i < model.variableMetaData.Count - 1)
         {
             rightMetadata = model.GetVariableMetaDataByReference(i + 1);
         }
         string rightToken = (rightMetadata == null) ? "$" : rightMetadata["TOKEN"];
         string rightPos   = (rightMetadata == null) ? "$" : rightMetadata["POS"];
         string rightChunk = (rightMetadata == null) ? "$" : rightMetadata["CHUNK"];
         // Add the unary factor
         GraphicalModel.Factor f = model.AddFactor(new int[] { i }, new int[] { tags.Count }, null);
         // This is the anonymous function that generates a feature vector for each assignment to the unary
         // factor
         System.Diagnostics.Debug.Assert((f.neigborIndices.Length == 1));
         System.Diagnostics.Debug.Assert((f.neigborIndices[0] == i));
         // If this is not the last variable, add a binary factor
         if (i < model.variableMetaData.Count - 1)
         {
             GraphicalModel.Factor jf = model.AddFactor(new int[] { i, i + 1 }, new int[] { tags.Count, tags.Count }, null);
             // This is the anonymous function that generates a feature vector for every joint assignment to the
             // binary factor
             System.Diagnostics.Debug.Assert((jf.neigborIndices.Length == 2));
             System.Diagnostics.Debug.Assert((jf.neigborIndices[0] == i));
             System.Diagnostics.Debug.Assert((jf.neigborIndices[1] == i + 1));
         }
     }
 }
        public virtual GraphicalModel GenerateSentenceModel(ConcatVectorNamespace @namespace, CoNLLBenchmark.CoNLLSentence sentence, IList <string> tags)
        {
            GraphicalModel model = new GraphicalModel();

            for (int i = 0; i < sentence.token.Count; i++)
            {
                // Add the training label
                IDictionary <string, string> metadata = model.GetVariableMetaDataByReference(i);
                metadata[LogLikelihoodDifferentiableFunction.VariableTrainingValue] = string.Empty + tags.IndexOf(sentence.ner[i]);
                metadata["TOKEN"] = string.Empty + sentence.token[i];
                metadata["POS"]   = string.Empty + sentence.pos[i];
                metadata["CHUNK"] = string.Empty + sentence.npchunk[i];
                metadata["TAG"]   = string.Empty + sentence.ner[i];
            }
            CoNLLFeaturizer.Annotate(model, tags, @namespace, embeddings);
            System.Diagnostics.Debug.Assert((model.factors != null));
            foreach (GraphicalModel.Factor f in model.factors)
            {
                System.Diagnostics.Debug.Assert((f != null));
            }
            return(model);
        }
Example #8
0
 /// <summary>
 /// 绘制图形
 /// </summary>
 public override void Drawing()
 {
     if (OwnerChart == null || !OwnerChart.IsLoaded ||
         OwnerChart.XAxis == null || !OwnerChart.XAxis.IsLoaded || OwnerChart.XAxis.ActualWidth == 0 ||
         OwnerChart.YAxis == null || !OwnerChart.YAxis.IsLoaded || OwnerChart.XAxis.ActualHeight == 0)
     {
         return;
     }
     if (GraphicalShape == null)
     {
         Binding binding = new Binding("Points")
         {
             Source = this.GraphicalModel, Converter = GraphicalPointsConverter, ConverterParameter = OwnerChart, Mode = BindingMode.TwoWay
         };
         var temp = new Polygon()
         {
             FillRule = FillRule.Nonzero, Cursor = Cursors.Hand
         };
         temp.SetBinding(Polygon.PointsProperty, binding);
         GraphicalShape = temp;
     }
     GraphicalModel.OnPropertyChanged("Points");
 }
        private void CheckMarginalsAgainstBruteForce(GraphicalModel model, ConcatVector weights, CliqueTree inference)
        {
            CliqueTree.MarginalResult result       = inference.CalculateMarginals();
            double[][] marginals                   = result.marginals;
            ICollection <TableFactor> tableFactors = model.factors.Stream().Map(null).Collect(Collectors.ToSet());

            System.Diagnostics.Debug.Assert((tableFactors.Count == model.factors.Count));
            // this is the super slow but obviously correct way to get global marginals
            TableFactor bruteForce = null;

            foreach (TableFactor factor in tableFactors)
            {
                if (bruteForce == null)
                {
                    bruteForce = factor;
                }
                else
                {
                    bruteForce = bruteForce.Multiply(factor);
                }
            }
            if (bruteForce != null)
            {
                // observe out all variables that have been registered
                TableFactor observed = bruteForce;
                for (int i = 0; i < bruteForce.neighborIndices.Length; i++)
                {
                    int n = bruteForce.neighborIndices[i];
                    if (model.GetVariableMetaDataByReference(n).Contains(CliqueTree.VariableObservedValue))
                    {
                        int value = System.Convert.ToInt32(model.GetVariableMetaDataByReference(n)[CliqueTree.VariableObservedValue]);
                        // Check that the marginals reflect the observation
                        for (int j = 0; j < marginals[n].Length; j++)
                        {
                            NUnit.Framework.Assert.AreEqual(marginals[n][j], 1.0e-9, j == value ? 1.0 : 0.0);
                        }
                        if (observed.neighborIndices.Length > 1)
                        {
                            observed = observed.Observe(n, value);
                        }
                        else
                        {
                            // If we've observed everything, then just quit
                            return;
                        }
                    }
                }
                bruteForce = observed;
                // Spot check each of the marginals in the brute force calculation
                double[][] bruteMarginals = bruteForce.GetSummedMarginals();
                int        index          = 0;
                foreach (int i_1 in bruteForce.neighborIndices)
                {
                    bool     isEqual = true;
                    double[] brute   = bruteMarginals[index];
                    index++;
                    System.Diagnostics.Debug.Assert((brute != null));
                    System.Diagnostics.Debug.Assert((marginals[i_1] != null));
                    for (int j = 0; j < brute.Length; j++)
                    {
                        if (double.IsNaN(brute[j]))
                        {
                            isEqual = false;
                            break;
                        }
                        if (Math.Abs(brute[j] - marginals[i_1][j]) > 3.0e-2)
                        {
                            isEqual = false;
                            break;
                        }
                    }
                    if (!isEqual)
                    {
                        System.Console.Error.WriteLine("Arrays not equal! Variable " + i_1);
                        System.Console.Error.WriteLine("\tGold: " + Arrays.ToString(brute));
                        System.Console.Error.WriteLine("\tResult: " + Arrays.ToString(marginals[i_1]));
                    }
                    Assert.AssertArrayEquals(marginals[i_1], 3.0e-2, brute);
                }
                // Spot check the partition function
                double goldPartitionFunction = bruteForce.ValueSum();
                // Correct to within 3%
                NUnit.Framework.Assert.AreEqual(result.partitionFunction, goldPartitionFunction * 3.0e-2, goldPartitionFunction);
                // Check the joint marginals
                foreach (GraphicalModel.Factor f in model.factors)
                {
                    NUnit.Framework.Assert.IsTrue(result.jointMarginals.Contains(f));
                    TableFactor bruteForceJointMarginal = bruteForce;
                    foreach (int n in bruteForce.neighborIndices)
                    {
                        foreach (int i_2 in f.neigborIndices)
                        {
                            if (i_2 == n)
                            {
                                goto outer_continue;
                            }
                        }
                        if (bruteForceJointMarginal.neighborIndices.Length > 1)
                        {
                            bruteForceJointMarginal = bruteForceJointMarginal.SumOut(n);
                        }
                        else
                        {
                            int[] fixedAssignment = new int[f.neigborIndices.Length];
                            for (int i_3 = 0; i_3 < fixedAssignment.Length; i_3++)
                            {
                                fixedAssignment[i_3] = System.Convert.ToInt32(model.GetVariableMetaDataByReference(f.neigborIndices[i_3])[CliqueTree.VariableObservedValue]);
                            }
                            foreach (int[] assn in result.jointMarginals[f])
                            {
                                if (Arrays.Equals(assn, fixedAssignment))
                                {
                                    NUnit.Framework.Assert.AreEqual(result.jointMarginals[f].GetAssignmentValue(assn), 1.0e-7, 1.0);
                                }
                                else
                                {
                                    if (result.jointMarginals[f].GetAssignmentValue(assn) != 0)
                                    {
                                        TableFactor j = result.jointMarginals[f];
                                        foreach (int[] assignment in j)
                                        {
                                            System.Console.Error.WriteLine(Arrays.ToString(assignment) + ": " + j.GetAssignmentValue(assignment));
                                        }
                                    }
                                    NUnit.Framework.Assert.AreEqual(result.jointMarginals[f].GetAssignmentValue(assn), 1.0e-7, 0.0);
                                }
                            }
                            goto marginals_continue;
                        }
                    }
                    outer_break :;
                    // Find the correspondence between the brute force joint marginal, which may be missing variables
                    // because they were observed out of the table, and the output joint marginals, which are always an exact
                    // match for the original factor
                    int[] backPointers  = new int[f.neigborIndices.Length];
                    int[] observedValue = new int[f.neigborIndices.Length];
                    for (int i_4 = 0; i_4 < backPointers.Length; i_4++)
                    {
                        if (model.GetVariableMetaDataByReference(f.neigborIndices[i_4]).Contains(CliqueTree.VariableObservedValue))
                        {
                            observedValue[i_4] = System.Convert.ToInt32(model.GetVariableMetaDataByReference(f.neigborIndices[i_4])[CliqueTree.VariableObservedValue]);
                            backPointers[i_4]  = -1;
                        }
                        else
                        {
                            observedValue[i_4] = -1;
                            backPointers[i_4]  = -1;
                            for (int j = 0; j < bruteForceJointMarginal.neighborIndices.Length; j++)
                            {
                                if (bruteForceJointMarginal.neighborIndices[j] == f.neigborIndices[i_4])
                                {
                                    backPointers[i_4] = j;
                                }
                            }
                            System.Diagnostics.Debug.Assert((backPointers[i_4] != -1));
                        }
                    }
                    double sum = bruteForceJointMarginal.ValueSum();
                    if (sum == 0.0)
                    {
                        sum = 1;
                    }
                    foreach (int[] assignment_1 in result.jointMarginals[f])
                    {
                        int[] bruteForceMarginalAssignment = new int[bruteForceJointMarginal.neighborIndices.Length];
                        for (int i_2 = 0; i_2 < assignment_1.Length; i_2++)
                        {
                            if (backPointers[i_2] != -1)
                            {
                                bruteForceMarginalAssignment[backPointers[i_2]] = assignment_1[i_2];
                            }
                            else
                            {
                                // Make sure all assignments that don't square with observations get 0 weight
                                System.Diagnostics.Debug.Assert((observedValue[i_2] != -1));
                                if (assignment_1[i_2] != observedValue[i_2])
                                {
                                    if (result.jointMarginals[f].GetAssignmentValue(assignment_1) != 0)
                                    {
                                        System.Console.Error.WriteLine("Joint marginals: " + Arrays.ToString(result.jointMarginals[f].neighborIndices));
                                        System.Console.Error.WriteLine("Assignment: " + Arrays.ToString(assignment_1));
                                        System.Console.Error.WriteLine("Observed Value: " + Arrays.ToString(observedValue));
                                        foreach (int[] assn in result.jointMarginals[f])
                                        {
                                            System.Console.Error.WriteLine("\t" + Arrays.ToString(assn) + ":" + result.jointMarginals[f].GetAssignmentValue(assn));
                                        }
                                    }
                                    NUnit.Framework.Assert.AreEqual(result.jointMarginals[f].GetAssignmentValue(assignment_1), 1.0e-7, 0.0);
                                    goto outer_continue;
                                }
                            }
                        }
                        NUnit.Framework.Assert.AreEqual(result.jointMarginals[f].GetAssignmentValue(assignment_1), 1.0e-3, bruteForceJointMarginal.GetAssignmentValue(bruteForceMarginalAssignment) / sum);
                    }
                    outer_break :;
                }
                marginals_break :;
            }
            else
            {
                foreach (double[] marginal in marginals)
                {
                    foreach (double d in marginal)
                    {
                        NUnit.Framework.Assert.AreEqual(d, 3.0e-2, 1.0 / marginal.Length);
                    }
                }
            }
        }
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.TypeLoadException"/>
        public static void Main(string[] args)
        {
            //////////////////////////////////////////////////////////////
            // Generate the CoNLL CliqueTrees to use during gameplay
            //////////////////////////////////////////////////////////////
            CoNLLBenchmark coNLL = new CoNLLBenchmark();
            IList <CoNLLBenchmark.CoNLLSentence> train   = coNLL.GetSentences(DataPath + "conll.iob.4class.train");
            IList <CoNLLBenchmark.CoNLLSentence> testA   = coNLL.GetSentences(DataPath + "conll.iob.4class.testa");
            IList <CoNLLBenchmark.CoNLLSentence> testB   = coNLL.GetSentences(DataPath + "conll.iob.4class.testb");
            IList <CoNLLBenchmark.CoNLLSentence> allData = new List <CoNLLBenchmark.CoNLLSentence>();

            Sharpen.Collections.AddAll(allData, train);
            Sharpen.Collections.AddAll(allData, testA);
            Sharpen.Collections.AddAll(allData, testB);
            ICollection <string> tagsSet = new HashSet <string>();

            foreach (CoNLLBenchmark.CoNLLSentence sentence in allData)
            {
                foreach (string nerTag in sentence.ner)
                {
                    tagsSet.Add(nerTag);
                }
            }
            IList <string> tags = new List <string>();

            Sharpen.Collections.AddAll(tags, tagsSet);
            coNLL.embeddings = coNLL.GetEmbeddings(DataPath + "google-300-trimmed.ser.gz", allData);
            log.Info("Making the training set...");
            ConcatVectorNamespace @namespace = new ConcatVectorNamespace();
            int trainSize = train.Count;

            GraphicalModel[] trainingSet = new GraphicalModel[trainSize];
            for (int i = 0; i < trainSize; i++)
            {
                if (i % 10 == 0)
                {
                    log.Info(i + "/" + trainSize);
                }
                trainingSet[i] = coNLL.GenerateSentenceModel(@namespace, train[i], tags);
            }
            //////////////////////////////////////////////////////////////
            // Generate the random human observation feature vectors that we'll use
            //////////////////////////////////////////////////////////////
            Random r             = new Random(10);
            int    numFeatures   = 5;
            int    featureLength = 30;

            ConcatVector[] humanFeatureVectors = new ConcatVector[1000];
            for (int i_1 = 0; i_1 < humanFeatureVectors.Length; i_1++)
            {
                humanFeatureVectors[i_1] = new ConcatVector(numFeatures);
                for (int j = 0; j < numFeatures; j++)
                {
                    if (r.NextBoolean())
                    {
                        humanFeatureVectors[i_1].SetSparseComponent(j, r.NextInt(featureLength), r.NextDouble());
                    }
                    else
                    {
                        double[] dense = new double[featureLength];
                        for (int k = 0; k < dense.Length; k++)
                        {
                            dense[k] = r.NextDouble();
                        }
                        humanFeatureVectors[i_1].SetDenseComponent(j, dense);
                    }
                }
            }
            ConcatVector weights = new ConcatVector(numFeatures);

            for (int i_2 = 0; i_2 < numFeatures; i_2++)
            {
                double[] dense = new double[featureLength];
                for (int j = 0; j < dense.Length; j++)
                {
                    dense[j] = r.NextDouble();
                }
                weights.SetDenseComponent(i_2, dense);
            }
            //////////////////////////////////////////////////////////////
            // Actually perform gameplay-like random mutations
            //////////////////////////////////////////////////////////////
            log.Info("Warming up the JIT...");
            for (int i_3 = 0; i_3 < 10; i_3++)
            {
                log.Info(i_3);
                Gameplay(r, trainingSet[i_3], weights, humanFeatureVectors);
            }
            log.Info("Timing actual run...");
            long start = Runtime.CurrentTimeMillis();

            for (int i_4 = 0; i_4 < 10; i_4++)
            {
                log.Info(i_4);
                Gameplay(r, trainingSet[i_4], weights, humanFeatureVectors);
            }
            long duration = Runtime.CurrentTimeMillis() - start;

            log.Info("Duration: " + duration);
        }
 /// <summary>See push() for an explanation.</summary>
 /// <param name="model">the model to pop this SampleState from</param>
 public virtual void Pop(GraphicalModel model)
 {
     System.Diagnostics.Debug.Assert((model.factors.Contains(addedFactor)));
     model.factors.Remove(addedFactor);
     Sharpen.Collections.Remove(model.GetVariableMetaDataByReference(variable), CliqueTree.VariableObservedValue);
 }
 /// <summary>This applies this SampleState to the model.</summary>
 /// <remarks>
 /// This applies this SampleState to the model. The name comes from an analogy to a stack. If we take a sample
 /// path, involving a number of steps through the model, we push() each SampleState onto the model one at a time,
 /// then when we return from the sample we can pop() each SampleState off the model, and be left with our
 /// original model state.
 /// </remarks>
 /// <param name="model">the model to push this SampleState onto</param>
 public virtual void Push(GraphicalModel model)
 {
     System.Diagnostics.Debug.Assert((!model.factors.Contains(addedFactor)));
     model.factors.Add(addedFactor);
     model.GetVariableMetaDataByReference(variable)[CliqueTree.VariableObservedValue] = string.Empty + observation;
 }
        private static GamePlayerBenchmark.SampleState SelectOrCreateChildAtRandom(Random r, GraphicalModel model, int[] variables, int[] variableSizes, IList <GamePlayerBenchmark.SampleState> children, ConcatVector[] humanFeatureVectors)
        {
            int i           = r.NextInt(variables.Length);
            int variable    = variables[i];
            int observation = r.NextInt(variableSizes[i]);

            foreach (GamePlayerBenchmark.SampleState s in children)
            {
                if (s.variable == variable && s.observation == observation)
                {
                    return(s);
                }
            }
            int humanObservationVariable = 0;

            foreach (GraphicalModel.Factor f in model.factors)
            {
                foreach (int j in f.neigborIndices)
                {
                    if (j >= humanObservationVariable)
                    {
                        humanObservationVariable = j + 1;
                    }
                }
            }
            GraphicalModel.Factor f_1 = model.AddFactor(new int[] { variable, humanObservationVariable }, new int[] { variableSizes[i], variableSizes[i] }, null);
            model.factors.Remove(f_1);
            GamePlayerBenchmark.SampleState newState = new GamePlayerBenchmark.SampleState(f_1, variable, observation);
            children.Add(newState);
            return(newState);
        }
        /// <exception cref="System.Exception"/>
        public virtual void BenchmarkOptimizer()
        {
            IList <CoNLLBenchmark.CoNLLSentence> train   = GetSentences(DataPath + "conll.iob.4class.train");
            IList <CoNLLBenchmark.CoNLLSentence> testA   = GetSentences(DataPath + "conll.iob.4class.testa");
            IList <CoNLLBenchmark.CoNLLSentence> testB   = GetSentences(DataPath + "conll.iob.4class.testb");
            IList <CoNLLBenchmark.CoNLLSentence> allData = new List <CoNLLBenchmark.CoNLLSentence>();

            Sharpen.Collections.AddAll(allData, train);
            Sharpen.Collections.AddAll(allData, testA);
            Sharpen.Collections.AddAll(allData, testB);
            ICollection <string> tagsSet = new HashSet <string>();

            foreach (CoNLLBenchmark.CoNLLSentence sentence in allData)
            {
                foreach (string nerTag in sentence.ner)
                {
                    tagsSet.Add(nerTag);
                }
            }
            IList <string> tags = new List <string>();

            Sharpen.Collections.AddAll(tags, tagsSet);
            embeddings = GetEmbeddings(DataPath + "google-300-trimmed.ser.gz", allData);
            log.Info("Making the training set...");
            ConcatVectorNamespace @namespace = new ConcatVectorNamespace();
            int trainSize = train.Count;

            GraphicalModel[] trainingSet = new GraphicalModel[trainSize];
            for (int i = 0; i < trainSize; i++)
            {
                if (i % 10 == 0)
                {
                    log.Info(i + "/" + trainSize);
                }
                trainingSet[i] = GenerateSentenceModel(@namespace, train[i], tags);
            }
            log.Info("Training system...");
            AbstractBatchOptimizer opt = new BacktrackingAdaGradOptimizer();
            // This training call is basically what we want the benchmark for. It should take 99% of the wall clock time
            ConcatVector weights = opt.Optimize(trainingSet, new LogLikelihoodDifferentiableFunction(), @namespace.NewWeightsVector(), 0.01, 1.0e-5, false);

            log.Info("Testing system...");
            // Evaluation method lifted from the CoNLL 2004 perl script
            IDictionary <string, double> correctChunk = new Dictionary <string, double>();
            IDictionary <string, double> foundCorrect = new Dictionary <string, double>();
            IDictionary <string, double> foundGuessed = new Dictionary <string, double>();
            double correct = 0.0;
            double total   = 0.0;

            foreach (CoNLLBenchmark.CoNLLSentence sentence_1 in testA)
            {
                GraphicalModel model      = GenerateSentenceModel(@namespace, sentence_1, tags);
                int[]          guesses    = new CliqueTree(model, weights).CalculateMAP();
                string[]       nerGuesses = new string[guesses.Length];
                for (int i_1 = 0; i_1 < guesses.Length; i_1++)
                {
                    nerGuesses[i_1] = tags[guesses[i_1]];
                    if (nerGuesses[i_1].Equals(sentence_1.ner[i_1]))
                    {
                        correct++;
                        correctChunk[nerGuesses[i_1]] = correctChunk.GetOrDefault(nerGuesses[i_1], 0.0) + 1;
                    }
                    total++;
                    foundCorrect[sentence_1.ner[i_1]] = foundCorrect.GetOrDefault(sentence_1.ner[i_1], 0.0) + 1;
                    foundGuessed[nerGuesses[i_1]]     = foundGuessed.GetOrDefault(nerGuesses[i_1], 0.0) + 1;
                }
            }
            log.Info("\nSystem results:\n");
            log.Info("Accuracy: " + (correct / total) + "\n");
            foreach (string tag in tags)
            {
                double precision = foundGuessed.GetOrDefault(tag, 0.0) == 0 ? 0.0 : correctChunk.GetOrDefault(tag, 0.0) / foundGuessed[tag];
                double recall    = foundCorrect.GetOrDefault(tag, 0.0) == 0 ? 0.0 : correctChunk.GetOrDefault(tag, 0.0) / foundCorrect[tag];
                double f1        = (precision + recall == 0.0) ? 0.0 : (precision * recall * 2) / (precision + recall);
                log.Info(tag + " (" + foundCorrect.GetOrDefault(tag, 0.0) + ")");
                log.Info("\tP:" + precision + " (" + correctChunk.GetOrDefault(tag, 0.0) + "/" + foundGuessed.GetOrDefault(tag, 0.0) + ")");
                log.Info("\tR:" + recall + " (" + correctChunk.GetOrDefault(tag, 0.0) + "/" + foundCorrect.GetOrDefault(tag, 0.0) + ")");
                log.Info("\tF1:" + f1);
            }
        }
        public virtual void CheckMAPAgainstBruteForce(GraphicalModel model, ConcatVector weights, CliqueTree inference)
        {
            int[] map = inference.CalculateMAP();
            ICollection <TableFactor> tableFactors = model.factors.Stream().Map(null).Collect(Collectors.ToSet());
            // this is the super slow but obviously correct way to get global marginals
            TableFactor bruteForce = null;

            foreach (TableFactor factor in tableFactors)
            {
                if (bruteForce == null)
                {
                    bruteForce = factor;
                }
                else
                {
                    bruteForce = bruteForce.Multiply(factor);
                }
            }
            System.Diagnostics.Debug.Assert((bruteForce != null));
            // observe out all variables that have been registered
            TableFactor observed = bruteForce;

            foreach (int n in bruteForce.neighborIndices)
            {
                if (model.GetVariableMetaDataByReference(n).Contains(CliqueTree.VariableObservedValue))
                {
                    int value = System.Convert.ToInt32(model.GetVariableMetaDataByReference(n)[CliqueTree.VariableObservedValue]);
                    if (observed.neighborIndices.Length > 1)
                    {
                        observed = observed.Observe(n, value);
                    }
                    else
                    {
                        // If we've observed everything, then just quit
                        return;
                    }
                }
            }
            bruteForce = observed;
            int largestVariableNum = 0;

            foreach (GraphicalModel.Factor f in model.factors)
            {
                foreach (int i in f.neigborIndices)
                {
                    if (i > largestVariableNum)
                    {
                        largestVariableNum = i;
                    }
                }
            }
            // this is presented in true order, where 0 corresponds to var 0
            int[] mapValueAssignment = new int[largestVariableNum + 1];
            // this is kept in the order that the factor presents to us
            int[] highestValueAssignment = new int[bruteForce.neighborIndices.Length];
            foreach (int[] assignment in bruteForce)
            {
                if (bruteForce.GetAssignmentValue(assignment) > bruteForce.GetAssignmentValue(highestValueAssignment))
                {
                    highestValueAssignment = assignment;
                    for (int i = 0; i < assignment.Length; i++)
                    {
                        mapValueAssignment[bruteForce.neighborIndices[i]] = assignment[i];
                    }
                }
            }
            int[] forcedAssignments = new int[largestVariableNum + 1];
            for (int i_1 = 0; i_1 < mapValueAssignment.Length; i_1++)
            {
                if (model.GetVariableMetaDataByReference(i_1).Contains(CliqueTree.VariableObservedValue))
                {
                    mapValueAssignment[i_1] = System.Convert.ToInt32(model.GetVariableMetaDataByReference(i_1)[CliqueTree.VariableObservedValue]);
                    forcedAssignments[i_1]  = mapValueAssignment[i_1];
                }
            }
            if (!Arrays.Equals(mapValueAssignment, map))
            {
                System.Console.Error.WriteLine("---");
                System.Console.Error.WriteLine("Relevant variables: " + Arrays.ToString(bruteForce.neighborIndices));
                System.Console.Error.WriteLine("Var Sizes: " + Arrays.ToString(bruteForce.GetDimensions()));
                System.Console.Error.WriteLine("MAP: " + Arrays.ToString(map));
                System.Console.Error.WriteLine("Brute force map: " + Arrays.ToString(mapValueAssignment));
                System.Console.Error.WriteLine("Forced assignments: " + Arrays.ToString(forcedAssignments));
            }
            foreach (int i_2 in bruteForce.neighborIndices)
            {
                // Only check defined variables
                NUnit.Framework.Assert.AreEqual(mapValueAssignment[i_2], map[i_2]);
            }
        }
        //////////////////////////////////////////////////////////////
        // This is an implementation of something like MCTS, trying to take advantage of the general speed gains due to fast
        // CliqueTree caching of dot products. It doesn't actually do any clever selection, preferring to select observations
        // at random.
        //////////////////////////////////////////////////////////////
        private static void Gameplay(Random r, GraphicalModel model, ConcatVector weights, ConcatVector[] humanFeatureVectors)
        {
            IList <int> variablesList     = new List <int>();
            IList <int> variableSizesList = new List <int>();

            foreach (GraphicalModel.Factor f in model.factors)
            {
                for (int i = 0; i < f.neigborIndices.Length; i++)
                {
                    int j = f.neigborIndices[i];
                    if (!variablesList.Contains(j))
                    {
                        variablesList.Add(j);
                        variableSizesList.Add(f.featuresTable.GetDimensions()[i]);
                    }
                }
            }
            int[] variables     = variablesList.Stream().MapToInt(null).ToArray();
            int[] variableSizes = variableSizesList.Stream().MapToInt(null).ToArray();
            IList <GamePlayerBenchmark.SampleState> childrenOfRoot = new List <GamePlayerBenchmark.SampleState>();
            CliqueTree tree           = new CliqueTree(model, weights);
            int        initialFactors = model.factors.Count;
            // Run some "samples"
            long start         = Runtime.CurrentTimeMillis();
            long marginalsTime = 0;

            for (int i_1 = 0; i_1 < 1000; i_1++)
            {
                log.Info("\tTaking sample " + i_1);
                Stack <GamePlayerBenchmark.SampleState> stack = new Stack <GamePlayerBenchmark.SampleState>();
                GamePlayerBenchmark.SampleState         state = SelectOrCreateChildAtRandom(r, model, variables, variableSizes, childrenOfRoot, humanFeatureVectors);
                long localMarginalsTime = 0;
                // Each "sample" is 10 moves deep
                for (int j = 0; j < 10; j++)
                {
                    // log.info("\t\tFrame "+j);
                    state.Push(model);
                    System.Diagnostics.Debug.Assert((model.factors.Count == initialFactors + j + 1));
                    ///////////////////////////////////////////////////////////
                    // This is the thing we're really benchmarking
                    ///////////////////////////////////////////////////////////
                    if (state.cachedMarginal == null)
                    {
                        long s = Runtime.CurrentTimeMillis();
                        state.cachedMarginal = tree.CalculateMarginalsJustSingletons();
                        localMarginalsTime  += Runtime.CurrentTimeMillis() - s;
                    }
                    stack.Push(state);
                    state = SelectOrCreateChildAtRandom(r, model, variables, variableSizes, state.children, humanFeatureVectors);
                }
                log.Info("\t\t" + localMarginalsTime + " ms");
                marginalsTime += localMarginalsTime;
                while (!stack.Empty())
                {
                    stack.Pop().Pop(model);
                }
                System.Diagnostics.Debug.Assert((model.factors.Count == initialFactors));
            }
            log.Info("Marginals time: " + marginalsTime + " ms");
            log.Info("Avg time per marginal: " + (marginalsTime / 200) + " ms");
            log.Info("Total time: " + (Runtime.CurrentTimeMillis() - start));
        }
 private void RandomlyMutateGraphicalModel(GraphicalModel model, Random r)
 {
     if (r.NextBoolean() && model.factors.Count > 1)
     {
         // Remove one factor at random
         model.factors.Remove(Sharpen.Collections.ToArray(model.factors, new GraphicalModel.Factor[model.factors.Count])[r.NextInt(model.factors.Count)]);
     }
     else
     {
         // Add a simple binary factor, attaching a variable we haven't touched yet, but do observe, to an
         // existing variable. This represents the human observation operation in LENSE
         int maxVar        = 0;
         int attachVar     = -1;
         int attachVarSize = 0;
         foreach (GraphicalModel.Factor f in model.factors)
         {
             for (int j = 0; j < f.neigborIndices.Length; j++)
             {
                 int k = f.neigborIndices[j];
                 if (k > maxVar)
                 {
                     maxVar = k;
                 }
                 if (r.NextDouble() > 0.3 || attachVar == -1)
                 {
                     attachVar     = k;
                     attachVarSize = f.featuresTable.GetDimensions()[j];
                 }
             }
         }
         int newVar     = maxVar + 1;
         int newVarSize = 1 + r.NextInt(2);
         if (maxVar >= 8)
         {
             bool[] seenVariables = new bool[maxVar + 1];
             foreach (GraphicalModel.Factor f_1 in model.factors)
             {
                 foreach (int n in f_1.neigborIndices)
                 {
                     seenVariables[n] = true;
                 }
             }
             for (int j = 0; j < seenVariables.Length; j++)
             {
                 if (!seenVariables[j])
                 {
                     newVar = j;
                     break;
                 }
             }
             // This means the model is already too gigantic to be tractable, so we don't add anything here
             if (newVar == maxVar + 1)
             {
                 return;
             }
         }
         if (model.GetVariableMetaDataByReference(newVar).Contains(CliqueTree.VariableObservedValue))
         {
             int assignment = System.Convert.ToInt32(model.GetVariableMetaDataByReference(newVar)[CliqueTree.VariableObservedValue]);
             if (assignment >= newVarSize)
             {
                 newVarSize = assignment + 1;
             }
         }
         GraphicalModel.Factor binary = model.AddFactor(new int[] { newVar, attachVar }, new int[] { newVarSize, attachVarSize }, null);
         // "Cook" the randomly generated feature vector thunks, so they don't change as we run the system
         foreach (int[] assignment_1 in binary.featuresTable)
         {
             ConcatVector randomlyGenerated = binary.featuresTable.GetAssignmentValue(assignment_1).Get();
             binary.featuresTable.SetAssignmentValue(assignment_1, null);
         }
     }
 }
Example #18
0
 /// <summary>Create an Inference object for a given set of weights, and a model.</summary>
 /// <remarks>
 /// Create an Inference object for a given set of weights, and a model.
 /// <p>
 /// The object is around to facilitate cacheing as an eventual optimization, when models are changing in minor ways
 /// and inference is required several times. Work is done lazily, so is left until actual inference is requested.
 /// </remarks>
 /// <param name="model">the model to be computed over, subject to change in the future</param>
 /// <param name="weights">
 /// the weights to dot product with model features to get log-linear factors, is cloned internally so
 /// that no changes to the weights vector will be reflected by the CliqueTree. If you want to change
 /// the weights, you must create a new CliqueTree.
 /// </param>
 public CliqueTree(GraphicalModel model, ConcatVector weights)
 {
     // This is the metadata key for the model to store an observed value for a variable, as an int
     this.model   = model;
     this.weights = weights.DeepClone();
 }