/// <summary>
/// Slowest possible way to calculate a derivative for a model: exhaustive definitional calculation, using the super
/// slow logLikelihood function from this test suite.
/// </summary>
/// <param name="model">the model the get the derivative for</param>
/// <param name="weights">the weights to get the derivative at</param>
/// <returns>the derivative of the log likelihood with respect to the weights</returns>
private ConcatVector DefinitionOfDerivative(GraphicalModel model, ConcatVector weights)
{
double epsilon = 1.0e-7;
ConcatVector goldGradient = new ConcatVector(ConcatVecComponents);
for (int i = 0; i < ConcatVecComponents; i++)
{
double[] component = new double[ConcatVecComponentLength];
for (int j = 0; j < ConcatVecComponentLength; j++)
{
// Create a unit vector pointing in the direction of this element of this component
ConcatVector unitVectorIJ = new ConcatVector(ConcatVecComponents);
unitVectorIJ.SetSparseComponent(i, j, 1.0);
// Create a +eps weight vector
ConcatVector weightsPlusEpsilon = weights.DeepClone();
weightsPlusEpsilon.AddVectorInPlace(unitVectorIJ, epsilon);
// Create a -eps weight vector
ConcatVector weightsMinusEpsilon = weights.DeepClone();
weightsMinusEpsilon.AddVectorInPlace(unitVectorIJ, -epsilon);
// Use the definition (f(x+eps) - f(x-eps))/(2*eps)
component[j] = (LogLikelihood(model, weightsPlusEpsilon) - LogLikelihood(model, weightsMinusEpsilon)) / (2 * epsilon);
// If we encounter an impossible assignment, logLikelihood will return negative infinity, which will
// screw with the definitional calculation
if (double.IsNaN(component[j]))
{
component[j] = 0.0;
}
}
goldGradient.SetDenseComponent(i, component);
}
return(goldGradient);
}
/// <summary>The slowest, but obviously correct way to get log likelihood.</summary>
/// <remarks>
/// The slowest, but obviously correct way to get log likelihood. We've already tested the partition function in
/// the CliqueTreeTest, but in the interest of making things as different as possible to catch any lurking bugs or
/// numerical issues, we use the brute force approach here.
/// </remarks>
/// <param name="model">the model to get the log-likelihood of, assumes labels for assignments</param>
/// <param name="weights">the weights to get the log-likelihood at</param>
/// <returns>the log-likelihood</returns>
private double LogLikelihood(GraphicalModel model, ConcatVector weights)
{
ICollection <TableFactor> tableFactors = model.factors.Stream().Map(null).Collect(Collectors.ToSet());
System.Diagnostics.Debug.Assert((tableFactors.Count == model.factors.Count));
// this is the super slow but obviously correct way to get global marginals
TableFactor bruteForce = null;
foreach (TableFactor factor in tableFactors)
{
if (bruteForce == null)
{
bruteForce = factor;
}
else
{
bruteForce = bruteForce.Multiply(factor);
}
}
System.Diagnostics.Debug.Assert((bruteForce != null));
// observe out all variables that have been registered
TableFactor observed = bruteForce;
foreach (int n in bruteForce.neighborIndices)
{
if (model.GetVariableMetaDataByReference(n).Contains(CliqueTree.VariableObservedValue))
{
int value = System.Convert.ToInt32(model.GetVariableMetaDataByReference(n)[CliqueTree.VariableObservedValue]);
if (observed.neighborIndices.Length > 1)
{
observed = observed.Observe(n, value);
}
else
{
// If we've observed everything, then just quit
return(0.0);
}
}
}
bruteForce = observed;
// Now we can get a partition function
double partitionFunction = bruteForce.ValueSum();
// For now, we'll assume that all the variables are given for training. EM is another problem altogether
int[] assignment = new int[bruteForce.neighborIndices.Length];
for (int i = 0; i < assignment.Length; i++)
{
System.Diagnostics.Debug.Assert((!model.GetVariableMetaDataByReference(bruteForce.neighborIndices[i]).Contains(CliqueTree.VariableObservedValue)));
assignment[i] = System.Convert.ToInt32(model.GetVariableMetaDataByReference(bruteForce.neighborIndices[i])[LogLikelihoodDifferentiableFunction.VariableTrainingValue]);
}
if (bruteForce.GetAssignmentValue(assignment) == 0 || partitionFunction == 0)
{
return(double.NegativeInfinity);
}
return(Math.Log(bruteForce.GetAssignmentValue(assignment)) - Math.Log(partitionFunction));
}
public virtual void TestCalculateMarginals(GraphicalModel model, ConcatVector weights)
{
CliqueTree inference = new CliqueTree(model, weights);
// This is the basic check that inference works when you first construct the model
CheckMarginalsAgainstBruteForce((GraphicalModel)model, (ConcatVector)weights, inference);
// Now we go through several random mutations to the model, and check that everything is still consistent
Random r = new Random();
for (int i = 0; i < 10; i++)
{
RandomlyMutateGraphicalModel((GraphicalModel)model, r);
CheckMarginalsAgainstBruteForce((GraphicalModel)model, (ConcatVector)weights, inference);
}
}
/// <summary>
/// This lets the system allocate work to threads evenly, which reduces the amount of blocking and can improve
/// runtimes by 20% or more.
/// </summary>
/// <param name="datum">the datum to estimate work for</param>
/// <returns>a work estimate, on a relative scale of single cpu wall time, for getting the gradient and log-likelihood</returns>
private int EstimateRelativeRuntime(T datum)
{
if (datum is GraphicalModel)
{
int cost = 0;
GraphicalModel model = (GraphicalModel)datum;
foreach (GraphicalModel.Factor f in model.factors)
{
cost += f.featuresTable.CombinatorialNeighborStatesCount();
}
return(cost);
}
else
{
return(1);
}
}
public override GraphicalModel[] Generate(SourceOfRandomness sourceOfRandomness, IGenerationStatus generationStatus)
{
GraphicalModel[] dataset = new GraphicalModel[sourceOfRandomness.NextInt(1, 10)];
for (int i = 0; i < dataset.Length; i++)
{
dataset[i] = modelGenerator.Generate(sourceOfRandomness, generationStatus);
foreach (GraphicalModel.Factor f in dataset[i].factors)
{
for (int j = 0; j < f.neigborIndices.Length; j++)
{
int n = f.neigborIndices[j];
int dim = f.featuresTable.GetDimensions()[j];
dataset[i].GetVariableMetaDataByReference(n)[LogLikelihoodDifferentiableFunction.VariableTrainingValue] = string.Empty + sourceOfRandomness.NextInt(dim);
}
}
}
return(dataset);
}
public static void Annotate(GraphicalModel model, IList <string> tags, ConcatVectorNamespace @namespace, IDictionary <string, double[]> embeddings)
{
for (int i = 0; i < model.variableMetaData.Count; i++)
{
IDictionary <string, string> metadata = model.GetVariableMetaDataByReference(i);
string token = metadata["TOKEN"];
string pos = metadata["POS"];
string chunk = metadata["CHUNK"];
IDictionary <string, string> leftMetadata = null;
if (i > 0)
{
leftMetadata = model.GetVariableMetaDataByReference(i - 1);
}
string leftToken = (leftMetadata == null) ? "^" : leftMetadata["TOKEN"];
string leftPos = (leftMetadata == null) ? "^" : leftMetadata["POS"];
string leftChunk = (leftMetadata == null) ? "^" : leftMetadata["CHUNK"];
IDictionary <string, string> rightMetadata = null;
if (i < model.variableMetaData.Count - 1)
{
rightMetadata = model.GetVariableMetaDataByReference(i + 1);
}
string rightToken = (rightMetadata == null) ? "$" : rightMetadata["TOKEN"];
string rightPos = (rightMetadata == null) ? "$" : rightMetadata["POS"];
string rightChunk = (rightMetadata == null) ? "$" : rightMetadata["CHUNK"];
// Add the unary factor
GraphicalModel.Factor f = model.AddFactor(new int[] { i }, new int[] { tags.Count }, null);
// This is the anonymous function that generates a feature vector for each assignment to the unary
// factor
System.Diagnostics.Debug.Assert((f.neigborIndices.Length == 1));
System.Diagnostics.Debug.Assert((f.neigborIndices[0] == i));
// If this is not the last variable, add a binary factor
if (i < model.variableMetaData.Count - 1)
{
GraphicalModel.Factor jf = model.AddFactor(new int[] { i, i + 1 }, new int[] { tags.Count, tags.Count }, null);
// This is the anonymous function that generates a feature vector for every joint assignment to the
// binary factor
System.Diagnostics.Debug.Assert((jf.neigborIndices.Length == 2));
System.Diagnostics.Debug.Assert((jf.neigborIndices[0] == i));
System.Diagnostics.Debug.Assert((jf.neigborIndices[1] == i + 1));
}
}
}
public virtual GraphicalModel GenerateSentenceModel(ConcatVectorNamespace @namespace, CoNLLBenchmark.CoNLLSentence sentence, IList <string> tags)
{
GraphicalModel model = new GraphicalModel();
for (int i = 0; i < sentence.token.Count; i++)
{
// Add the training label
IDictionary <string, string> metadata = model.GetVariableMetaDataByReference(i);
metadata[LogLikelihoodDifferentiableFunction.VariableTrainingValue] = string.Empty + tags.IndexOf(sentence.ner[i]);
metadata["TOKEN"] = string.Empty + sentence.token[i];
metadata["POS"] = string.Empty + sentence.pos[i];
metadata["CHUNK"] = string.Empty + sentence.npchunk[i];
metadata["TAG"] = string.Empty + sentence.ner[i];
}
CoNLLFeaturizer.Annotate(model, tags, @namespace, embeddings);
System.Diagnostics.Debug.Assert((model.factors != null));
foreach (GraphicalModel.Factor f in model.factors)
{
System.Diagnostics.Debug.Assert((f != null));
}
return(model);
}
/// <summary>
/// 绘制图形
/// </summary>
public override void Drawing()
{
if (OwnerChart == null || !OwnerChart.IsLoaded ||
OwnerChart.XAxis == null || !OwnerChart.XAxis.IsLoaded || OwnerChart.XAxis.ActualWidth == 0 ||
OwnerChart.YAxis == null || !OwnerChart.YAxis.IsLoaded || OwnerChart.XAxis.ActualHeight == 0)
{
return;
}
if (GraphicalShape == null)
{
Binding binding = new Binding("Points")
{
Source = this.GraphicalModel, Converter = GraphicalPointsConverter, ConverterParameter = OwnerChart, Mode = BindingMode.TwoWay
};
var temp = new Polygon()
{
FillRule = FillRule.Nonzero, Cursor = Cursors.Hand
};
temp.SetBinding(Polygon.PointsProperty, binding);
GraphicalShape = temp;
}
GraphicalModel.OnPropertyChanged("Points");
}
private void CheckMarginalsAgainstBruteForce(GraphicalModel model, ConcatVector weights, CliqueTree inference)
{
CliqueTree.MarginalResult result = inference.CalculateMarginals();
double[][] marginals = result.marginals;
ICollection <TableFactor> tableFactors = model.factors.Stream().Map(null).Collect(Collectors.ToSet());
System.Diagnostics.Debug.Assert((tableFactors.Count == model.factors.Count));
// this is the super slow but obviously correct way to get global marginals
TableFactor bruteForce = null;
foreach (TableFactor factor in tableFactors)
{
if (bruteForce == null)
{
bruteForce = factor;
}
else
{
bruteForce = bruteForce.Multiply(factor);
}
}
if (bruteForce != null)
{
// observe out all variables that have been registered
TableFactor observed = bruteForce;
for (int i = 0; i < bruteForce.neighborIndices.Length; i++)
{
int n = bruteForce.neighborIndices[i];
if (model.GetVariableMetaDataByReference(n).Contains(CliqueTree.VariableObservedValue))
{
int value = System.Convert.ToInt32(model.GetVariableMetaDataByReference(n)[CliqueTree.VariableObservedValue]);
// Check that the marginals reflect the observation
for (int j = 0; j < marginals[n].Length; j++)
{
NUnit.Framework.Assert.AreEqual(marginals[n][j], 1.0e-9, j == value ? 1.0 : 0.0);
}
if (observed.neighborIndices.Length > 1)
{
observed = observed.Observe(n, value);
}
else
{
// If we've observed everything, then just quit
return;
}
}
}
bruteForce = observed;
// Spot check each of the marginals in the brute force calculation
double[][] bruteMarginals = bruteForce.GetSummedMarginals();
int index = 0;
foreach (int i_1 in bruteForce.neighborIndices)
{
bool isEqual = true;
double[] brute = bruteMarginals[index];
index++;
System.Diagnostics.Debug.Assert((brute != null));
System.Diagnostics.Debug.Assert((marginals[i_1] != null));
for (int j = 0; j < brute.Length; j++)
{
if (double.IsNaN(brute[j]))
{
isEqual = false;
break;
}
if (Math.Abs(brute[j] - marginals[i_1][j]) > 3.0e-2)
{
isEqual = false;
break;
}
}
if (!isEqual)
{
System.Console.Error.WriteLine("Arrays not equal! Variable " + i_1);
System.Console.Error.WriteLine("\tGold: " + Arrays.ToString(brute));
System.Console.Error.WriteLine("\tResult: " + Arrays.ToString(marginals[i_1]));
}
Assert.AssertArrayEquals(marginals[i_1], 3.0e-2, brute);
}
// Spot check the partition function
double goldPartitionFunction = bruteForce.ValueSum();
// Correct to within 3%
NUnit.Framework.Assert.AreEqual(result.partitionFunction, goldPartitionFunction * 3.0e-2, goldPartitionFunction);
// Check the joint marginals
foreach (GraphicalModel.Factor f in model.factors)
{
NUnit.Framework.Assert.IsTrue(result.jointMarginals.Contains(f));
TableFactor bruteForceJointMarginal = bruteForce;
foreach (int n in bruteForce.neighborIndices)
{
foreach (int i_2 in f.neigborIndices)
{
if (i_2 == n)
{
goto outer_continue;
}
}
if (bruteForceJointMarginal.neighborIndices.Length > 1)
{
bruteForceJointMarginal = bruteForceJointMarginal.SumOut(n);
}
else
{
int[] fixedAssignment = new int[f.neigborIndices.Length];
for (int i_3 = 0; i_3 < fixedAssignment.Length; i_3++)
{
fixedAssignment[i_3] = System.Convert.ToInt32(model.GetVariableMetaDataByReference(f.neigborIndices[i_3])[CliqueTree.VariableObservedValue]);
}
foreach (int[] assn in result.jointMarginals[f])
{
if (Arrays.Equals(assn, fixedAssignment))
{
NUnit.Framework.Assert.AreEqual(result.jointMarginals[f].GetAssignmentValue(assn), 1.0e-7, 1.0);
}
else
{
if (result.jointMarginals[f].GetAssignmentValue(assn) != 0)
{
TableFactor j = result.jointMarginals[f];
foreach (int[] assignment in j)
{
System.Console.Error.WriteLine(Arrays.ToString(assignment) + ": " + j.GetAssignmentValue(assignment));
}
}
NUnit.Framework.Assert.AreEqual(result.jointMarginals[f].GetAssignmentValue(assn), 1.0e-7, 0.0);
}
}
goto marginals_continue;
}
}
outer_break :;
// Find the correspondence between the brute force joint marginal, which may be missing variables
// because they were observed out of the table, and the output joint marginals, which are always an exact
// match for the original factor
int[] backPointers = new int[f.neigborIndices.Length];
int[] observedValue = new int[f.neigborIndices.Length];
for (int i_4 = 0; i_4 < backPointers.Length; i_4++)
{
if (model.GetVariableMetaDataByReference(f.neigborIndices[i_4]).Contains(CliqueTree.VariableObservedValue))
{
observedValue[i_4] = System.Convert.ToInt32(model.GetVariableMetaDataByReference(f.neigborIndices[i_4])[CliqueTree.VariableObservedValue]);
backPointers[i_4] = -1;
}
else
{
observedValue[i_4] = -1;
backPointers[i_4] = -1;
for (int j = 0; j < bruteForceJointMarginal.neighborIndices.Length; j++)
{
if (bruteForceJointMarginal.neighborIndices[j] == f.neigborIndices[i_4])
{
backPointers[i_4] = j;
}
}
System.Diagnostics.Debug.Assert((backPointers[i_4] != -1));
}
}
double sum = bruteForceJointMarginal.ValueSum();
if (sum == 0.0)
{
sum = 1;
}
foreach (int[] assignment_1 in result.jointMarginals[f])
{
int[] bruteForceMarginalAssignment = new int[bruteForceJointMarginal.neighborIndices.Length];
for (int i_2 = 0; i_2 < assignment_1.Length; i_2++)
{
if (backPointers[i_2] != -1)
{
bruteForceMarginalAssignment[backPointers[i_2]] = assignment_1[i_2];
}
else
{
// Make sure all assignments that don't square with observations get 0 weight
System.Diagnostics.Debug.Assert((observedValue[i_2] != -1));
if (assignment_1[i_2] != observedValue[i_2])
{
if (result.jointMarginals[f].GetAssignmentValue(assignment_1) != 0)
{
System.Console.Error.WriteLine("Joint marginals: " + Arrays.ToString(result.jointMarginals[f].neighborIndices));
System.Console.Error.WriteLine("Assignment: " + Arrays.ToString(assignment_1));
System.Console.Error.WriteLine("Observed Value: " + Arrays.ToString(observedValue));
foreach (int[] assn in result.jointMarginals[f])
{
System.Console.Error.WriteLine("\t" + Arrays.ToString(assn) + ":" + result.jointMarginals[f].GetAssignmentValue(assn));
}
}
NUnit.Framework.Assert.AreEqual(result.jointMarginals[f].GetAssignmentValue(assignment_1), 1.0e-7, 0.0);
goto outer_continue;
}
}
}
NUnit.Framework.Assert.AreEqual(result.jointMarginals[f].GetAssignmentValue(assignment_1), 1.0e-3, bruteForceJointMarginal.GetAssignmentValue(bruteForceMarginalAssignment) / sum);
}
outer_break :;
}
marginals_break :;
}
else
{
foreach (double[] marginal in marginals)
{
foreach (double d in marginal)
{
NUnit.Framework.Assert.AreEqual(d, 3.0e-2, 1.0 / marginal.Length);
}
}
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.TypeLoadException"/>
public static void Main(string[] args)
{
//////////////////////////////////////////////////////////////
// Generate the CoNLL CliqueTrees to use during gameplay
//////////////////////////////////////////////////////////////
CoNLLBenchmark coNLL = new CoNLLBenchmark();
IList <CoNLLBenchmark.CoNLLSentence> train = coNLL.GetSentences(DataPath + "conll.iob.4class.train");
IList <CoNLLBenchmark.CoNLLSentence> testA = coNLL.GetSentences(DataPath + "conll.iob.4class.testa");
IList <CoNLLBenchmark.CoNLLSentence> testB = coNLL.GetSentences(DataPath + "conll.iob.4class.testb");
IList <CoNLLBenchmark.CoNLLSentence> allData = new List <CoNLLBenchmark.CoNLLSentence>();
Sharpen.Collections.AddAll(allData, train);
Sharpen.Collections.AddAll(allData, testA);
Sharpen.Collections.AddAll(allData, testB);
ICollection <string> tagsSet = new HashSet <string>();
foreach (CoNLLBenchmark.CoNLLSentence sentence in allData)
{
foreach (string nerTag in sentence.ner)
{
tagsSet.Add(nerTag);
}
}
IList <string> tags = new List <string>();
Sharpen.Collections.AddAll(tags, tagsSet);
coNLL.embeddings = coNLL.GetEmbeddings(DataPath + "google-300-trimmed.ser.gz", allData);
log.Info("Making the training set...");
ConcatVectorNamespace @namespace = new ConcatVectorNamespace();
int trainSize = train.Count;
GraphicalModel[] trainingSet = new GraphicalModel[trainSize];
for (int i = 0; i < trainSize; i++)
{
if (i % 10 == 0)
{
log.Info(i + "/" + trainSize);
}
trainingSet[i] = coNLL.GenerateSentenceModel(@namespace, train[i], tags);
}
//////////////////////////////////////////////////////////////
// Generate the random human observation feature vectors that we'll use
//////////////////////////////////////////////////////////////
Random r = new Random(10);
int numFeatures = 5;
int featureLength = 30;
ConcatVector[] humanFeatureVectors = new ConcatVector[1000];
for (int i_1 = 0; i_1 < humanFeatureVectors.Length; i_1++)
{
humanFeatureVectors[i_1] = new ConcatVector(numFeatures);
for (int j = 0; j < numFeatures; j++)
{
if (r.NextBoolean())
{
humanFeatureVectors[i_1].SetSparseComponent(j, r.NextInt(featureLength), r.NextDouble());
}
else
{
double[] dense = new double[featureLength];
for (int k = 0; k < dense.Length; k++)
{
dense[k] = r.NextDouble();
}
humanFeatureVectors[i_1].SetDenseComponent(j, dense);
}
}
}
ConcatVector weights = new ConcatVector(numFeatures);
for (int i_2 = 0; i_2 < numFeatures; i_2++)
{
double[] dense = new double[featureLength];
for (int j = 0; j < dense.Length; j++)
{
dense[j] = r.NextDouble();
}
weights.SetDenseComponent(i_2, dense);
}
//////////////////////////////////////////////////////////////
// Actually perform gameplay-like random mutations
//////////////////////////////////////////////////////////////
log.Info("Warming up the JIT...");
for (int i_3 = 0; i_3 < 10; i_3++)
{
log.Info(i_3);
Gameplay(r, trainingSet[i_3], weights, humanFeatureVectors);
}
log.Info("Timing actual run...");
long start = Runtime.CurrentTimeMillis();
for (int i_4 = 0; i_4 < 10; i_4++)
{
log.Info(i_4);
Gameplay(r, trainingSet[i_4], weights, humanFeatureVectors);
}
long duration = Runtime.CurrentTimeMillis() - start;
log.Info("Duration: " + duration);
}
/// <summary>See push() for an explanation.</summary>
/// <param name="model">the model to pop this SampleState from</param>
public virtual void Pop(GraphicalModel model)
{
System.Diagnostics.Debug.Assert((model.factors.Contains(addedFactor)));
model.factors.Remove(addedFactor);
Sharpen.Collections.Remove(model.GetVariableMetaDataByReference(variable), CliqueTree.VariableObservedValue);
}
/// <summary>This applies this SampleState to the model.</summary>
/// <remarks>
/// This applies this SampleState to the model. The name comes from an analogy to a stack. If we take a sample
/// path, involving a number of steps through the model, we push() each SampleState onto the model one at a time,
/// then when we return from the sample we can pop() each SampleState off the model, and be left with our
/// original model state.
/// </remarks>
/// <param name="model">the model to push this SampleState onto</param>
public virtual void Push(GraphicalModel model)
{
System.Diagnostics.Debug.Assert((!model.factors.Contains(addedFactor)));
model.factors.Add(addedFactor);
model.GetVariableMetaDataByReference(variable)[CliqueTree.VariableObservedValue] = string.Empty + observation;
}
private static GamePlayerBenchmark.SampleState SelectOrCreateChildAtRandom(Random r, GraphicalModel model, int[] variables, int[] variableSizes, IList <GamePlayerBenchmark.SampleState> children, ConcatVector[] humanFeatureVectors)
{
int i = r.NextInt(variables.Length);
int variable = variables[i];
int observation = r.NextInt(variableSizes[i]);
foreach (GamePlayerBenchmark.SampleState s in children)
{
if (s.variable == variable && s.observation == observation)
{
return(s);
}
}
int humanObservationVariable = 0;
foreach (GraphicalModel.Factor f in model.factors)
{
foreach (int j in f.neigborIndices)
{
if (j >= humanObservationVariable)
{
humanObservationVariable = j + 1;
}
}
}
GraphicalModel.Factor f_1 = model.AddFactor(new int[] { variable, humanObservationVariable }, new int[] { variableSizes[i], variableSizes[i] }, null);
model.factors.Remove(f_1);
GamePlayerBenchmark.SampleState newState = new GamePlayerBenchmark.SampleState(f_1, variable, observation);
children.Add(newState);
return(newState);
}
/// <exception cref="System.Exception"/>
public virtual void BenchmarkOptimizer()
{
IList <CoNLLBenchmark.CoNLLSentence> train = GetSentences(DataPath + "conll.iob.4class.train");
IList <CoNLLBenchmark.CoNLLSentence> testA = GetSentences(DataPath + "conll.iob.4class.testa");
IList <CoNLLBenchmark.CoNLLSentence> testB = GetSentences(DataPath + "conll.iob.4class.testb");
IList <CoNLLBenchmark.CoNLLSentence> allData = new List <CoNLLBenchmark.CoNLLSentence>();
Sharpen.Collections.AddAll(allData, train);
Sharpen.Collections.AddAll(allData, testA);
Sharpen.Collections.AddAll(allData, testB);
ICollection <string> tagsSet = new HashSet <string>();
foreach (CoNLLBenchmark.CoNLLSentence sentence in allData)
{
foreach (string nerTag in sentence.ner)
{
tagsSet.Add(nerTag);
}
}
IList <string> tags = new List <string>();
Sharpen.Collections.AddAll(tags, tagsSet);
embeddings = GetEmbeddings(DataPath + "google-300-trimmed.ser.gz", allData);
log.Info("Making the training set...");
ConcatVectorNamespace @namespace = new ConcatVectorNamespace();
int trainSize = train.Count;
GraphicalModel[] trainingSet = new GraphicalModel[trainSize];
for (int i = 0; i < trainSize; i++)
{
if (i % 10 == 0)
{
log.Info(i + "/" + trainSize);
}
trainingSet[i] = GenerateSentenceModel(@namespace, train[i], tags);
}
log.Info("Training system...");
AbstractBatchOptimizer opt = new BacktrackingAdaGradOptimizer();
// This training call is basically what we want the benchmark for. It should take 99% of the wall clock time
ConcatVector weights = opt.Optimize(trainingSet, new LogLikelihoodDifferentiableFunction(), @namespace.NewWeightsVector(), 0.01, 1.0e-5, false);
log.Info("Testing system...");
// Evaluation method lifted from the CoNLL 2004 perl script
IDictionary <string, double> correctChunk = new Dictionary <string, double>();
IDictionary <string, double> foundCorrect = new Dictionary <string, double>();
IDictionary <string, double> foundGuessed = new Dictionary <string, double>();
double correct = 0.0;
double total = 0.0;
foreach (CoNLLBenchmark.CoNLLSentence sentence_1 in testA)
{
GraphicalModel model = GenerateSentenceModel(@namespace, sentence_1, tags);
int[] guesses = new CliqueTree(model, weights).CalculateMAP();
string[] nerGuesses = new string[guesses.Length];
for (int i_1 = 0; i_1 < guesses.Length; i_1++)
{
nerGuesses[i_1] = tags[guesses[i_1]];
if (nerGuesses[i_1].Equals(sentence_1.ner[i_1]))
{
correct++;
correctChunk[nerGuesses[i_1]] = correctChunk.GetOrDefault(nerGuesses[i_1], 0.0) + 1;
}
total++;
foundCorrect[sentence_1.ner[i_1]] = foundCorrect.GetOrDefault(sentence_1.ner[i_1], 0.0) + 1;
foundGuessed[nerGuesses[i_1]] = foundGuessed.GetOrDefault(nerGuesses[i_1], 0.0) + 1;
}
}
log.Info("\nSystem results:\n");
log.Info("Accuracy: " + (correct / total) + "\n");
foreach (string tag in tags)
{
double precision = foundGuessed.GetOrDefault(tag, 0.0) == 0 ? 0.0 : correctChunk.GetOrDefault(tag, 0.0) / foundGuessed[tag];
double recall = foundCorrect.GetOrDefault(tag, 0.0) == 0 ? 0.0 : correctChunk.GetOrDefault(tag, 0.0) / foundCorrect[tag];
double f1 = (precision + recall == 0.0) ? 0.0 : (precision * recall * 2) / (precision + recall);
log.Info(tag + " (" + foundCorrect.GetOrDefault(tag, 0.0) + ")");
log.Info("\tP:" + precision + " (" + correctChunk.GetOrDefault(tag, 0.0) + "/" + foundGuessed.GetOrDefault(tag, 0.0) + ")");
log.Info("\tR:" + recall + " (" + correctChunk.GetOrDefault(tag, 0.0) + "/" + foundCorrect.GetOrDefault(tag, 0.0) + ")");
log.Info("\tF1:" + f1);
}
}
public virtual void CheckMAPAgainstBruteForce(GraphicalModel model, ConcatVector weights, CliqueTree inference)
{
int[] map = inference.CalculateMAP();
ICollection <TableFactor> tableFactors = model.factors.Stream().Map(null).Collect(Collectors.ToSet());
// this is the super slow but obviously correct way to get global marginals
TableFactor bruteForce = null;
foreach (TableFactor factor in tableFactors)
{
if (bruteForce == null)
{
bruteForce = factor;
}
else
{
bruteForce = bruteForce.Multiply(factor);
}
}
System.Diagnostics.Debug.Assert((bruteForce != null));
// observe out all variables that have been registered
TableFactor observed = bruteForce;
foreach (int n in bruteForce.neighborIndices)
{
if (model.GetVariableMetaDataByReference(n).Contains(CliqueTree.VariableObservedValue))
{
int value = System.Convert.ToInt32(model.GetVariableMetaDataByReference(n)[CliqueTree.VariableObservedValue]);
if (observed.neighborIndices.Length > 1)
{
observed = observed.Observe(n, value);
}
else
{
// If we've observed everything, then just quit
return;
}
}
}
bruteForce = observed;
int largestVariableNum = 0;
foreach (GraphicalModel.Factor f in model.factors)
{
foreach (int i in f.neigborIndices)
{
if (i > largestVariableNum)
{
largestVariableNum = i;
}
}
}
// this is presented in true order, where 0 corresponds to var 0
int[] mapValueAssignment = new int[largestVariableNum + 1];
// this is kept in the order that the factor presents to us
int[] highestValueAssignment = new int[bruteForce.neighborIndices.Length];
foreach (int[] assignment in bruteForce)
{
if (bruteForce.GetAssignmentValue(assignment) > bruteForce.GetAssignmentValue(highestValueAssignment))
{
highestValueAssignment = assignment;
for (int i = 0; i < assignment.Length; i++)
{
mapValueAssignment[bruteForce.neighborIndices[i]] = assignment[i];
}
}
}
int[] forcedAssignments = new int[largestVariableNum + 1];
for (int i_1 = 0; i_1 < mapValueAssignment.Length; i_1++)
{
if (model.GetVariableMetaDataByReference(i_1).Contains(CliqueTree.VariableObservedValue))
{
mapValueAssignment[i_1] = System.Convert.ToInt32(model.GetVariableMetaDataByReference(i_1)[CliqueTree.VariableObservedValue]);
forcedAssignments[i_1] = mapValueAssignment[i_1];
}
}
if (!Arrays.Equals(mapValueAssignment, map))
{
System.Console.Error.WriteLine("---");
System.Console.Error.WriteLine("Relevant variables: " + Arrays.ToString(bruteForce.neighborIndices));
System.Console.Error.WriteLine("Var Sizes: " + Arrays.ToString(bruteForce.GetDimensions()));
System.Console.Error.WriteLine("MAP: " + Arrays.ToString(map));
System.Console.Error.WriteLine("Brute force map: " + Arrays.ToString(mapValueAssignment));
System.Console.Error.WriteLine("Forced assignments: " + Arrays.ToString(forcedAssignments));
}
foreach (int i_2 in bruteForce.neighborIndices)
{
// Only check defined variables
NUnit.Framework.Assert.AreEqual(mapValueAssignment[i_2], map[i_2]);
}
}
//////////////////////////////////////////////////////////////
// This is an implementation of something like MCTS, trying to take advantage of the general speed gains due to fast
// CliqueTree caching of dot products. It doesn't actually do any clever selection, preferring to select observations
// at random.
//////////////////////////////////////////////////////////////
private static void Gameplay(Random r, GraphicalModel model, ConcatVector weights, ConcatVector[] humanFeatureVectors)
{
IList <int> variablesList = new List <int>();
IList <int> variableSizesList = new List <int>();
foreach (GraphicalModel.Factor f in model.factors)
{
for (int i = 0; i < f.neigborIndices.Length; i++)
{
int j = f.neigborIndices[i];
if (!variablesList.Contains(j))
{
variablesList.Add(j);
variableSizesList.Add(f.featuresTable.GetDimensions()[i]);
}
}
}
int[] variables = variablesList.Stream().MapToInt(null).ToArray();
int[] variableSizes = variableSizesList.Stream().MapToInt(null).ToArray();
IList <GamePlayerBenchmark.SampleState> childrenOfRoot = new List <GamePlayerBenchmark.SampleState>();
CliqueTree tree = new CliqueTree(model, weights);
int initialFactors = model.factors.Count;
// Run some "samples"
long start = Runtime.CurrentTimeMillis();
long marginalsTime = 0;
for (int i_1 = 0; i_1 < 1000; i_1++)
{
log.Info("\tTaking sample " + i_1);
Stack <GamePlayerBenchmark.SampleState> stack = new Stack <GamePlayerBenchmark.SampleState>();
GamePlayerBenchmark.SampleState state = SelectOrCreateChildAtRandom(r, model, variables, variableSizes, childrenOfRoot, humanFeatureVectors);
long localMarginalsTime = 0;
// Each "sample" is 10 moves deep
for (int j = 0; j < 10; j++)
{
// log.info("\t\tFrame "+j);
state.Push(model);
System.Diagnostics.Debug.Assert((model.factors.Count == initialFactors + j + 1));
///////////////////////////////////////////////////////////
// This is the thing we're really benchmarking
///////////////////////////////////////////////////////////
if (state.cachedMarginal == null)
{
long s = Runtime.CurrentTimeMillis();
state.cachedMarginal = tree.CalculateMarginalsJustSingletons();
localMarginalsTime += Runtime.CurrentTimeMillis() - s;
}
stack.Push(state);
state = SelectOrCreateChildAtRandom(r, model, variables, variableSizes, state.children, humanFeatureVectors);
}
log.Info("\t\t" + localMarginalsTime + " ms");
marginalsTime += localMarginalsTime;
while (!stack.Empty())
{
stack.Pop().Pop(model);
}
System.Diagnostics.Debug.Assert((model.factors.Count == initialFactors));
}
log.Info("Marginals time: " + marginalsTime + " ms");
log.Info("Avg time per marginal: " + (marginalsTime / 200) + " ms");
log.Info("Total time: " + (Runtime.CurrentTimeMillis() - start));
}
private void RandomlyMutateGraphicalModel(GraphicalModel model, Random r)
{
if (r.NextBoolean() && model.factors.Count > 1)
{
// Remove one factor at random
model.factors.Remove(Sharpen.Collections.ToArray(model.factors, new GraphicalModel.Factor[model.factors.Count])[r.NextInt(model.factors.Count)]);
}
else
{
// Add a simple binary factor, attaching a variable we haven't touched yet, but do observe, to an
// existing variable. This represents the human observation operation in LENSE
int maxVar = 0;
int attachVar = -1;
int attachVarSize = 0;
foreach (GraphicalModel.Factor f in model.factors)
{
for (int j = 0; j < f.neigborIndices.Length; j++)
{
int k = f.neigborIndices[j];
if (k > maxVar)
{
maxVar = k;
}
if (r.NextDouble() > 0.3 || attachVar == -1)
{
attachVar = k;
attachVarSize = f.featuresTable.GetDimensions()[j];
}
}
}
int newVar = maxVar + 1;
int newVarSize = 1 + r.NextInt(2);
if (maxVar >= 8)
{
bool[] seenVariables = new bool[maxVar + 1];
foreach (GraphicalModel.Factor f_1 in model.factors)
{
foreach (int n in f_1.neigborIndices)
{
seenVariables[n] = true;
}
}
for (int j = 0; j < seenVariables.Length; j++)
{
if (!seenVariables[j])
{
newVar = j;
break;
}
}
// This means the model is already too gigantic to be tractable, so we don't add anything here
if (newVar == maxVar + 1)
{
return;
}
}
if (model.GetVariableMetaDataByReference(newVar).Contains(CliqueTree.VariableObservedValue))
{
int assignment = System.Convert.ToInt32(model.GetVariableMetaDataByReference(newVar)[CliqueTree.VariableObservedValue]);
if (assignment >= newVarSize)
{
newVarSize = assignment + 1;
}
}
GraphicalModel.Factor binary = model.AddFactor(new int[] { newVar, attachVar }, new int[] { newVarSize, attachVarSize }, null);
// "Cook" the randomly generated feature vector thunks, so they don't change as we run the system
foreach (int[] assignment_1 in binary.featuresTable)
{
ConcatVector randomlyGenerated = binary.featuresTable.GetAssignmentValue(assignment_1).Get();
binary.featuresTable.SetAssignmentValue(assignment_1, null);
}
}
}
/// <summary>Create an Inference object for a given set of weights, and a model.</summary>
/// <remarks>
/// Create an Inference object for a given set of weights, and a model.
/// <p>
/// The object is around to facilitate cacheing as an eventual optimization, when models are changing in minor ways
/// and inference is required several times. Work is done lazily, so is left until actual inference is requested.
/// </remarks>
/// <param name="model">the model to be computed over, subject to change in the future</param>
/// <param name="weights">
/// the weights to dot product with model features to get log-linear factors, is cloned internally so
/// that no changes to the weights vector will be reflected by the CliqueTree. If you want to change
/// the weights, you must create a new CliqueTree.
/// </param>
public CliqueTree(GraphicalModel model, ConcatVector weights)
{
// This is the metadata key for the model to store an observed value for a variable, as an int
this.model = model;
this.weights = weights.DeepClone();
}