/// <summary>
/// Slowest possible way to calculate a derivative for a model: exhaustive definitional calculation, using the super
/// slow logLikelihood function from this test suite.
/// </summary>
/// <param name="model">the model the get the derivative for</param>
/// <param name="weights">the weights to get the derivative at</param>
/// <returns>the derivative of the log likelihood with respect to the weights</returns>
private ConcatVector DefinitionOfDerivative(GraphicalModel model, ConcatVector weights)
{
double epsilon = 1.0e-7;
ConcatVector goldGradient = new ConcatVector(ConcatVecComponents);
for (int i = 0; i < ConcatVecComponents; i++)
{
double[] component = new double[ConcatVecComponentLength];
for (int j = 0; j < ConcatVecComponentLength; j++)
{
// Create a unit vector pointing in the direction of this element of this component
ConcatVector unitVectorIJ = new ConcatVector(ConcatVecComponents);
unitVectorIJ.SetSparseComponent(i, j, 1.0);
// Create a +eps weight vector
ConcatVector weightsPlusEpsilon = weights.DeepClone();
weightsPlusEpsilon.AddVectorInPlace(unitVectorIJ, epsilon);
// Create a -eps weight vector
ConcatVector weightsMinusEpsilon = weights.DeepClone();
weightsMinusEpsilon.AddVectorInPlace(unitVectorIJ, -epsilon);
// Use the definition (f(x+eps) - f(x-eps))/(2*eps)
component[j] = (LogLikelihood(model, weightsPlusEpsilon) - LogLikelihood(model, weightsMinusEpsilon)) / (2 * epsilon);
// If we encounter an impossible assignment, logLikelihood will return negative infinity, which will
// screw with the definitional calculation
if (double.IsNaN(component[j]))
{
component[j] = 0.0;
}
}
goldGradient.SetDenseComponent(i, component);
}
return(goldGradient);
}
public virtual void ApplyToDerivative(ConcatVector derivative)
{
if (isSparse)
{
derivative.SetSparseComponent(component, index, 0.0);
}
else
{
derivative.SetDenseComponent(component, new double[] { 0.0 });
}
}
public virtual void ApplyToWeights(ConcatVector weights)
{
if (isSparse)
{
weights.SetSparseComponent(component, index, value);
}
else
{
weights.SetDenseComponent(component, arr);
}
}
// Creates the multivector
public virtual ConcatVector Create()
{
ConcatVector mv = new ConcatVector(componentSizes.Length);
for (int i = 0; i < componentSizes.Length; i++)
{
if (componentSizes[i] == -1)
{
mv.SetSparseComponent(i, sparseOffsets[i], sparseValues[i]);
}
else
{
mv.SetDenseComponent(i, densePieces[i]);
}
}
return(mv);
}
public virtual void TestOptimizeLogLikelihood(AbstractBatchOptimizer optimizer, GraphicalModel[] dataset, ConcatVector initialWeights, double l2regularization)
{
AbstractDifferentiableFunction <GraphicalModel> ll = new LogLikelihoodDifferentiableFunction();
ConcatVector finalWeights = optimizer.Optimize((GraphicalModel[])dataset, ll, (ConcatVector)initialWeights, (double)l2regularization, 1.0e-9, true);
System.Console.Error.WriteLine("Finished optimizing");
double logLikelihood = GetValueSum((GraphicalModel[])dataset, finalWeights, ll, (double)l2regularization);
// Check in a whole bunch of random directions really nearby that there is no nearby point with a higher log
// likelihood
Random r = new Random(42);
for (int i = 0; i < 1000; i++)
{
int size = finalWeights.GetNumberOfComponents();
ConcatVector randomDirection = new ConcatVector(size);
for (int j = 0; j < size; j++)
{
double[] dense = new double[finalWeights.IsComponentSparse(j) ? finalWeights.GetSparseIndex(j) + 1 : finalWeights.GetDenseComponent(j).Length];
for (int k = 0; k < dense.Length; k++)
{
dense[k] = (r.NextDouble() - 0.5) * 1.0e-3;
}
randomDirection.SetDenseComponent(j, dense);
}
ConcatVector randomPerturbation = finalWeights.DeepClone();
randomPerturbation.AddVectorInPlace(randomDirection, 1.0);
double randomPerturbedLogLikelihood = GetValueSum((GraphicalModel[])dataset, randomPerturbation, ll, (double)l2regularization);
// Check that we're within a very small margin of error (around 3 decimal places) of the randomly
// discovered value
if (logLikelihood < randomPerturbedLogLikelihood - (1.0e-3 * Math.Max(1.0, Math.Abs(logLikelihood))))
{
System.Console.Error.WriteLine("Thought optimal point was: " + logLikelihood);
System.Console.Error.WriteLine("Discovered better point: " + randomPerturbedLogLikelihood);
}
NUnit.Framework.Assert.IsTrue(logLikelihood >= randomPerturbedLogLikelihood - (1.0e-3 * Math.Max(1.0, Math.Abs(logLikelihood))));
}
}
private void GenerateCliques(int[] variableSizes, IList <int> startSet, ICollection <int> alreadyRepresented, GraphicalModel model, SourceOfRandomness randomness)
{
if (alreadyRepresented.Count == variableSizes.Length)
{
return;
}
// Generate the clique variable set
IList <int> cliqueContents = new List <int>();
Sharpen.Collections.AddAll(cliqueContents, startSet);
Sharpen.Collections.AddAll(alreadyRepresented, startSet);
while (true)
{
if (alreadyRepresented.Count == variableSizes.Length)
{
break;
}
if (cliqueContents.Count == 0 || randomness.NextDouble(0, 1) < 0.7)
{
int gen;
do
{
gen = randomness.NextInt(variableSizes.Length);
}while (alreadyRepresented.Contains(gen));
alreadyRepresented.Add(gen);
cliqueContents.Add(gen);
}
else
{
break;
}
}
// Create the actual table
int[] neighbors = new int[cliqueContents.Count];
int[] neighborSizes = new int[neighbors.Length];
for (int j = 0; j < neighbors.Length; j++)
{
neighbors[j] = cliqueContents[j];
neighborSizes[j] = variableSizes[neighbors[j]];
}
ConcatVectorTable table = new ConcatVectorTable(neighborSizes);
foreach (int[] assignment in table)
{
// Generate a vector
ConcatVector v = new ConcatVector(ConcatVecComponents);
for (int x = 0; x < ConcatVecComponents; x++)
{
if (randomness.NextBoolean())
{
v.SetSparseComponent(x, randomness.NextInt(32), randomness.NextDouble());
}
else
{
double[] val = new double[randomness.NextInt(12)];
for (int y = 0; y < val.Length; y++)
{
val[y] = randomness.NextDouble();
}
v.SetDenseComponent(x, val);
}
}
// set vec in table
table.SetAssignmentValue(assignment, null);
}
model.AddFactor(table, neighbors);
// Pick the number of children
IList <int> availableVariables = new List <int>();
Sharpen.Collections.AddAll(availableVariables, cliqueContents);
availableVariables.RemoveAll(startSet);
int numChildren = randomness.NextInt(0, availableVariables.Count);
if (numChildren == 0)
{
return;
}
IList <IList <int> > children = new List <IList <int> >();
for (int i = 0; i < numChildren; i++)
{
children.Add(new List <int>());
}
// divide up the shared variables across the children
int cursor = 0;
while (true)
{
if (availableVariables.Count == 0)
{
break;
}
if (children[cursor].Count == 0 || randomness.NextBoolean())
{
int gen = randomness.NextInt(availableVariables.Count);
children[cursor].Add(availableVariables[gen]);
availableVariables.Remove(availableVariables[gen]);
}
else
{
break;
}
cursor = (cursor + 1) % numChildren;
}
foreach (IList <int> shared1 in children)
{
foreach (int i_1 in shared1)
{
foreach (IList <int> shared2 in children)
{
System.Diagnostics.Debug.Assert((shared1 == shared2 || !shared2.Contains(i_1)));
}
}
}
foreach (IList <int> shared in children)
{
if (shared.Count > 0)
{
GenerateCliques(variableSizes, shared, alreadyRepresented, model, randomness);
}
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.TypeLoadException"/>
public static void Main(string[] args)
{
//////////////////////////////////////////////////////////////
// Generate the CoNLL CliqueTrees to use during gameplay
//////////////////////////////////////////////////////////////
CoNLLBenchmark coNLL = new CoNLLBenchmark();
IList <CoNLLBenchmark.CoNLLSentence> train = coNLL.GetSentences(DataPath + "conll.iob.4class.train");
IList <CoNLLBenchmark.CoNLLSentence> testA = coNLL.GetSentences(DataPath + "conll.iob.4class.testa");
IList <CoNLLBenchmark.CoNLLSentence> testB = coNLL.GetSentences(DataPath + "conll.iob.4class.testb");
IList <CoNLLBenchmark.CoNLLSentence> allData = new List <CoNLLBenchmark.CoNLLSentence>();
Sharpen.Collections.AddAll(allData, train);
Sharpen.Collections.AddAll(allData, testA);
Sharpen.Collections.AddAll(allData, testB);
ICollection <string> tagsSet = new HashSet <string>();
foreach (CoNLLBenchmark.CoNLLSentence sentence in allData)
{
foreach (string nerTag in sentence.ner)
{
tagsSet.Add(nerTag);
}
}
IList <string> tags = new List <string>();
Sharpen.Collections.AddAll(tags, tagsSet);
coNLL.embeddings = coNLL.GetEmbeddings(DataPath + "google-300-trimmed.ser.gz", allData);
log.Info("Making the training set...");
ConcatVectorNamespace @namespace = new ConcatVectorNamespace();
int trainSize = train.Count;
GraphicalModel[] trainingSet = new GraphicalModel[trainSize];
for (int i = 0; i < trainSize; i++)
{
if (i % 10 == 0)
{
log.Info(i + "/" + trainSize);
}
trainingSet[i] = coNLL.GenerateSentenceModel(@namespace, train[i], tags);
}
//////////////////////////////////////////////////////////////
// Generate the random human observation feature vectors that we'll use
//////////////////////////////////////////////////////////////
Random r = new Random(10);
int numFeatures = 5;
int featureLength = 30;
ConcatVector[] humanFeatureVectors = new ConcatVector[1000];
for (int i_1 = 0; i_1 < humanFeatureVectors.Length; i_1++)
{
humanFeatureVectors[i_1] = new ConcatVector(numFeatures);
for (int j = 0; j < numFeatures; j++)
{
if (r.NextBoolean())
{
humanFeatureVectors[i_1].SetSparseComponent(j, r.NextInt(featureLength), r.NextDouble());
}
else
{
double[] dense = new double[featureLength];
for (int k = 0; k < dense.Length; k++)
{
dense[k] = r.NextDouble();
}
humanFeatureVectors[i_1].SetDenseComponent(j, dense);
}
}
}
ConcatVector weights = new ConcatVector(numFeatures);
for (int i_2 = 0; i_2 < numFeatures; i_2++)
{
double[] dense = new double[featureLength];
for (int j = 0; j < dense.Length; j++)
{
dense[j] = r.NextDouble();
}
weights.SetDenseComponent(i_2, dense);
}
//////////////////////////////////////////////////////////////
// Actually perform gameplay-like random mutations
//////////////////////////////////////////////////////////////
log.Info("Warming up the JIT...");
for (int i_3 = 0; i_3 < 10; i_3++)
{
log.Info(i_3);
Gameplay(r, trainingSet[i_3], weights, humanFeatureVectors);
}
log.Info("Timing actual run...");
long start = Runtime.CurrentTimeMillis();
for (int i_4 = 0; i_4 < 10; i_4++)
{
log.Info(i_4);
Gameplay(r, trainingSet[i_4], weights, humanFeatureVectors);
}
long duration = Runtime.CurrentTimeMillis() - start;
log.Info("Duration: " + duration);
}