/// <summary>
/// Creates the initial scatter plot graphs containing some random
/// data. This data is generated by sampling Gaussian distributions.
/// </summary>
///
private void btnGenerateRandom_Click(object sender, EventArgs e)
{
k = (int)numClusters.Value;
// Generate data with n Gaussian distributions
double[][][] data = new double[k][][];
for (int i = 0; i < k; i++)
{
// Create random centroid to place the Gaussian distribution
double[] mean = Matrix.Random(2, -6.0, +6.0);
// Create random covariance matrix for the distribution
double[,] covariance = Accord.Statistics.Tools.RandomCovariance(2, -5, 5);
// Create the Gaussian distribution
var gaussian = new MultivariateNormalDistribution(mean, covariance);
int samples = Accord.Math.Tools.Random.Next(150, 250);
data[i] = gaussian.Generate(samples);
}
// Join the generated data
mixture = Matrix.Stack(data);
// Update the scatter plot
CreateScatterplot(graph, mixture, k);
// Forget previous initialization
kmeans = null;
}
public static HiddenMarkovModel<MultivariateNormalDistribution> deserialize(string filename)
{
using (StreamReader r = new StreamReader(filename))
{
String data = r.ReadToEnd();
String[] dataArr = data.Split('#');
// probDelimitedStr dataArr[0]
double[] prob = createSingleDimDoubleArray(dataArr[0], '|');
// transDelimitedStr dataArr[1]
double[,] trans = createDoubleDimDoubleArray(dataArr[1], '|', '*');
// emissionsDelimitedStr dataArr[2]
String[] emissions = dataArr[2].Split('$');
MultivariateNormalDistribution[] e2 = new MultivariateNormalDistribution[emissions.Length];
for (int i = 0; i < emissions.Length; i++)
{
String[] meansNCovariance = emissions[i].Split('&');
String meansStr = meansNCovariance[0];
String covarianceStr = meansNCovariance[1];
double[] means = createSingleDimDoubleArray(meansStr, '|');
double[,] covariance = createDoubleDimDoubleArray(covarianceStr, '|', '*');
MultivariateNormalDistribution dist = new MultivariateNormalDistribution(means, covariance);
e2[i] = dist;
}
HiddenMarkovModel<MultivariateNormalDistribution> hmm = new HiddenMarkovModel<MultivariateNormalDistribution>(trans, e2, prob);
return hmm;
}
}
public void MultivariateNormalGenerateTest()
{
// mean vector
double[] mu = { 2.0, 6.0 };
// covariance
double[,] cov =
{
{ 2, 1 },
{ 1, 5 }
};
// Create a multivariate Normal distribution
var normal = new MultivariateNormalDistribution(mu, cov);
// Generate 1000000 samples from it
double[][] samples = normal.Generate(1000000);
// Try to estimate a new Normal distribution from
// generated samples to check if they indeed match
var actual = MultivariateNormalDistribution.Estimate(samples);
Assert.IsTrue(mu.IsEqual(actual.Mean, 0.1));
Assert.IsTrue(cov.IsEqual(actual.Covariance, 0.1));
}
public static HiddenMarkovClassifier<MultivariateNormalDistribution> CreateModel1()
{
// Create a Continuous density Hidden Markov Model Sequence Classifier
// to detect a multivariate sequence and the same sequence backwards.
double[][][] sequences = new double[][][]
{
new double[][]
{
// This is the first sequence with label = 0
new double[] { 0 },
new double[] { 1 },
new double[] { 2 },
new double[] { 3 },
new double[] { 4 },
},
new double[][]
{
// This is the second sequence with label = 1
new double[] { 4 },
new double[] { 3 },
new double[] { 2 },
new double[] { 1 },
new double[] { 0 },
}
};
// Labels for the sequences
int[] labels = { 0, 1 };
// Creates a sequence classifier containing 2 hidden Markov Models
// with 2 states and an underlying Normal distribution as density.
MultivariateNormalDistribution density = new MultivariateNormalDistribution(1);
var classifier = new HiddenMarkovClassifier<MultivariateNormalDistribution>(2, new Ergodic(2), density);
// Configure the learning algorithms to train the sequence classifier
var teacher = new HiddenMarkovClassifierLearning<MultivariateNormalDistribution>(classifier,
// Train each model until the log-likelihood changes less than 0.001
modelIndex => new BaumWelchLearning<MultivariateNormalDistribution>(classifier.Models[modelIndex])
{
Tolerance = 0.0001,
Iterations = 0
}
);
// Train the sequence classifier using the algorithm
double logLikelihood = teacher.Run(sequences, labels);
return classifier;
}
public void ConstructorTest1()
{
MultivariateNormalDistribution[] components = new MultivariateNormalDistribution[2];
components[0] = new MultivariateNormalDistribution(new double[] { 2 }, new double[,] { { 1 } });
components[1] = new MultivariateNormalDistribution(new double[] { 5 }, new double[,] { { 1 } });
var mixture = new MultivariateMixture<MultivariateNormalDistribution>(components);
double[] expected = { 0.5, 0.5 };
Assert.IsTrue(expected.IsEqual(mixture.Coefficients));
Assert.AreEqual(components, mixture.Components);
}
public void ConstructorTest4()
{
// Create a multivariate Gaussian distribution
var dist = new MultivariateNormalDistribution(
// mean vector mu
mean: new double[]
{
4, 2
},
// covariance matrix sigma
covariance: new double[,]
{
{ 0.3, 0.1 },
{ 0.1, 0.7 }
}
);
// Common measures
double[] mean = dist.Mean; // { 4, 2 }
double[] median = dist.Median; // { 4, 2 }
double[] var = dist.Variance; // { 0.3, 0.7 } (diagonal from cov)
double[,] cov = dist.Covariance; // { { 0.3, 0.1 }, { 0.1, 0.7 } }
// Probability mass functions
double pdf1 = dist.ProbabilityDensityFunction(new double[] { 2, 5 }); // 0.000000018917884164743237
double pdf2 = dist.ProbabilityDensityFunction(new double[] { 4, 2 }); // 0.35588127170858852
double pdf3 = dist.ProbabilityDensityFunction(new double[] { 3, 7 }); // 0.000000000036520107734505265
double lpdf = dist.LogProbabilityDensityFunction(new double[] { 3, 7 }); // -24.033158110192296
// Cumulative distribution function (for up to two dimensions)
double cdf = dist.DistributionFunction(new double[] { 3, 5 }); // 0.033944035782101548
Assert.AreEqual(4, mean[0]);
Assert.AreEqual(2, mean[1]);
Assert.AreEqual(4, median[0]);
Assert.AreEqual(2, median[1]);
Assert.AreEqual(0.3, var[0]);
Assert.AreEqual(0.7, var[1]);
Assert.AreEqual(0.3, cov[0, 0]);
Assert.AreEqual(0.1, cov[0, 1]);
Assert.AreEqual(0.1, cov[1, 0]);
Assert.AreEqual(0.7, cov[1, 1]);
Assert.AreEqual(0.000000018917884164743237, pdf1);
Assert.AreEqual(0.35588127170858852, pdf2);
Assert.AreEqual(0.000000000036520107734505265, pdf3);
Assert.AreEqual(-24.033158110192296, lpdf);
Assert.AreEqual(0.033944035782101548, cdf);
}
public void ProbabilityDensityFunctionTest()
{
MultivariateNormalDistribution[] components = new MultivariateNormalDistribution[2];
components[0] = new MultivariateNormalDistribution(new double[] { 2 }, new double[,] { { 1 } });
components[1] = new MultivariateNormalDistribution(new double[] { 5 }, new double[,] { { 1 } });
double[] coefficients = { 0.3, 0.7 };
var mixture = new MultivariateMixture<MultivariateNormalDistribution>(coefficients, components);
double[] x = { 1.2 };
double expected =
0.3 * components[0].ProbabilityDensityFunction(x) +
0.7 * components[1].ProbabilityDensityFunction(x);
double actual = mixture.ProbabilityDensityFunction(x);
Assert.AreEqual(expected, actual);
}
/*public static void SaveSequenceList(SequenceList seqList, string path)
{
Stream writeStream = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.None);
seqList.Save(writeStream);
writeStream.Close();
}
public static SequenceList LoadSequenceList(string path)
{
Stream readStream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read);
SequenceList seqList = SequenceList.Load(readStream);
readStream.Close();
return seqList;
}*/
public static HiddenMarkovModel<MultivariateNormalDistribution> CreateModelFromFrames(List<List<Frame>> frames)
{
SequenceList sequences = Utils.FramesToSequenceList(frames);
HiddenMarkovModel<MultivariateNormalDistribution> hmm;
MultivariateNormalDistribution mnd = new MultivariateNormalDistribution(sequences.GetDimensions());
hmm = new HiddenMarkovModel<MultivariateNormalDistribution>(new Forward(5), mnd);
var teacher = new BaumWelchLearning<MultivariateNormalDistribution>(hmm);
teacher.Tolerance = 0.0001;
teacher.Iterations = 0;
teacher.FittingOptions = new NormalOptions()
{
Diagonal = true, // only diagonal covariance matrices
Regularization = 1e-5 // avoid non-positive definite errors
};
teacher.Run(sequences.GetArray());
return hmm;
}
public void ConstructorTest1()
{
NormalDistribution normal = new NormalDistribution(4.2, 1.2);
MultivariateNormalDistribution target = new MultivariateNormalDistribution(new[] { 4.2 }, new[,] { { 1.2 * 1.2 } });
double[] mean = target.Mean;
double[] median = target.Median;
double[] var = target.Variance;
double[,] cov = target.Covariance;
double apdf1 = target.ProbabilityDensityFunction(new double[] { 2 });
double apdf2 = target.ProbabilityDensityFunction(new double[] { 4 });
double apdf3 = target.ProbabilityDensityFunction(new double[] { 3 });
double alpdf = target.LogProbabilityDensityFunction(new double[] { 3 });
double acdf = target.DistributionFunction(new double[] { 3 });
double accdf = target.ComplementaryDistributionFunction(new double[] { 3 });
double epdf1 = target.ProbabilityDensityFunction(new double[] { 2 });
double epdf2 = target.ProbabilityDensityFunction(new double[] { 4 });
double epdf3 = target.ProbabilityDensityFunction(new double[] { 3 });
double elpdf = target.LogProbabilityDensityFunction(new double[] { 3 });
double ecdf = target.DistributionFunction(new double[] { 3 });
double eccdf = target.ComplementaryDistributionFunction(new double[] { 3 });
Assert.AreEqual(normal.Mean, target.Mean[0]);
Assert.AreEqual(normal.Median, target.Median[0]);
Assert.AreEqual(normal.Variance, target.Variance[0]);
Assert.AreEqual(normal.Variance, target.Covariance[0, 0]);
Assert.AreEqual(epdf1, apdf1);
Assert.AreEqual(epdf2, apdf2);
Assert.AreEqual(epdf3, apdf3);
Assert.AreEqual(elpdf, alpdf);
Assert.AreEqual(ecdf, acdf);
Assert.AreEqual(eccdf, accdf);
Assert.AreEqual(1.0 - ecdf, eccdf);
}
public void MixtureWeightsFitTest()
{
// Randomly initialize some mixture components
MultivariateNormalDistribution[] components = new MultivariateNormalDistribution[2];
components[0] = new MultivariateNormalDistribution(new double[] { 2 }, new double[,] { { 1 } });
components[1] = new MultivariateNormalDistribution(new double[] { 5 }, new double[,] { { 1 } });
// Create an initial mixture
var mixture = new MultivariateMixture<MultivariateNormalDistribution>(components);
// Now, suppose we have a weighted data
// set. Those will be the input points:
double[][] points = new double[] { 0, 3, 1, 7, 3, 5, 1, 2, -1, 2, 7, 6, 8, 6 } // (14 points)
.ToArray();
// And those are their respective unormalized weights:
double[] weights = { 1, 1, 1, 2, 2, 1, 1, 1, 2, 1, 2, 3, 1, 1 }; // (14 weights)
// Let's normalize the weights so they sum up to one:
weights = weights.Divide(weights.Sum());
// Now we can fit our model to the data:
mixture.Fit(points, weights); // done!
// Our model will be:
double mean1 = mixture.Components[0].Mean[0]; // 1.41126
double mean2 = mixture.Components[1].Mean[0]; // 6.53301
// If we need the GaussianMixtureModel functionality, we can
// use the estimated mixture to initialize a new model:
GaussianMixtureModel gmm = new GaussianMixtureModel(mixture);
Assert.AreEqual(mean1, gmm.Gaussians[0].Mean[0]);
Assert.AreEqual(mean2, gmm.Gaussians[1].Mean[0]);
Assert.AreEqual(1.4112610766836404, mean1, 1e-10);
Assert.AreEqual(6.5330177004151082, mean2, 1e-10);
Assert.AreEqual(mixture.Coefficients[0], gmm.Gaussians[0].Proportion);
Assert.AreEqual(mixture.Coefficients[1], gmm.Gaussians[1].Proportion);
}
public void FitTest2()
{
double[] coefficients = { 0.50, 0.50 };
MultivariateNormalDistribution[] components = new MultivariateNormalDistribution[2];
components[0] = new MultivariateNormalDistribution(new double[] { 2 }, new double[,] { { 1 } });
components[1] = new MultivariateNormalDistribution(new double[] { 5 }, new double[,] { { 1 } });
var target = new MultivariateMixture<MultivariateNormalDistribution>(coefficients, components);
double[][] values = { new double[] { 2512512312 },
new double[] { 1 },
new double[] { 1 },
new double[] { 0 },
new double[] { 1 },
new double[] { 6 },
new double[] { 6 },
new double[] { 5 },
new double[] { 7 },
new double[] { 5 } };
double[] weights = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 1 };
weights = weights.Divide(weights.Sum());
double[][] part1 = values.Submatrix(1, 4);
double[][] part2 = values.Submatrix(5, 9);
target.Fit(values, weights);
var mean1 = Accord.Statistics.Tools.Mean(part1);
var var1 = Accord.Statistics.Tools.Variance(part1);
Assert.AreEqual(mean1[0], target.Components[0].Mean[0], 1e-5);
Assert.AreEqual(var1[0], target.Components[0].Variance[0], 1e-5);
var mean2 = Accord.Statistics.Tools.Mean(part2);
var var2 = Accord.Statistics.Tools.Variance(part2);
Assert.AreEqual(mean2[0], target.Components[1].Mean[0], 1e-5);
Assert.AreEqual(var2[0], target.Components[1].Variance[0], 1e-5);
var expectedMean = Accord.Statistics.Tools.WeightedMean(values, weights);
var expectedVar = Accord.Statistics.Tools.WeightedCovariance(values, weights);
var actualMean = target.Mean;
var actualVar = target.Covariance;
Assert.AreEqual(expectedMean[0], actualMean[0], 0.0000001);
Assert.AreEqual(expectedVar[0, 0], actualVar[0, 0], 0.68);
}
public void ProbabilityDensityFunctionTest()
{
double[] mean = { 1, -1 };
double[,] covariance =
{
{ 0.9, 0.4 },
{ 0.4, 0.3 },
};
var target = new MultivariateNormalDistribution(mean, covariance);
double[] x = { 1.2, -0.8 };
double expected = 0.446209421363460;
double actual = target.ProbabilityDensityFunction(x);
Assert.AreEqual(expected, actual, 0.00000001);
}
public void FitTest2()
{
double[][] observations =
{
new double[] { 1, 2 },
new double[] { 1, 2 },
new double[] { 1, 2 },
new double[] { 1, 2 }
};
var target = new MultivariateNormalDistribution(2);
bool thrown = false;
try { target.Fit(observations); }
catch (NonPositiveDefiniteMatrixException) { thrown = true; }
Assert.IsTrue(thrown);
NormalOptions options = new NormalOptions() { Regularization = double.Epsilon };
// No exception thrown
target.Fit(observations, options);
}
public void FitTest()
{
double[][] observations =
{
new double[] { 0.1000, -0.2000 },
new double[] { 0.4000, 0.6000 },
new double[] { 2.0000, 0.2000 },
new double[] { 2.0000, 0.3000 }
};
double[] mean = Accord.Statistics.Tools.Mean(observations);
double[,] cov = Accord.Statistics.Tools.Covariance(observations);
var target = new MultivariateNormalDistribution(2);
double[] weigths = { 0.25, 0.25, 0.25, 0.25 };
target.Fit(observations, weigths);
Assert.IsTrue(Matrix.IsEqual(mean, target.Mean));
Assert.IsTrue(Matrix.IsEqual(cov, target.Covariance));
}
public void ConstructorTest()
{
double[] mean = { 1, -1 };
double[,] covariance =
{
{ 2, 1 },
{ 1, 3 }
};
MultivariateNormalDistribution target = new MultivariateNormalDistribution(mean, covariance);
Assert.AreEqual(covariance, target.Covariance);
Assert.AreEqual(mean, target.Mean);
Assert.AreEqual(2, target.Variance.Length);
Assert.AreEqual(2, target.Variance[0]);
Assert.AreEqual(3, target.Variance[1]);
Assert.AreEqual(2, target.Dimension);
}
private static HiddenMarkovModel<MultivariateNormalDistribution> createModel()
{
double[][][] sequences =
{
new double[][]
{
new double[] { 1, 2 },
new double[] { 6, 7 },
new double[] { 2, 3 },
},
new double[][]
{
new double[] { 2, 2 },
new double[] { 9, 8 },
new double[] { 1, 0 },
},
new double[][]
{
new double[] { 1, 3 },
new double[] { 8, 9 },
new double[] { 3, 3 },
},
};
var density = new MultivariateNormalDistribution(dimension: 2);
var model = new HiddenMarkovModel<MultivariateNormalDistribution>(new Forward(2), density);
var teacher = new BaumWelchLearning<MultivariateNormalDistribution>(model)
{
Tolerance = 0.0001,
Iterations = 0,
};
double logLikelihood = teacher.Run(sequences);
return model;
}
public void GenerateTest2()
{
Accord.Math.Tools.SetupGenerator(0);
var normal = new MultivariateNormalDistribution(
new double[] { 2, 6 },
new double[,] { { 2, 1 }, { 1, 5 } });
double[][] sample = new double[1000000][];
for (int i = 0; i < sample.Length; i++)
sample[i] = normal.Generate();
double[] mean = sample.Mean();
double[,] cov = sample.Covariance();
Assert.AreEqual(2, mean[0], 1e-2);
Assert.AreEqual(6, mean[1], 1e-2);
Assert.AreEqual(2, cov[0, 0], 1e-2);
Assert.AreEqual(1, cov[0, 1], 1e-2);
Assert.AreEqual(1, cov[1, 0], 1e-2);
Assert.AreEqual(5, cov[1, 1], 2e-2);
}
public void DecodeTest4()
{
var density = new MultivariateNormalDistribution(3);
var hmm = new HiddenMarkovModel<MultivariateNormalDistribution>(2, density);
bool thrown = false;
try
{
double logLikelihood;
int[] path = hmm.Decode(new double[] { 0, 1, 2 }, out logLikelihood);
}
catch
{
thrown = true;
}
Assert.IsTrue(thrown);
}
public void LogForwardTest3()
{
MultivariateNormalDistribution density = new MultivariateNormalDistribution(3);
var hmm = new HiddenMarkovClassifier<MultivariateNormalDistribution>(2, new Ergodic(2), density);
double[][][] inputs =
{
new [] { new double[] { 0, 1, 0 }, new double[] { 0, 1, 0 }, new double[] { 0, 1, 0 } },
new [] { new double[] { 1, 6, 2 }, new double[] { 2, 1, 6 }, new double[] { 1, 1, 0 } },
new [] { new double[] { 9, 1, 0 }, new double[] { 0, 1, 5 }, new double[] { 0, 0, 0 } },
};
int[] outputs =
{
0, 0, 1
};
var function = new MarkovMultivariateFunction(hmm);
var observations = inputs[0];
double[,] expected = Matrix.Log(Accord.Statistics.Models.Fields.
ForwardBackwardAlgorithm.Forward(function.Factors[0], observations, 0));
double logLikelihood;
double[,] actual = Accord.Statistics.Models.Fields.
ForwardBackwardAlgorithm.LogForward(function.Factors[0], observations, 0, out logLikelihood);
Assert.IsTrue(expected.IsEqual(actual, 1e-10));
double p = 0;
for (int i = 0; i < hmm[0].States; i++)
p += Math.Exp(actual[observations.Length - 1, i]);
Assert.AreEqual(Math.Exp(logLikelihood), p, 1e-8);
Assert.IsFalse(double.IsNaN(p));
}
public void ConstructorTest2()
{
double[,] A = new double[,]
{
{ 0.5, 0.5 },
{ 0.5, 0.5 }
};
double[] pi = new double[] { 1, 0 };
var distribution = new MultivariateNormalDistribution(3);
var hmm = new HiddenMarkovModel<MultivariateNormalDistribution>(2, distribution);
for (int i = 0; i < hmm.Emissions.Length; i++)
{
IDistribution b = hmm.Emissions[i];
Assert.AreNotSame(distribution, b);
Assert.IsTrue(b is MultivariateNormalDistribution);
MultivariateNormalDistribution n = b as MultivariateNormalDistribution;
Assert.AreEqual(n.Dimension, hmm.Dimension);
Assert.AreNotEqual(n.Covariance, distribution.Covariance);
Assert.IsTrue(n.Covariance.IsEqual(distribution.Covariance));
Assert.AreNotEqual(n.Mean, distribution.Mean);
Assert.IsTrue(n.Mean.IsEqual(distribution.Mean));
}
Assert.AreEqual(2, hmm.States);
Assert.AreEqual(3, hmm.Dimension);
Assert.AreEqual(2, hmm.Emissions.Length);
var logA = Matrix.Log(A);
var logPi = Matrix.Log(pi);
Assert.IsTrue(logA.IsEqual(hmm.Transitions));
Assert.IsTrue(logPi.IsEqual(hmm.Probabilities));
}
public void FittingOptionsTest()
{
// Create a degenerate problem
double[][] sequences = new double[][]
{
new double[] { 1,1,1,1,1,0,1,1,1,1 },
new double[] { 1,1,1,1,0,1,1,1,1,1 },
new double[] { 1,1,1,1,1,1,1,1,1,1 },
new double[] { 1,1,1,1,1,1 },
new double[] { 1,1,1,1,1,1,1 },
new double[] { 1,1,1,1,1,1,1,1,1,1 },
new double[] { 1,1,1,1,1,1,1,1,1,1 },
};
// Creates a continuous hidden Markov Model with two states organized in a ergodic
// topology and an underlying multivariate Normal distribution as density.
var density = new MultivariateNormalDistribution(1);
var model = new HiddenMarkovModel<MultivariateNormalDistribution>(new Ergodic(2), density);
// Configure the learning algorithms to train the sequence classifier
var teacher = new BaumWelchLearning<MultivariateNormalDistribution>(model)
{
Tolerance = 0.0001,
Iterations = 0,
// Configure options for fitting the normal distribution
FittingOptions = new NormalOptions() { Regularization = 0.0001, }
};
// Fit the model. No exceptions will be thrown
double logLikelihood = teacher.Run(sequences);
double likelihood = Math.Exp(logLikelihood);
Assert.AreEqual(47.434837528491286, logLikelihood, 1e-15);
Assert.IsFalse(double.IsNaN(logLikelihood));
Assert.AreEqual(0.0001, (teacher.FittingOptions as NormalOptions).Regularization);
// Try without a regularization constant to get an exception
bool thrown;
thrown = false;
density = new MultivariateNormalDistribution(1);
model = new HiddenMarkovModel<MultivariateNormalDistribution>(new Ergodic(2), density);
teacher = new BaumWelchLearning<MultivariateNormalDistribution>(model) { Tolerance = 0.0001, Iterations = 0, };
Assert.IsNull(teacher.FittingOptions);
try { teacher.Run(sequences); }
catch { thrown = true; }
Assert.IsTrue(thrown);
thrown = false;
density = new Accord.Statistics.Distributions.Multivariate.MultivariateNormalDistribution(1);
model = new HiddenMarkovModel<MultivariateNormalDistribution>(new Ergodic(2), density);
teacher = new BaumWelchLearning<MultivariateNormalDistribution>(model)
{
Tolerance = 0.0001,
Iterations = 0,
FittingOptions = new NormalOptions() { Regularization = 0 }
};
Assert.IsNotNull(teacher.FittingOptions);
try { teacher.Run(sequences); }
catch { thrown = true; }
Assert.IsTrue(thrown);
}
public void LearnTest10()
{
// Create sequences of vector-valued observations. In the
// sequence below, a single observation is composed of two
// coordinate values, such as (x, y). There seems to be two
// states, one for (x,y) values less than (5,5) and another
// for higher values. The states seems to be switched on
// every observation.
double[][][] sequences =
{
new double[][] // sequence 1
{
new double[] { 1, 2 }, // observation 1 of sequence 1
new double[] { 6, 7 }, // observation 2 of sequence 1
new double[] { 2, 3 }, // observation 3 of sequence 1
},
new double[][] // sequence 2
{
new double[] { 2, 2 }, // observation 1 of sequence 2
new double[] { 9, 8 }, // observation 2 of sequence 2
new double[] { 1, 0 }, // observation 3 of sequence 2
},
new double[][] // sequence 3
{
new double[] { 1, 3 }, // observation 1 of sequence 3
new double[] { 8, 9 }, // observation 2 of sequence 3
new double[] { 3, 3 }, // observation 3 of sequence 3
},
};
// Specify a initial normal distribution for the samples.
var density = new MultivariateNormalDistribution(dimension: 2);
// Creates a continuous hidden Markov Model with two states organized in a forward
// topology and an underlying univariate Normal distribution as probability density.
var model = new HiddenMarkovModel<MultivariateNormalDistribution>(new Forward(2), density);
// Configure the learning algorithms to train the sequence classifier until the
// difference in the average log-likelihood changes only by as little as 0.0001
var teacher = new BaumWelchLearning<MultivariateNormalDistribution>(model)
{
Tolerance = 0.0001,
Iterations = 0,
};
// Fit the model
double logLikelihood = teacher.Run(sequences);
// See the likelihood of the sequences learned
double a1 = Math.Exp(model.Evaluate(new[] {
new double[] { 1, 2 },
new double[] { 6, 7 },
new double[] { 2, 3 }})); // 0.000208
double a2 = Math.Exp(model.Evaluate(new[] {
new double[] { 2, 2 },
new double[] { 9, 8 },
new double[] { 1, 0 }})); // 0.0000376
// See the likelihood of an unrelated sequence
double a3 = Math.Exp(model.Evaluate(new[] {
new double[] { 8, 7 },
new double[] { 9, 8 },
new double[] { 1, 0 }})); // 2.10 x 10^(-89)
Assert.AreEqual(0.00020825319093038984, a1);
Assert.AreEqual(0.000037671116792519834, a2);
Assert.AreEqual(2.1031924118199194E-89, a3);
}
public void MixtureWeightsFitTest2()
{
MemoryStream stream = new MemoryStream(Resources.CircleWithWeights);
ExcelReader reader = new ExcelReader(stream, xlsx: false);
DataTable table = reader.GetWorksheet("Sheet1");
double[,] matrix = table.ToMatrix();
double[][] points = matrix.Submatrix(null, 0, 1).ToArray();
double[] weights = matrix.GetColumn(2);
// Randomly initialize some mixture components
MultivariateNormalDistribution[] components = new MultivariateNormalDistribution[2];
components[0] = new MultivariateNormalDistribution(new double[] { 0, 1 }, Matrix.Identity(2));
components[1] = new MultivariateNormalDistribution(new double[] { 1, 0 }, Matrix.Identity(2));
// Create an initial mixture
var mixture = new MultivariateMixture<MultivariateNormalDistribution>(components);
mixture.Fit(points, weights);
// Our model will be:
double mean00 = mixture.Components[0].Mean[0];
double mean01 = mixture.Components[0].Mean[1];
double mean10 = mixture.Components[1].Mean[0];
double mean11 = mixture.Components[1].Mean[1];
Assert.AreEqual(-0.11704994950834195, mean00, 1e-10);
Assert.AreEqual(0.11603470123007256, mean01, 1e-10);
Assert.AreEqual(0.11814483652855159, mean10, 1e-10);
Assert.AreEqual(-0.12029275652994373, mean11, 1e-10);
}
public void ProbabilityDensityFunctionTest3()
{
double[] mean = new double[3];
double[,] covariance = Matrix.Identity(3);
var target = new MultivariateNormalDistribution(mean, covariance);
double[] x = { 1.2, -0.8 };
bool thrown = false;
try
{
target.ProbabilityDensityFunction(x);
}
catch (DimensionMismatchException)
{
thrown = true;
}
Assert.IsTrue(thrown);
}
public void FittingOptionsTest()
{
// Create a degenerate problem
double[][] sequences = new double[][]
{
new double[] { 1, 1, 1, 1, 1, 0, 1, 1, 1, 1 },
new double[] { 1, 1, 1, 1, 0, 1, 1, 1, 1, 1 },
new double[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 },
new double[] { 1, 1, 1, 1, 1, 1 },
new double[] { 1, 1, 1, 1, 1, 1, 1 },
new double[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 },
new double[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 },
};
// Creates a continuous hidden Markov Model with two states organized in a ergodic
// topology and an underlying multivariate Normal distribution as density.
var density = new MultivariateNormalDistribution(1);
var model = new HiddenMarkovModel <MultivariateNormalDistribution>(new Ergodic(2), density);
// Configure the learning algorithms to train the sequence classifier
var teacher = new BaumWelchLearning <MultivariateNormalDistribution>(model)
{
Tolerance = 0.0001,
Iterations = 0,
// Configure options for fitting the normal distribution
FittingOptions = new NormalOptions()
{
Regularization = 0.0001,
}
};
// Fit the model. No exceptions will be thrown
double logLikelihood = teacher.Run(sequences);
double likelihood = Math.Exp(logLikelihood);
Assert.AreEqual(47.434837528491286, logLikelihood, 1e-15);
Assert.IsFalse(double.IsNaN(logLikelihood));
Assert.AreEqual(0.0001, (teacher.FittingOptions as NormalOptions).Regularization);
// Try without a regularization constant to get an exception
bool thrown;
thrown = false;
density = new MultivariateNormalDistribution(1);
model = new HiddenMarkovModel <MultivariateNormalDistribution>(new Ergodic(2), density);
teacher = new BaumWelchLearning <MultivariateNormalDistribution>(model)
{
Tolerance = 0.0001, Iterations = 0,
};
Assert.IsNull(teacher.FittingOptions);
try { teacher.Run(sequences); }
catch { thrown = true; }
Assert.IsTrue(thrown);
thrown = false;
density = new Accord.Statistics.Distributions.Multivariate.MultivariateNormalDistribution(1);
model = new HiddenMarkovModel <MultivariateNormalDistribution>(new Ergodic(2), density);
teacher = new BaumWelchLearning <MultivariateNormalDistribution>(model)
{
Tolerance = 0.0001,
Iterations = 0,
FittingOptions = new NormalOptions()
{
Regularization = 0
}
};
Assert.IsNotNull(teacher.FittingOptions);
try { teacher.Run(sequences); }
catch { thrown = true; }
Assert.IsTrue(thrown);
}
public void GradientTest()
{
// Creates a sequence classifier containing 2 hidden Markov Models
// with 2 states and an underlying Normal distribution as density.
MultivariateNormalDistribution density = new MultivariateNormalDistribution(3);
var hmm = new HiddenMarkovClassifier<MultivariateNormalDistribution>(2, new Ergodic(2), density);
double[][][] inputs =
{
new [] { new double[] { 0, 1, 0 }, new double[] { 0, 1, 0 }, new double[] { 0, 1, 0 } },
new [] { new double[] { 1, 6, 2 }, new double[] { 2, 1, 6 }, new double[] { 1, 1, 0 } },
new [] { new double[] { 9, 1, 0 }, new double[] { 0, 1, 5 }, new double[] { 0, 0, 0 } },
};
int[] outputs =
{
0, 0, 1
};
var function = new MarkovMultivariateFunction(hmm);
var model = new HiddenConditionalRandomField<double[]>(function);
var target = new ForwardBackwardGradient<double[]>(model);
FiniteDifferences diff = new FiniteDifferences(function.Weights.Length);
diff.Function = parameters => func(model, parameters, inputs, outputs);
double[] expected = diff.Compute(function.Weights);
double[] actual = target.Gradient(function.Weights, inputs, outputs);
for (int i = 0; i < actual.Length; i++)
{
Assert.AreEqual(expected[i], actual[i], 0.05);
Assert.IsFalse(double.IsNaN(actual[i]));
Assert.IsFalse(double.IsNaN(expected[i]));
}
}
public void DecodeTest5()
{
var density = new MultivariateNormalDistribution(3);
var hmm = new HiddenMarkovModel<MultivariateNormalDistribution>(2, density);
double logLikelihood;
int[] path = hmm.Decode(new double[][]
{
new double[] { 0, 1, 2 },
new double[] { 0, 1, 2 },
}, out logLikelihood);
Assert.AreEqual(-11.206778379787982, logLikelihood);
}
public void GenerateTest1()
{
Accord.Math.Tools.SetupGenerator(0);
double[] mean = { 2, 6 };
double[,] cov =
{
{ 2, 1 },
{ 1, 5 }
};
var normal = new MultivariateNormalDistribution(mean, cov);
double[][] source = normal.Generate(10000000);
var target = new MultivariateEmpiricalDistribution(source);
Assert.IsTrue(mean.IsEqual(target.Mean, 0.001));
Assert.IsTrue(cov.IsEqual(target.Covariance, 0.003));
double[][] samples = target.Generate(10000000);
double[] sampleMean = samples.Mean();
double[,] sampleCov = samples.Covariance();
Assert.AreEqual(2, sampleMean[0], 1e-2);
Assert.AreEqual(6, sampleMean[1], 1e-2);
Assert.AreEqual(2, sampleCov[0, 0], 1e-2);
Assert.AreEqual(1, sampleCov[0, 1], 1e-2);
Assert.AreEqual(1, sampleCov[1, 0], 1e-2);
Assert.AreEqual(5, sampleCov[1, 1], 2e-2);
}
/// <summary>
/// Generates a random vector of observations from a distribution with the given parameters.
/// </summary>
///
/// <param name="samples">The number of samples to generate.</param>
/// <param name="mean">The mean vector μ (mu) for the distribution.</param>
/// <param name="covariance">The covariance matrix Σ (sigma) for the distribution.</param>
///
/// <returns>A random vector of observations drawn from this distribution.</returns>
///
public static double[][] Generate(int samples, double[] mean, double[,] covariance)
{
var normal = new MultivariateNormalDistribution(mean, covariance);
return(normal.Generate(samples));
}
public void LearnTest9()
{
// Include this example in the documentattion
var observations = new double[][][]
{
#region example
new double[][]
{
new double[] {2.58825719356537, -6.10018078957452, -3.51826652951428,},
new double[] {1.5637531876564, -8.92844874836103, -9.09330631370717,},
new double[] {2.12242007255554, -14.8117769726059, -9.04211363915664,},
new double[] {0.39045587182045, -10.3548189544216, -7.69608701297759,},
new double[] {-0.553155690431595, -34.9185135663671, 14.6941023804174,},
new double[] {-0.923129916191101, -6.06337512248124, 8.28106954197084,},
new double[] {0.478342920541763, -4.93066650122859, 3.1120912556361,},
},
new double[][]
{
new double[] {1.89824998378754, -8.21581113387553, -7.88790716806936,},
new double[] {2.24453508853912, -10.281886698766, -9.67846789539227,},
new double[] {0.946296751499176, -22.0276392511088, -6.52238763834787,},
new double[] {-0.251136720180511, -13.3010653290676, 8.47499524273859,},
new double[] {-2.35625505447388, -18.1542111199742, 6.25564428645639,},
new double[] {0.200483202934265, -5.48215328147925, 5.88811639894938,},
},
new double[][]
{
new double[] {2.7240589261055, -3.71720542338046, -3.75092324997593,},
new double[] {2.19917744398117, -7.18434871865373, -4.92539999824263,},
new double[] {1.40723958611488, -11.5545592998714, -5.14780194932221,},
new double[] {1.61909088492393, -12.5262932665595, -6.34366687651826,},
new double[] {-2.54745036363602, -8.64924529565274, 4.15127988308386,},
new double[] {0.815489888191223, -33.8531051237431, 4.3954106953589,},
new double[] {-2.2090271115303, -7.17818258102413, 8.9117419130814,},
new double[] {-1.9000232219696, -2.4331659041997, 6.91224717766923,},
},
new double[][]
{
new double[] {4.88746017217636, -4.36384651224969, -5.45526891285354,},
new double[] {1.07786506414413, -12.9399071692788, -5.88248026843442,},
new double[] {2.28888094425201, -15.4017823367163, -9.36490649113217,},
new double[] {-1.16468518972397, -35.4200913138333, 5.44735305966353,},
new double[] {-1.1483296751976, -13.5454911068913, 7.83577905727326,},
new double[] {-2.58188247680664, -1.10149600205281, 10.5928750605715,},
new double[] {-0.277529656887054, -6.96828661824016, 4.59381106840823,},
},
new double[][]
{
new double[] {3.39118540287018, -2.9173207268871, -5.66795398530988,},
new double[] {1.44856870174408, -9.21319243840922, -5.74986260778932,},
new double[] {1.45215392112732, -10.3989582187704, -7.06932768129103,},
new double[] {0.640938431024551, -15.319525165245, -7.68866476960221,},
new double[] {-0.77500119805336, -20.8335910793105, -1.56702420087282,},
new double[] {-3.48337143659592, -18.0461677940976, 12.3393172987974,},
new double[] {-1.17014795541763, -5.59624373275155, 6.09176828712909,},
},
new double[][]
{
new double[] {-3.984335064888, -6.2406475893692, -8.13815178201645,},
new double[] {-2.12110131978989, -5.60649378910647, -7.69551693188544,},
new double[] {-1.62762850522995, -24.1160212319193, -14.9683354815265,},
new double[] {-1.15231424570084, -17.1336790735458, -5.70731951079186,},
new double[] {0.00514835119247437, -35.4256585588532, 11.0357975880744,},
new double[] {0.247226655483246, -4.87705331087666, 8.47028869639136,},
new double[] {-1.28729045391083, -4.4684855254196, 4.45432778840328,},
},
new double[][]
{
new double[] {-5.14926165342331, -14.4168633009146, -14.4808205022332,},
new double[] {-3.93681302666664, -13.6040611430423, -9.52852874304709,},
new double[] {-4.0200162678957, -17.9772444010218, -10.9145425003168,},
new double[] {2.99205146729946, -11.3995995445577, 10.0112700536762,},
new double[] {-1.80960297584534, -25.9626088707583, 3.84153700324761,},
new double[] {-0.47445073723793, -3.15995343875038, 3.81288679772555,},
},
new double[][]
{
new double[] {-3.10730338096619, -4.90623566171983, -7.71155001801384,},
new double[] {-2.58265435695648, -12.8249488039327, -7.81701695282102,},
new double[] {-3.70455086231232, -10.9642675851383, -10.3474496036822,},
new double[] {2.34457105398178, -22.575668228196, -4.00681935468317,},
new double[] {-0.137023627758026, -22.8846781066673, 6.49448229892285,},
new double[] {-1.04487389326096, -10.8106353197974, 6.89123118904132,},
new double[] {-0.807777792215347, -6.72485967042486, 6.44026679233423,},
new double[] {-0.0864192843437195, -1.82784244477527, 5.21446167464657,},
},
new double[][]
{
new double[] {-3.68375554680824, -8.91158395500054, -9.35894038244743,},
new double[] {-3.42774018645287, -8.90966793048099, -12.0502934183779,},
new double[] {-2.21796408295631, -20.1283824753482, -9.3404551995806,},
new double[] {0.275979936122894, -24.8898254667703, -1.95441472953041,},
new double[] {2.8757631778717, -25.5929744730134, 15.9213204397452,},
new double[] {-0.0532664358615875, -5.41014381829368, 7.0702071664098,},
new double[] {-0.523447245359421, -2.21351362388411, 5.47910029515575,},
},
new double[][]
{
new double[] {-2.87790596485138, -4.67335526533981, -5.23215633615683,},
new double[] {-2.4156779050827, -3.99829080603495, -4.85576151355235,},
new double[] {-2.6987336575985, -7.76589206730162, -5.81054787011341,},
new double[] {-2.65482440590858, -10.5628263066491, -5.60468502395908,},
new double[] {-2.54620611667633, -13.0387387107748, -5.36223367466908,},
new double[] {-0.349991768598557, -6.54244110985515, -4.35843018634009,},
new double[] {1.43021196126938, -14.1423935327282, 11.3171592025544,},
new double[] {-0.248833745718002, -25.6880129237476, 3.6943247495434,},
new double[] {-0.191526114940643, -7.40986142342928, 5.01053017361167,},
new double[] {0.0262223184108734, -2.32355649224634, 5.02960958030255,},
},
new double[][]
{
new double[] {-0.491838902235031, -6.14010393559236, 0.827477332024586,},
new double[] {-0.806065648794174, -7.15029676810841, -1.19623376104369,},
new double[] {-0.376655906438828, -8.79062775480082, -1.90518908829517,},
new double[] {0.0747844576835632, -8.78933441325732, -1.96265207353993,},
new double[] {-0.375023484230042, 3.89681155173501, 9.01643231817069,},
new double[] {-2.8106614947319, -11.460008093918, 2.27801912994775,},
new double[] {8.87353122234344, -36.8569805718597, 6.36432395690119,},
new double[] {2.17160433530808, -6.57312981892095, 6.99683358454453,},
},
new double[][]
{
new double[] {-2.59969010949135, -3.67992698430228, 1.09594294144671,},
new double[] {-1.09673067927361, -5.84256216502719, -0.576662929456575,},
new double[] {-1.31642892956734, -7.75851355520771, -2.38379618379558,},
new double[] {-0.119869410991669, -8.5749576027529, -1.84393133510667,},
new double[] {1.6157403588295, -8.50491836461337, 1.75083250596366,},
new double[] {1.66225507855415, -26.4882911957686, 1.98153904369032,},
new double[] {2.55657434463501, -10.5098938623168, 11.632377227365,},
new double[] {1.91832333803177, -9.98753621777953, 7.38483383044985,},
new double[] {2.16058492660522, -2.7784029746222, 7.8378896386686,},
},
#endregion
};
var density = new MultivariateNormalDistribution(3);
var model = new HiddenMarkovModel<MultivariateNormalDistribution>(new Forward(5), density);
var learning = new BaumWelchLearning<MultivariateNormalDistribution>(model)
{
Tolerance = 0.0001,
Iterations = 0,
FittingOptions = new NormalOptions() { Regularization = 0.0001 }
};
double logLikelihood = learning.Run(observations);
Assert.IsFalse(Double.IsNaN(logLikelihood));
foreach (double value in model.Transitions)
Assert.IsFalse(Double.IsNaN(value));
foreach (double value in model.Probabilities)
Assert.IsFalse(Double.IsNaN(value));
}
public static HiddenMarkovClassifier<MultivariateNormalDistribution> CreateModel3(
int states = 4, bool priors = true)
{
MultivariateNormalDistribution density = new MultivariateNormalDistribution(2);
var classifier = new HiddenMarkovClassifier<MultivariateNormalDistribution>(6,
new Forward(states), density);
string[] labels = { "1", "2", "3", "4", "5", "6" };
for (int i = 0; i < classifier.Models.Length; i++)
classifier.Models[i].Tag = labels[i];
// Create the learning algorithm for the ensemble classifier
var teacher = new HiddenMarkovClassifierLearning<MultivariateNormalDistribution>(classifier,
// Train each model using the selected convergence criteria
i => new BaumWelchLearning<MultivariateNormalDistribution>(classifier.Models[i])
{
Tolerance = 0.1,
Iterations = 0,
FittingOptions = new NormalOptions() { Diagonal = true, Regularization = 1e-10 }
}
);
teacher.Empirical = priors;
// Run the learning algorithm
teacher.Run(inputTest, outputTest);
return classifier;
}
public void ProbabilityDensityFunctionTest2()
{
double[] mean = new double[64];
double[,] covariance = Accord.Tests.Math.CholeskyDecompositionTest.bigmatrix;
var target = new MultivariateNormalDistribution(mean, covariance);
double expected = 1.0;
double actual = target.ProbabilityDensityFunction(mean);
Assert.AreEqual(expected, actual, 0.00000001);
double[] x = Matrix.Diagonal(covariance).Multiply(1.5945e7);
expected = 4.781042576287362e-12;
actual = target.ProbabilityDensityFunction(x);
Assert.AreEqual(expected, actual, 1e-21);
}