/// <summary>
/// Learns a model that can map the given inputs to the desired outputs.
/// </summary>
/// <param name="x">The model inputs.</param>
/// <param name="weights">The weight of importance for each input sample.</param>
/// <returns>A model that has learned how to produce suitable outputs
/// given the input data <paramref name="x" />.</returns>
public GaussianClusterCollection Learn(double[][] x, double[] weights = null)
{
if (clusters.Model == null)
{
Initialize(x, weights);
}
// Create the mixture options
var mixtureOptions = new MixtureOptions()
{
Threshold = this.Tolerance,
InnerOptions = this.Options,
Iterations = this.Iterations,
Logarithm = this.UseLogarithm,
};
MultivariateMixture <MultivariateNormalDistribution> model = clusters.Model;
// Fit a multivariate Gaussian distribution
model.Fit(x, weights, mixtureOptions);
for (int i = 0; i < clusters.Model.Components.Length; i++)
{
clusters.Centroids[i] = new MixtureComponent <MultivariateNormalDistribution>(model, i);
}
if (ComputeLogLikelihood)
{
LogLikelihood = model.LogLikelihood(x);
}
return(clusters);
}
/// <summary>
/// Divides the input data into K clusters modeling each
/// cluster as a multivariate Gaussian distribution.
/// </summary>
///
public double Compute(double[][] data, GaussianMixtureModelOptions options)
{
if (data == null)
{
throw new ArgumentNullException("data");
}
if (model == null)
{
// TODO: Perform K-Means multiple times to avoid
// a poor Gaussian Mixture model initialization.
Initialize(data, options.Threshold);
}
// Create the mixture options
var mixtureOptions = new MixtureOptions()
{
Threshold = options.Threshold,
InnerOptions = options.NormalOptions,
Iterations = options.Iterations,
Logarithm = options.Logarithm
};
// Check if we have weighted samples
double[] weights = options.Weights;
// Fit a multivariate Gaussian distribution
model.Fit(data, weights, mixtureOptions);
// Return the log-likelihood as a measure of goodness-of-fit
return(model.LogLikelihood(data));
}
/// <summary>
/// Divides the input data into K clusters modeling each
/// cluster as a multivariate Gaussian distribution.
/// </summary>
///
public double Compute(double[][] data, GaussianMixtureModelOptions options)
{
if (data == null)
{
throw new ArgumentNullException("data");
}
int components = this.clusters.Count;
if (model == null)
{
// TODO: Perform K-Means multiple times to avoid
// a poor Gaussian Mixture model initialization.
double error = Initialize(data, options.Threshold);
}
// Fit a multivariate Gaussian distribution
var mixtureOptions = new MixtureOptions()
{
Threshold = options.Threshold,
InnerOptions = options.NormalOptions,
};
model.Fit(data, mixtureOptions);
// Return the log-likelihood as a measure of goodness-of-fit
return(model.LogLikelihood(data));
}
/// <summary>
/// Fits the underlying distribution to a given set of observations.
/// </summary>
///
/// <param name="observations">The array of observations to fit the model against. The array
/// elements can be either of type double (for univariate data) or
/// type double[] (for multivariate data).</param>
/// <param name="weights">The weight vector containing the weight for each of the samples.</param>
/// <param name="options">Optional arguments which may be used during fitting, such
/// as regularization constants and additional parameters.</param>
///
public override void Fit(double[] observations, double[] weights, IFittingOptions options)
{
MixtureOptions mixOptions = options as MixtureOptions;
if (options != null && mixOptions == null)
{
throw new ArgumentException("The specified options' type is invalid.", "options");
}
Fit(observations, weights, mixOptions);
}
/// <summary>
/// Fits the underlying distribution to a given set of observations.
/// </summary>
///
/// <param name="observations">The array of observations to fit the model against. The array
/// elements can be either of type double (for univariate data) or
/// type double[] (for multivariate data).</param>
/// <param name="weights">The weight vector containing the weight for each of the samples.</param>
/// <param name="options">Optional arguments which may be used during fitting, such
/// as regularization constants and additional parameters.</param>
///
public void Fit(double[] observations, double[] weights, MixtureOptions options)
{
var pdf = new IFittableDistribution <double> [coefficients.Length];
for (int i = 0; i < components.Length; i++)
{
pdf[i] = (IFittableDistribution <double>)components[i];
}
bool log = (options != null && options.Logarithm);
if (log)
{
if (weights != null)
{
throw new ArgumentException("The model fitting algorithm does not"
+ " currently support different weights when the logarithm option"
+ " is enabled. To avoid this exception, pass 'null' as the second"
+ " parameter's value when calling this method.");
}
var em = new LogExpectationMaximization <double>(coefficients, pdf);
if (options != null)
{
em.InnerOptions = options.InnerOptions;
em.Convergence.Iterations = options.Iterations;
em.Convergence.Tolerance = options.Threshold;
}
em.Compute(observations);
}
else
{
var em = new ExpectationMaximization <double>(coefficients, pdf);
if (options != null)
{
em.InnerOptions = options.InnerOptions;
em.Convergence.Iterations = options.Iterations;
em.Convergence.Tolerance = options.Threshold;
}
em.Compute(observations, weights);
}
for (int i = 0; i < components.Length; i++)
{
cache[i] = components[i] = (T)pdf[i];
}
this.initialize();
}
/// <summary>
/// Estimates a new mixture model from a given set of observations.
/// </summary>
///
/// <param name="data">A set of observations.</param>
/// <param name="components">The initial components of the mixture model.</param>
/// <param name="coefficients">The initial mixture coefficients.</param>
/// <param name="threshold">The convergence threshold for the Expectation-Maximization estimation.</param>
/// <returns>Returns a new Mixture fitted to the given observations.</returns>
///
public static MultivariateMixture <T> Estimate(double[][] data, double threshold, double[] coefficients, params T[] components)
{
IFittingOptions options = new MixtureOptions()
{
Threshold = threshold
};
var mixture = new MultivariateMixture <T>(coefficients, components);
mixture.Fit(data, options);
return(mixture);
}
/// <summary>
/// Creates a Baum-Welch with default configurations for
/// hidden Markov models with normal mixture densities.
/// </summary>
///
public static BaumWelchLearning <MultivariateMixture <MultivariateNormalDistribution> > FromMixtureModel(
HiddenMarkovModel <MultivariateMixture <MultivariateNormalDistribution> > model, NormalOptions options)
{
MixtureOptions mixOptions = new MixtureOptions()
{
Iterations = 1,
InnerOptions = options
};
return(new BaumWelchLearning <MultivariateMixture <MultivariateNormalDistribution> >(model)
{
FittingOptions = mixOptions
});
}
/// <summary>
/// Fits the underlying distribution to a given set of observations.
/// </summary>
///
/// <param name="observations">The array of observations to fit the model against. The array
/// elements can be either of type double (for univariate data) or
/// type double[] (for multivariate data).</param>
/// <param name="weights">The weight vector containing the weight for each of the samples.</param>
/// <param name="options">Optional arguments which may be used during fitting, such
/// as regularization constants and additional parameters.</param>
///
public override void Fit(double[] observations, double[] weights, IFittingOptions options)
{
MixtureOptions mixOptions = options as MixtureOptions;
if (options != null && mixOptions == null)
{
mixOptions = new MixtureOptions()
{
InnerOptions = options
};
}
Fit(observations, weights, mixOptions);
}
/// <summary>
/// Creates a Baum-Welch with default configurations for
/// hidden Markov models with normal mixture densities.
/// </summary>
///
public static BaumWelchLearning <Mixture <NormalDistribution> > FromMixtureModel(
HiddenMarkovModel <Mixture <NormalDistribution> > model, NormalOptions options)
{
MixtureOptions mixOptions = new MixtureOptions()
{
Iterations = 1,
InnerOptions = options,
//ParallelOptions = ParallelOptions, // TODO:
};
return(new BaumWelchLearning <Mixture <NormalDistribution> >(model)
{
FittingOptions = mixOptions
});
}
/// <summary>
/// Divides the input data into K clusters modeling each
/// cluster as a multivariate Gaussian distribution.
/// </summary>
///
public double Compute(double[][] data, GaussianMixtureModelOptions options)
{
if (data == null)
{
throw new ArgumentNullException("data");
}
int components = this.clusters.Count;
if (model == null)
{
// TODO: Perform K-Means multiple times to avoid
// a poor Gaussian Mixture model initialization.
double error = Initialize(data, options.Threshold);
}
// Create the mixture options
var mixtureOptions = new MixtureOptions()
{
Threshold = options.Threshold,
InnerOptions = options.NormalOptions,
};
// Check if we have weighted samples
double[] weights = options.Weights;
if (weights != null)
{
// Normalize if necessary
double sum = weights.Sum();
if (sum != 1)
{
weights.Divide(sum, inPlace: true);
}
System.Diagnostics.Debug.Assert(weights.Sum() - 1 < 1e-5);
}
// Fit a multivariate Gaussian distribution
model.Fit(data, weights, mixtureOptions);
// Return the log-likelihood as a measure of goodness-of-fit
return(model.LogLikelihood(data));
}
public void FitTest()
{
double[] coefficients = { 0.50, 0.50 };
NormalDistribution[] components = new NormalDistribution[2];
components[0] = new NormalDistribution(2, 1);
components[1] = new NormalDistribution(5, 1);
var target = new Mixture <NormalDistribution>(coefficients, components);
double[] values = { 0, 1, 1, 0, 1, 6, 6, 5, 7, 5 };
double[] part1 = values.Submatrix(0, 4);
double[] part2 = values.Submatrix(5, 9);
MixtureOptions options = new MixtureOptions()
{
Threshold = 1e-10
};
target.Fit(values, options);
var actual = target;
var mean1 = Accord.Statistics.Tools.Mean(part1);
var var1 = Accord.Statistics.Tools.Variance(part1);
Assert.AreEqual(mean1, actual.Components[0].Mean, 1e-6);
Assert.AreEqual(var1, actual.Components[0].Variance, 1e-6);
var mean2 = Accord.Statistics.Tools.Mean(part2);
var var2 = Accord.Statistics.Tools.Variance(part2);
Assert.AreEqual(mean2, actual.Components[1].Mean, 1e-6);
Assert.AreEqual(var2, actual.Components[1].Variance, 1e-5);
var expectedMean = Accord.Statistics.Tools.Mean(values);
var actualMean = actual.Mean;
Assert.AreEqual(expectedMean, actualMean, 1e-7);
var expectedVar = Accord.Statistics.Tools.Variance(values, false);
var actualVar = actual.Variance;
Assert.AreEqual(expectedVar, actualVar, 0.15);
}
/// <summary>
/// Learns a model that can map the given inputs to the desired outputs.
/// </summary>
/// <param name="x">The model inputs.</param>
/// <param name="weights">The weight of importance for each input sample.</param>
/// <returns>A model that has learned how to produce suitable outputs
/// given the input data <paramref name="x" />.</returns>
public GaussianClusterCollection Learn(double[][] x, double[] weights = null)
{
if (clusters.Model == null)
{
Initialize(x, weights);
}
// Create the mixture options
var mixtureOptions = new MixtureOptions()
{
Threshold = this.Tolerance,
InnerOptions = this.Options,
MaxIterations = this.MaxIterations,
Logarithm = this.UseLogarithm,
ParallelOptions = ParallelOptions,
};
MultivariateMixture <MultivariateNormalDistribution> model = clusters.Model;
// Fit a multivariate Gaussian distribution
model.Fit(x, weights, mixtureOptions);
#pragma warning disable 612, 618
this.Iterations = mixtureOptions.Iterations;
#pragma warning restore 612, 618
for (int i = 0; i < clusters.Model.Components.Length; i++)
{
clusters.Centroids[i] = new MixtureComponent <MultivariateNormalDistribution>(model, i);
}
if (ComputeLogLikelihood)
{
LogLikelihood = model.LogLikelihood(x);
}
Accord.Diagnostics.Debug.Assert(clusters.NumberOfClasses == clusters.Model.Components.Length);
Accord.Diagnostics.Debug.Assert(clusters.NumberOfOutputs == clusters.Model.Components.Length);
Accord.Diagnostics.Debug.Assert(clusters.NumberOfInputs == x[0].Length);
return(clusters);
}
public void FitTest2()
{
double[] coefficients = { 0.50, 0.50 };
NormalDistribution[] components = new NormalDistribution[2];
components[0] = new NormalDistribution(2, 1);
components[1] = new NormalDistribution(5, 1);
var target = new Mixture <NormalDistribution>(coefficients, components);
double[] values = { 12512, 1, 1, 0, 1, 6, 6, 5, 7, 5 };
double[] weights = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 1 };
weights = weights.Divide(weights.Sum());
double[] part1 = values.Submatrix(1, 4);
double[] part2 = values.Submatrix(5, 9);
MixtureOptions opt = new MixtureOptions()
{
Threshold = 0.000001
};
target.Fit(values, weights, opt);
var mean1 = Accord.Statistics.Tools.Mean(part1);
var var1 = Accord.Statistics.Tools.Variance(part1);
Assert.AreEqual(mean1, target.Components[0].Mean, 1e-5);
Assert.AreEqual(var1, target.Components[0].Variance, 1e-5);
var mean2 = Accord.Statistics.Tools.Mean(part2);
var var2 = Accord.Statistics.Tools.Variance(part2);
Assert.AreEqual(mean2, target.Components[1].Mean, 1e-5);
Assert.AreEqual(var2, target.Components[1].Variance, 1e-5);
var expectedMean = Accord.Statistics.Tools.WeightedMean(values, weights);
var actualMean = target.Mean;
Assert.AreEqual(expectedMean, actualMean, 1e-5);
}
/// <summary>
/// Fits the underlying distribution to a given set of observations.
/// </summary>
/// <param name="observations">The array of observations to fit the model against. The array
/// elements can be either of type double (for univariate data) or
/// type double[] (for multivariate data).</param>
/// <param name="weights">The weight vector containing the weight for each of the samples.</param>
/// <param name="options">Optional arguments which may be used during fitting, such
/// as regularization constants and additional parameters.</param>
/// <remarks>
/// Although both double[] and double[][] arrays are supported,
/// providing a double[] for a multivariate distribution or a
/// double[][] for a univariate distribution may have a negative
/// impact in performance.
/// </remarks>
public override void Fit(double[] observations, double[] weights, IFittingOptions options)
{
// Estimation parameters
double threshold = 1e-3;
IFittingOptions innerOptions = null;
if (options != null)
{
// Process optional arguments
MixtureOptions o = (MixtureOptions)options;
threshold = o.Threshold;
innerOptions = o.InnerOptions;
}
// 1. Initialize means, covariances and mixing coefficients
// and evaluate the initial value of the log-likelihood
int N = observations.Length;
int K = components.Length;
weights = weights.Multiply(N);
var gamma = new double[K][];
for (int k = 0; k < K; k++)
{
gamma[k] = new double[N];
}
var pi = (double[])coefficients.Clone();
var pdf = (T[])components.Clone();
double likelihood = logLikelihood(pi, pdf, observations, weights);
bool converged = false;
while (!converged)
{
// 2. Expectation: Evaluate the responsibilities using the
// current parameter values.
for (int n = 0; n < N; n++)
{
double den = 0.0;
double w = weights[n];
var x = observations[n];
for (int k = 0; k < K; k++)
{
den += gamma[k][n] = pi[k] * pdf[k].ProbabilityFunction(x) * w;
}
if (den != 0)
{
for (int k = 0; k < K; k++)
{
gamma[k][n] /= den;
}
}
}
// 3. Maximization: Re-estimate the parameters using the
// current responsibilities
for (int k = 0; k < K; k++)
{
double Nk = gamma[k].Sum();
double[] w = gamma[k].Divide(Nk);
pi[k] = Nk / N;
pdf[k].Fit(observations, w, innerOptions);
}
// 4. Evaluate the log-likelihood and check for convergence
double newLikelihood = logLikelihood(pi, pdf, observations, weights);
if (Double.IsNaN(newLikelihood) || Double.IsInfinity(newLikelihood))
{
throw new ConvergenceException("Fitting did not converge.");
}
if (System.Math.Abs(likelihood - newLikelihood) < threshold)
{
converged = true;
}
likelihood = newLikelihood;
}
// Become the newly fitted distribution.
this.coefficients = pi;
this.components = pdf;
}
/// <summary>
/// Fits the underlying distribution to a given set of observations.
/// </summary>
///
/// <param name="observations">The array of observations to fit the model against. The array
/// elements can be either of type double (for univariate data) or
/// type double[] (for multivariate data).</param>
/// <param name="weights">The weight vector containing the weight for each of the samples.</param>
/// <param name="options">Optional arguments which may be used during fitting, such
/// as regularization constants and additional parameters.</param>
///
/// <remarks>
/// Although both double[] and double[][] arrays are supported,
/// providing a double[] for a multivariate distribution or a
/// double[][] for a univariate distribution may have a negative
/// impact in performance.
/// </remarks>
///
public override void Fit(double[][] observations, double[] weights, IFittingOptions options)
{
// Estimation parameters
double threshold = 1e-3;
IFittingOptions innerOptions = null;
if (options != null)
{
// Process optional arguments
MixtureOptions o = (MixtureOptions)options;
threshold = o.Threshold;
innerOptions = o.InnerOptions;
}
// 1. Initialize means, covariances and mixing coefficients
// and evaluate the initial value of the log-likelihood
int N = observations.Length;
int K = components.Length;
weights = weights.Multiply(N);
double[][] gamma = new double[K][];
for (int k = 0; k < gamma.Length; k++)
{
gamma[k] = new double[N];
}
double[] pi = (double[])coefficients.Clone();
T[] pdf = new T[components.Length];
for (int i = 0; i < components.Length; i++)
{
pdf[i] = (T)components[i].Clone();
}
double likelihood = logLikelihood(pi, pdf, observations, weights);
bool converged = false;
int numTries = 0;
while (!converged && numTries < 500)
{
// 2. Expectation: Evaluate the responsibilities using the
// current parameter values.
for (int i = 0; i < observations.Length; i++)
{
double den = 0.0;
double w = weights[i];
double[] x = observations[i];
for (int k = 0; k < gamma.Length; k++)
{
den += gamma[k][i] = pi[k] * pdf[k].ProbabilityFunction(x) * w;
}
if (den != 0)
{
for (int k = 0; k < gamma.Length; k++)
{
gamma[k][i] /= den;
}
}
}
// 3. Maximization: Re-estimate the parameters using the
// current responsibilities
for (int k = 0; k < gamma.Length; k++)
{
double Nk = gamma[k].Sum();
double[] w;
if (Nk == 1.0 || Nk == 0) // Two cases will result in cov = NaN
{
pi[k] = 0;
}
else
{
w = gamma[k].Divide(Nk);
pi[k] = Nk / N;
pdf[k].Fit(observations, w, innerOptions);
}
}
// 4. Evaluate the log-likelihood and check for convergence
double newLikelihood = logLikelihood(pi, pdf, observations, weights);
if (Double.IsNaN(newLikelihood) || Double.IsInfinity(newLikelihood))
{
throw new ConvergenceException("Fitting did not converge.");
}
if (Math.Abs(likelihood - newLikelihood) < threshold * Math.Abs(likelihood))
{
converged = true;
}
if (newLikelihood == 0)
{
converged = true;
}
likelihood = newLikelihood;
numTries++;
}
// Become the newly fitted distribution.
this.initialize(pi, pdf);
}
/// <summary>
/// Fits the underlying distribution to a given set of observations.
/// </summary>
///
/// <param name="observations">The array of observations to fit the model against. The array
/// elements can be either of type double (for univariate data) or
/// type double[] (for multivariate data).</param>
/// <param name="weights">The weight vector containing the weight for each of the samples.</param>
/// <param name="options">Optional arguments which may be used during fitting, such
/// as regularization constants and additional parameters.</param>
///
public void Fit(double[] observations, double[] weights, MixtureOptions options)
{
// Estimation parameters
int maxIterations = 0;
double threshold = 1e-3;
IFittingOptions innerOptions = null;
#if DEBUG
if (weights != null)
{
for (int i = 0; i < weights.Length; i++)
{
if (Double.IsNaN(weights[i]) || Double.IsInfinity(weights[i]))
{
throw new ArgumentException("Invalid numbers in the weight vector.", "weights");
}
}
}
#endif
if (options != null)
{
// Process optional arguments
threshold = options.Threshold;
innerOptions = options.InnerOptions;
maxIterations = options.Iterations;
}
// 1. Initialize means, covariances and mixing coefficients
// and evaluate the initial value of the log-likelihood
int N = observations.Length;
int K = components.Length;
double weightSum;
if (weights == null)
{
weights = new double[observations.Length];
for (int i = 0; i < weights.Length; i++)
{
weights[i] = 1.0 / weights.Length;
}
weightSum = 1.0;
}
else
{
weightSum = weights.Sum();
}
// Initialize responsibilities
double[] norms = new double[N];
double[][] gamma = new double[K][];
for (int k = 0; k < gamma.Length; k++)
{
gamma[k] = new double[N];
}
// Clone the current distribution values
double[] pi = (double[])coefficients.Clone();
T[] pdf = new T[components.Length];
for (int i = 0; i < components.Length; i++)
{
pdf[i] = (T)components[i].Clone();
}
// Prepare the iteration
double likelihood = logLikelihood(pi, pdf, observations, weights);
bool converged = false;
int iteration = 0;
// Start
while (!converged)
{
iteration++;
// 2. Expectation: Evaluate the component distributions
// responsibilities using the current parameter values.
Array.Clear(norms, 0, norms.Length);
for (int k = 0; k < gamma.Length; k++)
{
for (int i = 0; i < observations.Length; i++)
{
norms[i] += gamma[k][i] = pi[k] * pdf[k].ProbabilityFunction(observations[i]);
}
}
for (int k = 0; k < gamma.Length; k++)
{
for (int i = 0; i < weights.Length; i++)
{
if (norms[i] != 0)
{
gamma[k][i] *= weights[i] / norms[i];
}
}
}
// 3. Maximization: Re-estimate the distribution parameters
// using the previously computed responsibilities
for (int k = 0; k < gamma.Length; k++)
{
double sum = gamma[k].Sum();
if (sum == 0)
{
pi[k] = 0.0;
continue;
}
System.Diagnostics.Debug.Assert(sum != 0);
System.Diagnostics.Debug.Assert(weightSum != 0);
System.Diagnostics.Debug.Assert(!gamma[k].HasNaN());
for (int i = 0; i < gamma[k].Length; i++)
{
gamma[k][i] /= sum;
}
pi[k] = sum / weightSum;
pdf[k].Fit(observations, gamma[k], innerOptions);
}
// 4. Evaluate the log-likelihood and check for convergence
double newLikelihood = logLikelihood(pi, pdf, observations, weights);
if (Double.IsNaN(newLikelihood) || Double.IsInfinity(newLikelihood))
{
throw new ConvergenceException("Fitting did not converge.");
}
if ((maxIterations > 0 && iteration >= maxIterations) ||
Math.Abs(likelihood - newLikelihood) < threshold * Math.Abs(likelihood))
{
converged = true;
}
likelihood = newLikelihood;
}
// Become the newly fitted distribution.
this.initialize(pi, pdf);
}