public void ForwardBackwardGenericTest()
{
var discreteModel = CreateModel1();
var genericModel = CreateModel4();
int[] discreteObservations = { 2, 2, 1, 0 };
double[][] genericObservations =
{
new double[] { 2 }, new double[] { 2 },
new double[] { 1 }, new double[] { 0 }
};
double[] scaling = new double[3];
double discreteFwdLogLikelihood;
double[,] discreteFwd = ForwardBackwardAlgorithm.Forward(discreteModel,
discreteObservations, out scaling, out discreteFwdLogLikelihood);
double[,] discreteBwd = ForwardBackwardAlgorithm.Backward(discreteModel,
discreteObservations, scaling);
double genericFwdLogLikelihood;
double[,] genericFwd = ForwardBackwardAlgorithm.Forward(genericModel,
genericObservations, out scaling, out genericFwdLogLikelihood);
double[,] genericBwd = ForwardBackwardAlgorithm.Backward(genericModel,
genericObservations, scaling);
Assert.AreEqual(discreteFwdLogLikelihood, genericFwdLogLikelihood);
for (int i = 0; i < discreteFwd.GetLength(0); i++)
{
for (int j = 0; j < discreteFwd.GetLength(1); j++)
{
Assert.AreEqual(discreteFwd[i, j], genericFwd[i, j]);
Assert.AreEqual(discreteBwd[i, j], genericBwd[i, j]);
}
}
}
public void ForwardScalingTest()
{
HiddenMarkovModel hmm = CreateModel1();
var P = Matrix.Exp(hmm.Probabilities);
var A = Matrix.Exp(hmm.Transitions);
var B = Matrix.Exp(hmm.Emissions);
// G G C A
int[] observations = { 2, 2, 1, 0 };
double[] scaling;
double logLikelihood;
double[,] actual = ForwardBackwardAlgorithm.Forward(hmm, observations, out scaling, out logLikelihood);
double a00 = P[0] * B[0, 2];
double a01 = P[1] * B[1, 2];
double t0 = a00 + a01;
a00 /= t0;
a01 /= t0;
double a10 = (a00 * A[0, 0] + a01 * A[1, 0]) * B[0, 2];
double a11 = (a01 * A[1, 1] + a00 * A[0, 1]) * B[1, 2];
double t1 = a10 + a11;
a10 /= t1;
a11 /= t1;
double a20 = (a10 * A[0, 0] + a11 * A[1, 0]) * B[0, 1];
double a21 = (a11 * A[1, 1] + a10 * A[0, 1]) * B[1, 1];
double t2 = a20 + a21;
a20 /= t2;
a21 /= t2;
double a30 = (a20 * A[0, 0] + a21 * A[1, 0]) * B[0, 0];
double a31 = (a21 * A[1, 1] + a20 * A[0, 1]) * B[1, 0];
double t3 = a30 + a31;
a30 /= t3;
a31 /= t3;
Assert.AreEqual(a00, actual[0, 0]);
Assert.AreEqual(a01, actual[0, 1]);
Assert.AreEqual(a10, actual[1, 0]);
Assert.AreEqual(a11, actual[1, 1]);
Assert.AreEqual(a20, actual[2, 0]);
Assert.AreEqual(a21, actual[2, 1]);
Assert.AreEqual(a30, actual[3, 0]);
Assert.AreEqual(a31, actual[3, 1]);
double p = System.Math.Exp(logLikelihood);
Assert.AreEqual(0.00384315, p, 1e-8);
Assert.IsFalse(double.IsNaN(p));
}
/// <summary>
/// Computes the forward and backward probabilities matrices
/// for a given observation referenced by its index in the
/// input training data.
/// </summary>
/// <param name="index">The index of the observation in the input training data.</param>
/// <param name="fwd">Returns the computed forward probabilities matrix.</param>
/// <param name="bwd">Returns the computed backward probabilities matrix.</param>
/// <param name="scaling">Returns the scaling parameters used during calculations.</param>
protected override void ComputeForwardBackward(int index, out double[,] fwd, out double[,] bwd,
out double[] scaling)
{
fwd = ForwardBackwardAlgorithm.Forward(model, continuousObservations[index], out scaling);
bwd = ForwardBackwardAlgorithm.Backward(model, continuousObservations[index], scaling);
}
private double[] gradient(T[][] observations, int[][] labels)
{
int N = observations.Length;
var function = model.Function;
var states = model.States;
double[] g = new double[function.Weights.Length];
// Compute sequence probabilities
var P = new double[N][, ][];
for (int i = 0; i < N; i++)
{
var Pi = P[i] = new double[states + 1, states][];
T[] x = observations[i];
var fwd = ForwardBackwardAlgorithm.Forward(function.Factors[0], x, 0);
var bwd = ForwardBackwardAlgorithm.Backward(function.Factors[0], x, 0);
double z = partition(fwd, x);
for (int prev = -1; prev < states; prev++)
{
for (int next = 0; next < states; next++)
{
double[] Pis = new double[x.Length];
for (int t = 0; t < x.Length; t++)
{
Pis[t] = p(prev, next, x, t, fwd, bwd, function) / z;
}
Pi[prev + 1, next] = Pis;
}
}
}
// Compute the gradient w.r.t. each feature
// function in the model's potential function.
for (int k = 0; k < g.Length; k++)
{
var feature = function.Features[k];
double sum1 = 0.0, sum2 = 0.0;
for (int i = 0; i < N; i++)
{
T[] x = observations[i];
int[] y = labels[i];
var Pi = P[i];
// Compute first term of the partial derivative
sum1 += feature.Compute(-1, y[0], x, 0);
for (int t = 1; t < x.Length; t++)
{
sum1 += feature.Compute(y[t - 1], y[t], x, t);
}
// Compute second term of the partial derivative
for (int prev = -1; prev < states; prev++)
{
for (int next = 0; next < states; next++)
{
double[] Pis = Pi[prev + 1, next];
for (int t = 0; t < Pis.Length; t++)
{
sum2 += feature.Compute(prev, next, x, t) * Pis[t];
}
}
}
}
g[k] = -(sum1 - sum2);
}
return(g);
}
private double[] gradient(T[][] observations, int[][] labels, double[] g)
{
var model = Model;
var function = model.Function;
int states = model.States;
int n = observations.Length;
int d = Model.Function.Weights.Length;
int Tmax = observations.Max(x => x.Length);
int progress = 0;
g.Clear();
// Compute sequence probabilities
Parallel.For(0, observations.Length, ParallelOptions,
() =>
{
// Create thread-local storage
var work = new double[states + 1, states][];
for (int j = 0; j < states + 1; j++)
{
for (int k = 0; k < states; k++)
{
work[j, k] = new double[Tmax];
}
}
return(new
{
bwd = new double[Tmax, states],
fwd = new double[Tmax, states],
sum1 = new double[d],
sum2 = new double[d],
work = work,
count = new int[] { 0 }
});
},
(i, state, local) =>
{
T[] x = observations[i];
var fwd = local.fwd;
var bwd = local.bwd;
var sum1 = local.sum1;
var sum2 = local.sum2;
var work = local.work;
ForwardBackwardAlgorithm.Forward(function.Factors[0], x, fwd);
ForwardBackwardAlgorithm.Backward(function.Factors[0], x, bwd);
double z = partition(fwd, x);
for (int prev = -1; prev < states; prev++)
{
for (int next = 0; next < states; next++)
{
double[] Pis = work[prev + 1, next];
for (int t = 0; t < x.Length; t++)
{
Pis[t] = p(prev, next, x, t, fwd, bwd, function) / z;
}
}
}
// Compute the gradient w.r.t. each feature
// function in the model's potential function.
int[] y = labels[i];
Parallel.For(0, g.Length, ParallelOptions, k =>
{
IFeature <T> feature = function.Features[k];
// Compute first term of the partial derivative
sum1[k] += feature.Compute(-1, y[0], x, 0);
for (int t = 1; t < x.Length; t++)
{
sum1[k] += feature.Compute(y[t - 1], y[t], x, t);
}
// Compute second term of the partial derivative
for (int prev = -1; prev < states; prev++)
{
for (int next = 0; next < states; next++)
{
double[] Pis = work[prev + 1, next];
for (int t = 0; t < Pis.Length; t++)
{
sum2[k] += feature.Compute(prev, next, x, t) * Pis[t];
}
}
}
});
local.count[0]++;
return(local);
},
(local) =>
{
lock (g)
{
for (int k = 0; k < g.Length; k++)
{
g[k] -= (local.sum1[k] - local.sum2[k]);
}
progress += local.count[0];
}
}
);
return(g);
}