/// <summary>
/// Constructs a new Hidden Markov Model.
/// </summary>
///
/// <param name="topology">
/// A <see cref="Topology"/> object specifying the initial values of the matrix of transition
/// probabilities <c>A</c> and initial state probabilities <c>pi</c> to be used by this model.
/// </param>
/// <param name="symbols">The number of output symbols used for this model.</param>
/// <param name="random">Whether to initialize emissions with random probabilities
/// or uniformly with <c>1 / number of symbols</c>. Default is false (default is
/// to use <c>1/symbols</c>).</param>
///
public HiddenMarkovModel(ITopology topology, int symbols, bool random)
: base(topology)
{
this.symbols = symbols;
double[][] b;
if (random)
{
b = Jagged.Random(States, symbols);
}
else
{
b = Jagged.Ones(States, symbols);
}
base.Emissions = GeneralDiscreteDistribution.FromMatrix(b.Log(), true);
}
internal static void run(double[][] X, double[][] Y, double perplexity, double theta, bool skip_random_init = false)
{
int N = X.Rows();
int D = X.Columns();
int no_dims = Y.Columns();
// Determine whether we are using an exact algorithm
if (N - 1 < 3 * perplexity)
{
throw new Exception(String.Format("Perplexity too large for the number of data points. For {0} points, should be less than {1}", N, (N - 1) / 3.0));
}
Debug.Write(String.Format("Using no_dims = {0}, perplexity = {1}, and theta = {2}", no_dims, perplexity, theta));
bool exact = (theta == 0.0);
// Set learning parameters
TimeSpan total_time = TimeSpan.Zero;
Stopwatch start;
TimeSpan end;
int max_iter = 1000;
int stop_lying_iter = 250;
int mom_switch_iter = 250;
double momentum = 0.5;
double final_momentum = 0.8;
double eta = 200.0;
// Allocate some memory
double[][] dY = Jagged.Create <double>(N, no_dims);
double[][] uY = Jagged.Create <double>(N, no_dims);
double[][] gains = Jagged.Ones <double>(N, no_dims);
// Normalize input data (to prevent numerical problems)
Debug.Write("Computing input similarities...");
start = Stopwatch.StartNew();
Accord.Statistics.Tools.Center(X, inPlace: true);
X.Divide(X.Max(), result: X);
// Compute input similarities for exact t-SNE
double[][] P = null;
int[] row_P = null;
int[] col_P = null;
double[] val_P = null;
if (exact)
{
Trace.Write("Exact?");
// Compute similarities
P = Jagged.Create <double>(N, N);
computeGaussianPerplexity(X, N, D, ref P, perplexity);
// Symmetrize input similarities
Debug.Write("Symmetrizing...");
for (int n = 0; n < N; n++)
{
for (int m = n + 1; m < N; m++)
{
P[n][m] += P[m][n];
P[m][n] = P[n][m];
}
}
P.Divide(P.Sum(), result: P);
}
// Compute input similarities for approximate t-SNE
else
{
// Compute asymmetric pairwise input similarities
computeGaussianPerplexity(X, N, D, ref row_P, ref col_P, ref val_P, perplexity, (int)(3 * perplexity));
// Symmetrize input similarities
symmetrizeMatrix(ref row_P, ref col_P, ref val_P, N);
double sum_P = 0.0;
for (int i = 0; i < row_P[N]; i++)
{
sum_P += val_P[i];
}
for (int i = 0; i < row_P[N]; i++)
{
val_P[i] /= sum_P;
}
}
end = start.Elapsed;
// Lie about the P-values
if (exact)
{
P.Multiply(12.0, result: P);
}
else
{
for (int i = 0; i < row_P[N]; i++)
{
val_P[i] *= 12.0;
}
}
if (!skip_random_init)
{
// Initialize solution (randomly)
for (int i = 0; i < Y.Length; i++)
{
for (int j = 0; j < Y[i].Length; j++)
{
Y[i][j] = randn() * 0.0001;
}
}
}
// Perform main training loop
if (exact)
{
Debug.Write(String.Format("Input similarities computed in {0} seconds!", end));
Debug.Write("Learning embedding...");
}
else
{
Debug.Write(String.Format("Input similarities computed in {0} seconds (sparsity = {1})!", end, (double)row_P[N] / ((double)N * (double)N)));
Debug.Write("Learning embedding...");
}
start = Stopwatch.StartNew();
for (int iter = 0; iter < max_iter; iter++)
{
// Compute (approximate) gradient
if (exact)
{
computeExactGradient(P, Y, N, no_dims, dY);
}
else
{
computeGradient(P, row_P, col_P, val_P, Y, N, no_dims, dY, theta);
}
// Update gains
for (int i = 0; i < gains.Length; i++)
{
for (int j = 0; j < gains[i].Length; j++)
{
gains[i][j] = (System.Math.Sign(dY[i][j]) != System.Math.Sign(uY[i][j])) ? (gains[i][j] + 0.2) : (gains[i][j] * 0.8);
}
}
for (int i = 0; i < gains.Length; i++)
{
for (int j = 0; j < gains[i].Length; j++)
{
if (gains[i][j] < 0.01)
{
gains[i][j] = 0.01;
}
}
}
// Perform gradient update (with momentum and gains)
for (int i = 0; i < uY.Length; i++)
{
for (int j = 0; j < uY[i].Length; j++)
{
uY[i][j] = momentum * uY[i][j] - eta * gains[i][j] * dY[i][j];
}
}
for (int i = 0; i < Y.Length; i++)
{
for (int j = 0; j < Y[i].Length; j++)
{
Y[i][j] = Y[i][j] + uY[i][j];
}
}
// Make solution zero-mean
Accord.Statistics.Tools.Center(Y, inPlace: true);
// Stop lying about the P-values after a while, and switch momentum
if (iter == stop_lying_iter)
{
if (exact)
{
P.Divide(12.0, result: P);
}
else
{
for (int i = 0; i < row_P[N]; i++)
{
val_P[i] /= 12.0;
}
}
}
if (iter == mom_switch_iter)
{
momentum = final_momentum;
}
// Print out progress
if (iter > 0 && (iter % 50 == 0 || iter == max_iter - 1))
{
end = start.Elapsed;
double C = 0.0;
if (exact)
{
C = evaluateError(P, Y, N, no_dims);
}
else
{
C = evaluateError(row_P, col_P, val_P, Y, N, no_dims, theta); // doing approximate computation here!
}
if (iter == 0)
{
Debug.WriteLine(String.Format("Iteration {0}: error is {1}", iter + 1, C));
}
else
{
total_time += end;
Debug.WriteLine(String.Format("Iteration {0}: error is {1} (50 iterations in {2} seconds)", iter, C, end));
}
start = Stopwatch.StartNew();
}
}
end = start.Elapsed;
total_time += end;
Debug.WriteLine(String.Format("Fitting performed in {0} seconds.", total_time));
}
