// Performs whitening training for each column separately. Notice that for both PCA and ZCA, _models and _invModels
// will have dimension input_vec_size x input_vec_size. In the getter, the matrix will be truncated to only keep
// PcaNum columns, and thus produce the desired output size.
private static void TrainModels(IHostEnvironment env, IChannel ch, float[][] columnData, int[] rowCounts,
ref float[][] models, ref float[][] invModels, ColumnType[] srcTypes, params ColumnInfo[] columns)
{
ch.Assert(columnData.Length == rowCounts.Length);
for (int iinfo = 0; iinfo < columns.Length; iinfo++)
{
var ex = columns[iinfo];
var data = columnData[iinfo];
int crow = rowCounts[iinfo];
int ccol = srcTypes[iinfo].ValueCount;
// If there is no training data, simply initialize the model matrices to identity matrices.
if (crow == 0)
{
var matrixSize = ccol * ccol;
models[iinfo] = new float[matrixSize];
invModels[iinfo] = new float[matrixSize];
for (int i = 0; i < ccol; i++)
{
models[iinfo][i * ccol + i] = 1;
invModels[iinfo][i * ccol + i] = 1;
}
continue;
}
// Compute covariance matrix.
var u = new float[ccol * ccol];
ch.Info("Computing covariance matrix...");
Mkl.Gemm(Layout, Mkl.Transpose.Trans, Mkl.Transpose.NoTrans,
ccol, ccol, crow, 1 / (float)crow, data, ccol, data, ccol, 0, u, ccol);
ch.Info("Computing SVD...");
var eigValues = new float[ccol]; // Eigenvalues.
var unconv = new float[ccol]; // Superdiagonal unconverged values (if any). Not used but seems to be required by MKL.
// After the next call, values in U will be ovewritten by left eigenvectors.
// Each column in U will be an eigenvector.
int r = Mkl.Svd(Layout, Mkl.SvdJob.MinOvr, Mkl.SvdJob.None,
ccol, ccol, u, ccol, eigValues, null, ccol, null, ccol, unconv);
ch.Assert(r == 0);
if (r > 0)
{
throw ch.Except("SVD did not converge.");
}
if (r < 0)
{
throw ch.Except("Invalid arguments to LAPACK gesvd, error: {0}", r);
}
ch.Info("Scaling eigenvectors...");
// Scale eigenvalues first so we don't have to compute sqrt for every matrix element.
// Scaled eigenvalues are used to compute inverse transformation matrix
// while reciprocal (eigValuesRcp) values are used to compute whitening matrix.
for (int i = 0; i < eigValues.Length; i++)
{
eigValues[i] = MathUtils.Sqrt(Math.Max(0, eigValues[i]) + ex.Epsilon);
}
var eigValuesRcp = new float[eigValues.Length];
for (int i = 0; i < eigValuesRcp.Length; i++)
{
eigValuesRcp[i] = 1 / eigValues[i];
}
// Scale eigenvectors. Note that resulting matrix is transposed, so the scaled
// eigenvectors are stored row-wise.
var uScaled = new float[u.Length];
var uInvScaled = new float[u.Length];
int isrc = 0;
for (int irowSrc = 0; irowSrc < ccol; irowSrc++)
{
int idst = irowSrc;
for (int icolSrc = 0; icolSrc < ccol; icolSrc++)
{
uScaled[idst] = u[isrc] * eigValuesRcp[icolSrc];
uInvScaled[idst] = u[isrc] * eigValues[icolSrc];
isrc++;
idst += ccol;
}
}
// For ZCA need to do additional multiply by U.
if (ex.Kind == WhiteningKind.Pca)
{
// Save all components for PCA. Retained components will be selected during evaluation.
models[iinfo] = uScaled;
if (ex.SaveInv)
{
invModels[iinfo] = uInvScaled;
}
}
else if (ex.Kind == WhiteningKind.Zca)
{
models[iinfo] = new float[u.Length];
Mkl.Gemm(Layout, Mkl.Transpose.NoTrans, Mkl.Transpose.NoTrans,
ccol, ccol, ccol, 1, u, ccol, uScaled, ccol, 0, models[iinfo], ccol);
if (ex.SaveInv)
{
invModels[iinfo] = new float[u.Length];
Mkl.Gemm(Layout, Mkl.Transpose.NoTrans, Mkl.Transpose.NoTrans,
ccol, ccol, ccol, 1, u, ccol, uInvScaled, ccol, 0, invModels[iinfo], ccol);
}
}
else
{
ch.Assert(false);
}
}
}
