/// <summary>
/// Writes a Matrix object to the underlying stream.
/// </summary>
/// <param name="matrix"></param>
public void WriteMatrix(Matrix matrix)
{
WriteBeginArray();
bool first = true;
foreach (var vector in matrix.GetRows())
{
if (!first) WriteToken(JsonConstants.COMMA);
WriteArray(vector as IEnumerable);
first = false;
}
WriteEndArray();
}
/// <summary>
/// Generates a new Collaborative Filtering model.
/// </summary>
/// <param name="X">Training matrix values.</param>
/// <param name="y">Vector of entity identifiers.</param>
/// <returns></returns>
public override IModel Generate(Matrix X, Vector y)
{
this.Preprocess(X.Copy());
// inputs are ratings from each user (X = entities x ratings), y = entity id.
// create rating range in case we don't have one already
if (this.Ratings == null)
this.Ratings = new Range(X.Min(), X.Max());
// indicator matrix of 1's where rating was provided otherwise 0's.
Matrix R = X.ToBinary(f => this.Ratings.Test(f));
// The mean needs to be values within rating range only.
Vector mean = X.GetRows().Select(s =>
s.Where(w => this.Ratings.Test(w)).Sum() /
s.Where(w => this.Ratings.Test(w)).Count()
).ToVector();
// update feature averages before preprocessing features.
this.FeatureProperties.Average = mean;
this.Preprocess(X);
// where references could be user ratings and entities are movies / books, etc.
int references = X.Cols, entities = X.Rows;
// initialize Theta parameters
Matrix ThetaX = Matrix.Rand(entities, this.CollaborativeFeatures, -1d);
Matrix ThetaY = Matrix.Rand(references, this.CollaborativeFeatures, -1d);
numl.Math.Functions.Cost.ICostFunction costFunction = new numl.Math.Functions.Cost.CofiCostFunction()
{
CollaborativeFeatures = this.CollaborativeFeatures,
Lambda = this.Lambda,
R = R,
Regularizer = null,
X = ThetaX,
Y = X.Unshape()
};
// we're optimising two params so combine them
Vector Theta = Vector.Combine(ThetaX.Unshape(), ThetaY.Unshape());
numl.Math.Optimization.Optimizer optimizer = new numl.Math.Optimization.Optimizer(Theta, this.MaxIterations, this.LearningRate)
{
CostFunction = costFunction
};
optimizer.Run();
// extract the optimised parameter Theta
ThetaX = optimizer.Properties.Theta.Slice(0, (ThetaX.Rows * ThetaX.Cols) - 1).Reshape(entities, VectorType.Row);
ThetaY = optimizer.Properties.Theta.Slice(ThetaX.Rows * ThetaX.Cols, Theta.Length - 1).Reshape(references, VectorType.Row);
// create reference mappings, each value is the original index.
this.ReferenceFeatureMap = (this.ReferenceFeatureMap == null ? Vector.Create(references, i => i) : this.ReferenceFeatureMap);
this.EntityFeatureMap = (this.EntityFeatureMap == null ? Vector.Create(entities, i => i) : this.EntityFeatureMap);
return new CofiRecommenderModel()
{
Descriptor = this.Descriptor,
NormalizeFeatures = this.NormalizeFeatures,
FeatureNormalizer = this.FeatureNormalizer,
FeatureProperties = this.FeatureProperties,
Ratings = this.Ratings,
ReferenceFeatureMap = this.ReferenceFeatureMap,
EntityFeatureMap = this.EntityFeatureMap,
Mu = mean,
Y = y,
Reference = X,
ThetaX = ThetaX,
ThetaY = ThetaY
};
}
/// <summary>
/// Slices the input matrix using starting and stopping positions.
/// </summary>
/// <param name="m">Source matrix.</param>
/// <param name="minIndex">Minimum index to slice from.</param>
/// <param name="maxIndex">Maximum index to slice.</param>
/// <param name="t"></param>
/// <returns></returns>
public static Matrix Slice(this Matrix m, int minIndex, int maxIndex, VectorType t = VectorType.Row)
{
IEnumerable <Vector> array = (t == VectorType.Row ? m.GetRows() : m.GetCols());
return(array.Skip(minIndex).Take((maxIndex - minIndex) + 1).ToMatrix(t));
}
/// <summary>Generates.</summary>
/// <param name="X">The Matrix to process.</param>
/// <param name="k">The int to process.</param>
public void Generate(Matrix X, int k)
{
int n = X.Rows;
int d = X.Cols;
/***********************
* initialize parameters
***********************/
// convergence params
var log_probability = 0d;
var probability_difference = double.MaxValue;
var mu_difference = double.MaxValue;
// initialize centers with KMeans
KMeans kmeans = new KMeans();
var asgn = kmeans.Generate(X, k, new EuclidianDistance());
// tentative centers
var mu_k = kmeans.Centers;
// initial covariances (stored as diag(cov) 1 of k)
var sg_k = new Matrix(k, d);
for (int i = 0; i < k; i++)
{
var indices = asgn.Select((a, b) => new Tuple<int, int>(a, b)).Where(t => t.Item1 == i).Select(t => t.Item2);
var matrix = X.Slice(indices, VectorType.Row);
sg_k[i] = matrix.CovarianceDiag();
}
// mixing coefficient
var pi_k = asgn
.OrderBy(i => i)
.GroupBy(j => j)
.Select(g => (double)g.Count() / (double)asgn.Length)
.ToVector();
int max_iter = 100;
do
{
/***********************
* Expectation Step
***********************/
// responsibilty matrix: how much is gaussian k responsible for this point x
var z_nk = new Matrix(n, k);
for (int i = 0; i < n; i++)
{
// pi_j * N(x_n | mu_j, sigma_j)
for (int j = 0; j < k; j++)
z_nk[i, j] = pi_k[j] * Normal(X[i], mu_k[j], sg_k[j]);
var dn = z_nk[i].Sum();
if(dn == 0)
Console.WriteLine("Uh oh....");
z_nk[i].Each(z => z / dn);
}
/***********************
* Maximization Step
***********************/
var N_k = z_nk.Sum(VectorType.Row);
var mu_k_new = new Matrix(mu_k.Rows, mu_k.Cols);
for (int i = 0; i < k; i++)
{
var sum = Vector.Zeros(d);
for (int j = 0; j < n; j++)
sum += z_nk[j, i] * X[j];
mu_k_new[i] = sum / N_k[i];
}
var sg_k_new = new Matrix(k, d);
for (int i = 0; i < k; i++)
{
var sum = Vector.Zeros(d);
for (int j = 0; j < n; j++)
sum += z_nk[j, i] * (X[j] - mu_k_new[i]).Each(s => s * s);
sg_k_new[i] = sum / N_k[i];
}
var pi_k_new = N_k / n;
/***********************
* Convergence Check
***********************/
var new_log_prob = 0d;
for (int i = 0; i < n; i++)
{
var acc = 0d;
// pi_j * N(x_n | mu_j, sigma_j)
for (int j = 0; j < k; j++)
acc += pi_k[j] * Normal(X[i], mu_k[j], sg_k[j]);
new_log_prob += System.Math.Log(acc, System.Math.E);
}
// log likelihood differences
probability_difference = System.Math.Abs(log_probability - new_log_prob);
Console.WriteLine("Log Likelihoods (Total Points: {0}, k={1}, d={2})\nO: {3}\nN: {4}\nDifference: {5}\n", n, k, d, log_probability, new_log_prob, probability_difference);
log_probability = new_log_prob;
// centers differences
mu_difference = mu_k.GetRows()
.Zip(mu_k_new.GetRows(), (v1, v2) => new { V1 = v1, V2 = v2 })
.Sum(a => (a.V1 - a.V2).Norm());
Console.WriteLine("Centers:\nO: {0}\nN: {1}\nDifference: {2}\n", mu_k, mu_k_new, mu_difference);
mu_k = mu_k_new;
// covariance differences
double diff = sg_k.GetRows()
.Zip(sg_k_new.GetRows(), (v1, v2) => new { V1 = v1, V2 = v2 })
.Sum(a => (a.V1 - a.V2).Norm());
Console.WriteLine("Covariance:\nO: {0}\nN: {1}\nDifference: {2}\n", sg_k, sg_k_new, diff);
sg_k = sg_k_new;
// mixing differences
diff = (pi_k - pi_k_new).Each(s => System.Math.Abs(s)).Sum();
Console.WriteLine("Mixing Coeffs:\nO: {0}\nN: {1}\nDifference: {2}\n", pi_k, pi_k_new, diff);
pi_k = pi_k_new;
Console.WriteLine("-------------------------------------------------------------");
} while (probability_difference > .0000000001 && mu_difference > .0000000001 && --max_iter >= 0);
}
/// <summary>
/// Returns a vector of the median values for each row or column.
/// </summary>
/// <param name="source">Matrix.</param>
/// <param name="t">VectorType.</param>
/// <returns></returns>
public static Vector Median(Matrix source, VectorType t = VectorType.Col)
{
var vectors = (t == VectorType.Row ? source.GetCols() : source.GetRows());
return(vectors.Select(s => s.Median()).ToVector());
}
/// <summary>
/// Unshapes the given Matrix into a Vector form along the <paramref name="dimensionType"/> axis.
/// <para>Reads from the source Matrix and stacks from right to left when <paramref name="dimensionType"/> equals 'Col' otherwise uses a bottom up approach.</para>
/// </summary>
/// <param name="m">The Matrix to act on.</param>
/// <param name="dimensionType">Type of the dimension to use when unrolling the Matrix.</param>
/// <returns>Matrix.</returns>
public static Vector Unshape(Matrix m, VectorType dimensionType = VectorType.Col)
{
return(Vector.Combine((dimensionType == VectorType.Col ? m.GetCols().ToArray() : m.GetRows().ToArray())));
}
/// <summary>
/// Generates a GRU neural network model for predicting sequences.
/// </summary>
/// <param name="X">Matrix of training data.</param>
/// <param name="Y">Matrix of matching sequence labels.</param>
/// <returns>GatedRecurrentModel.</returns>
public ISequenceModel Generate(Matrix X, Matrix Y)
{
this.Preprocess(X);
// because Seth said so...
if (MaxIterations <= 0)
MaxIterations = 500;
Network network = Network.New().Create(X.Cols, Y.Cols, Activation, OutputFunction,
fnNodeInitializer: (i, j) => new RecurrentNeuron()
{
ActivationFunction = this.Activation,
ResetGate = this.ResetGate,
MemoryGate = this.UpdateGate,
DeltaH = Vector.Zeros(this.SequenceLength)
}, epsilon: Epsilon);
var model = new GatedRecurrentModel
{
Descriptor = Descriptor,
NormalizeFeatures = base.NormalizeFeatures,
FeatureNormalizer = base.FeatureNormalizer,
FeatureProperties = base.FeatureProperties,
Network = network,
OutputFunction = this.OutputFunction
};
int m = X.Rows;
OnModelChanged(this, ModelEventArgs.Make(model, "Initialized"));
NetworkTrainingProperties properties = NetworkTrainingProperties.Create(network, X.Rows, X.Cols, this.LearningRate, this.Lambda, this.MaxIterations,
new { this.SequenceLength });
Vector loss = Vector.Zeros(MaxIterations);
var tuples = X.GetRows().Select((s, si) => new Tuple<Vector, Vector>(s, Y[si]));
for (int pass = 0; pass < MaxIterations; pass++)
{
properties.Iteration = pass;
tuples.Batch(SequenceLength, (idx, items) =>
{
network.ResetStates(properties);
for (int i = 0; idx < items.Count(); idx++)
{
network.Forward(items.ElementAt(i).Item1);
network.Back(items.ElementAt(i).Item2, properties);
}
}, asParallel: false);
loss[pass] = network.Cost;
var output = String.Format("Run ({0}/{1}): {2}", pass, MaxIterations, network.Cost);
OnModelChanged(this, ModelEventArgs.Make(model, output));
}
return model;
}
/// <summary>
/// Slices the input matrix using starting and stopping positions.
/// </summary>
/// <param name="m">Source matrix.</param>
/// <param name="minIndex">Minimum index to slice from.</param>
/// <param name="maxIndex">Maximum index to slice.</param>
/// <param name="t"></param>
/// <returns></returns>
public static Matrix Slice(this Matrix m, int minIndex, int maxIndex, VectorType t = VectorType.Row)
{
var array = t == VectorType.Row ? m.GetRows() : m.GetCols();
return(array.Skip(minIndex).Take(maxIndex - minIndex + 1).ToMatrix(t));
}
/// <summary>
/// Unshapes the given Matrix into a Vector form along the <paramref name="dimensionType"/> axis.
/// <para>Reads from the source Matrix and stacks from right to left when <paramref name="dimensionType"/> equals 'Col' otherwise uses a bottom up approach.</para>
/// </summary>
/// <param name="m">The Matrix to act on.</param>
/// <param name="dimensionType">Type of the dimension to use when unrolling the Matrix.</param>
/// <returns>Matrix.</returns>
public static Vector Unshape(Matrix m, VectorType dimensionType = VectorType.Col)
{
return Vector.Combine((dimensionType == VectorType.Col ? m.GetCols().ToArray() : m.GetRows().ToArray()));
}
/// <summary>
/// Sorts the given Matrix by the specified row or column selector and returns the new Matrix
/// along with the original indices.
/// </summary>
/// <param name="source">The Matrix</param>
/// <param name="keySelector">Property selector to sort by.</param>
/// <param name="t">Specifies whether to sort horizontally or vertically.</param>
/// <param name="ascending">Determines whether to sort ascending or descending (Default: True)</param>
/// <param name="indices">Vector of <paramref name="t"/> indices in the original Matrix before the sort operation.</param>
/// <returns>New Matrix and Vector of original indices.</returns>
public static Matrix Sort(Matrix source, Func<Vector, double> keySelector, VectorType t, bool ascending, out Vector indices)
{
int max = (t == VectorType.Row ? source.Rows : source.Cols);
indices = Vector.Zeros(max);
List<Vector> vects = new List<Vector>(max);
IEnumerable<Vector> arrays = (t == VectorType.Row ? source.GetRows() : source.GetCols());
KeyValuePair<Vector, int>[] sort = (ascending ?
arrays.Select((i, v) => new KeyValuePair<Vector, int>(i, v))
.OrderBy(o => keySelector(o.Key))
:
arrays.Select((i, v) => new KeyValuePair<Vector, int>(i, v))
.OrderByDescending(o => keySelector(o.Key))).ToArray();
indices = sort.Select(s => s.Value).ToVector();
return sort.Select(s => s.Key).ToMatrix(t);
}
/// <summary>
/// Returns a vector of the median values for each row or column.
/// </summary>
/// <param name="source">Matrix.</param>
/// <param name="t">VectorType.</param>
/// <returns></returns>
public static Vector Median(Matrix source, VectorType t = VectorType.Col)
{
var vectors = (t == VectorType.Row ? source.GetCols() : source.GetRows());
return vectors.Select(s => s.Median()).ToVector();
}
