public GaussianProcessModel(IDataset ds, string targetVariable, IEnumerable <string> allowedInputVariables, IEnumerable <int> rows,
IEnumerable <double> hyp, IMeanFunction meanFunction, ICovarianceFunction covarianceFunction,
bool scaleInputs = true)
: base()
{
this.name = ItemName;
this.description = ItemDescription;
this.meanFunction = (IMeanFunction)meanFunction.Clone();
this.covarianceFunction = (ICovarianceFunction)covarianceFunction.Clone();
this.targetVariable = targetVariable;
this.allowedInputVariables = allowedInputVariables.ToArray();
int nVariables = this.allowedInputVariables.Length;
meanParameter = hyp
.Take(this.meanFunction.GetNumberOfParameters(nVariables))
.ToArray();
covarianceParameter = hyp.Skip(this.meanFunction.GetNumberOfParameters(nVariables))
.Take(this.covarianceFunction.GetNumberOfParameters(nVariables))
.ToArray();
sqrSigmaNoise = Math.Exp(2.0 * hyp.Last());
try {
CalculateModel(ds, rows, scaleInputs);
} catch (alglib.alglibexception ae) {
// wrap exception so that calling code doesn't have to know about alglib implementation
throw new ArgumentException("There was a problem in the calculation of the Gaussian process model", ae);
}
}
public void ClearState()
{
meanFunc = null;
covFunc = null;
bestQ = double.NegativeInfinity;
bestHyperParameters = null;
}
private GaussianProcessModel(GaussianProcessModel original, Cloner cloner)
: base(original, cloner)
{
this.meanFunction = cloner.Clone(original.meanFunction);
this.covarianceFunction = cloner.Clone(original.covarianceFunction);
if (original.inputScaling != null)
{
this.inputScaling = cloner.Clone(original.inputScaling);
}
this.trainingDataset = cloner.Clone(original.trainingDataset);
this.negativeLogLikelihood = original.negativeLogLikelihood;
this.targetVariable = original.targetVariable;
this.sqrSigmaNoise = original.sqrSigmaNoise;
if (original.meanParameter != null)
{
this.meanParameter = (double[])original.meanParameter.Clone();
}
if (original.covarianceParameter != null)
{
this.covarianceParameter = (double[])original.covarianceParameter.Clone();
}
// shallow copies of arrays because they cannot be modified
this.trainingRows = original.trainingRows;
this.allowedInputVariables = original.allowedInputVariables;
this.alpha = original.alpha;
this.l = original.l;
this.x = original.x;
}
// cloning
private GaussianProcessCovarianceOptimizationProblem(GaussianProcessCovarianceOptimizationProblem original, Cloner cloner)
: base(original, cloner)
{
bestQ = original.bestQ;
meanFunc = cloner.Clone(original.meanFunc);
covFunc = cloner.Clone(original.covFunc);
if (bestHyperParameters != null)
{
bestHyperParameters = new double[original.bestHyperParameters.Length];
Array.Copy(original.bestHyperParameters, bestHyperParameters, bestHyperParameters.Length);
}
}
// updates the overall best quality and overall best model for Analyze()
private void UpdateBestSoFar(double bestQ, double[] bestHyperParameters, IMeanFunction meanFunc, ICovarianceFunction covFunc)
{
lock (problemStateLocker) {
if (bestQ > this.bestQ)
{
this.bestQ = bestQ;
this.bestHyperParameters = new double[bestHyperParameters.Length];
Array.Copy(bestHyperParameters, this.bestHyperParameters, this.bestHyperParameters.Length);
this.meanFunc = meanFunc;
this.covFunc = covFunc;
}
}
}
private KernelRidgeRegressionModel(KernelRidgeRegressionModel original, Cloner cloner)
: base(original, cloner)
{
// shallow copies of arrays because they cannot be modified
allowedInputVariables = original.allowedInputVariables;
alpha = original.alpha;
trainX = original.trainX;
scaling = original.scaling;
lambda = original.lambda;
LooCvRMSE = original.LooCvRMSE;
yOffset = original.yOffset;
yScale = original.yScale;
kernel = original.kernel;
}
private static double[,] BuildGramMatrix(double[,] data, double lambda, ICovarianceFunction kernel)
{
var n = data.GetLength(0);
var dim = data.GetLength(1);
var cov = kernel.GetParameterizedCovarianceFunction(new double[0], Enumerable.Range(0, dim).ToArray());
var gram = new double[n, n];
// G = (K + λ I)
for (var i = 0; i < n; i++)
{
for (var j = i; j < n; j++)
{
gram[i, j] = gram[j, i] = cov.Covariance(data, i, j); // symmetric matrix
}
gram[i, i] += lambda;
}
return(gram);
}
private KernelRidgeRegressionModel(IDataset dataset, string targetVariable, IEnumerable <string> allowedInputVariables, int[] rows,
bool scaleInputs, ICovarianceFunction kernel, double lambda = 0.1) : base(targetVariable)
{
this.allowedInputVariables = allowedInputVariables.ToArray();
if (kernel.GetNumberOfParameters(this.allowedInputVariables.Length) > 0)
{
throw new ArgumentException("All parameters in the kernel function must be specified.");
}
name = ItemName;
description = ItemDescription;
this.kernel = (ICovarianceFunction)kernel.Clone();
this.lambda = lambda;
if (scaleInputs)
{
scaling = CreateScaling(dataset, rows, this.allowedInputVariables);
}
trainX = ExtractData(dataset, rows, this.allowedInputVariables, scaling);
var y = dataset.GetDoubleValues(targetVariable, rows).ToArray();
yOffset = y.Average();
yScale = 1.0 / y.StandardDeviation();
alpha = new double[trainX.GetLength(0)];
}
private void TestCovarianceFunction(ICovarianceFunction cf, double hypValue, double[,] expectedCov, double[][,] expectedGradients, double delta = 1E-3) {
var x = GetData();
var xt = GetDataTest();
int nHyp = cf.GetNumberOfParameters(x.GetLength(1));
var hyp = Enumerable.Repeat(hypValue, nHyp).ToArray();
int rows0 = x.GetLength(0);
int rows1 = xt.GetLength(0);
var actualCov = new double[rows0, rows1];
var covFunction = cf.GetParameterizedCovarianceFunction(hyp, Enumerable.Range(0, x.GetLength(1)).ToArray());
for (int i = 0; i < rows0; i++)
for (int j = 0; j < rows1; j++)
actualCov[i, j] = covFunction.CrossCovariance(x, xt, i, j);
AssertEqual(expectedCov, actualCov, delta);
for (int i = 0; i < rows0; i++)
for (int j = 0; j < rows1; j++) {
var g = covFunction.CovarianceGradient(x, i, j).ToArray();
for (int k = 0; k < nHyp; k++)
Assert.AreEqual(expectedGradients[k][i, j], g[k], delta);
}
}
public static IGaussianProcessModel Create(IRegressionProblemData problemData, double[] hyperparameter, IMeanFunction meanFunction, ICovarianceFunction covarianceFunction, bool scaleInputs = true) {
return new GaussianProcessModel(problemData.Dataset, problemData.TargetVariable, problemData.AllowedInputVariables, problemData.TrainingIndices, hyperparameter, meanFunction, covarianceFunction, scaleInputs);
}
public GaussianProcessModel(IDataset ds, string targetVariable, IEnumerable<string> allowedInputVariables, IEnumerable<int> rows,
IEnumerable<double> hyp, IMeanFunction meanFunction, ICovarianceFunction covarianceFunction,
bool scaleInputs = true)
: base(targetVariable) {
this.name = ItemName;
this.description = ItemDescription;
this.meanFunction = (IMeanFunction)meanFunction.Clone();
this.covarianceFunction = (ICovarianceFunction)covarianceFunction.Clone();
this.allowedInputVariables = allowedInputVariables.ToArray();
int nVariables = this.allowedInputVariables.Length;
meanParameter = hyp
.Take(this.meanFunction.GetNumberOfParameters(nVariables))
.ToArray();
covarianceParameter = hyp.Skip(this.meanFunction.GetNumberOfParameters(nVariables))
.Take(this.covarianceFunction.GetNumberOfParameters(nVariables))
.ToArray();
sqrSigmaNoise = Math.Exp(2.0 * hyp.Last());
try {
CalculateModel(ds, rows, scaleInputs);
}
catch (alglib.alglibexception ae) {
// wrap exception so that calling code doesn't have to know about alglib implementation
throw new ArgumentException("There was a problem in the calculation of the Gaussian process model", ae);
}
}
private GaussianProcessModel(GaussianProcessModel original, Cloner cloner)
: base(original, cloner) {
this.meanFunction = cloner.Clone(original.meanFunction);
this.covarianceFunction = cloner.Clone(original.covarianceFunction);
if (original.inputScaling != null)
this.inputScaling = cloner.Clone(original.inputScaling);
this.trainingDataset = cloner.Clone(original.trainingDataset);
this.negativeLogLikelihood = original.negativeLogLikelihood;
this.sqrSigmaNoise = original.sqrSigmaNoise;
if (original.meanParameter != null) {
this.meanParameter = (double[])original.meanParameter.Clone();
}
if (original.covarianceParameter != null) {
this.covarianceParameter = (double[])original.covarianceParameter.Clone();
}
// shallow copies of arrays because they cannot be modified
this.trainingRows = original.trainingRows;
this.allowedInputVariables = original.allowedInputVariables;
this.alpha = original.alpha;
this.l = original.l;
this.x = original.x;
}
public static IRegressionSolution CreateRadialBasisRegressionSolution(IRegressionProblemData problemData, ICovarianceFunction kernel, double lambda, bool scaleInputs, out double rmsError, out double looCvRMSE)
{
var model = KernelRidgeRegressionModel.Create(problemData.Dataset, problemData.TargetVariable, problemData.AllowedInputVariables, problemData.TrainingIndices, scaleInputs, kernel, lambda);
rmsError = double.NaN;
if (problemData.TestIndices.Any())
{
rmsError = Math.Sqrt(model.GetEstimatedValues(problemData.Dataset, problemData.TestIndices)
.Zip(problemData.TargetVariableTestValues, (a, b) => (a - b) * (a - b))
.Average());
}
var solution = model.CreateRegressionSolution((IRegressionProblemData)problemData.Clone());
solution.Model.Name = "Kernel ridge regression model";
solution.Name = SolutionResultName;
looCvRMSE = model.LooCvRMSE;
return(solution);
}
public static IGaussianProcessModel Create(IClassificationProblemData problemData, double[] hyperparameter, IMeanFunction meanFunction, ICovarianceFunction covarianceFunction, bool scaleInputs = true)
{
return(new GaussianProcessModel(problemData.Dataset, problemData.TargetVariable, problemData.AllowedInputVariables, problemData.TrainingIndices, hyperparameter, meanFunction, covarianceFunction, scaleInputs));
}
public static KernelRidgeRegressionModel Create(IDataset dataset, string targetVariable, IEnumerable <string> allowedInputVariables, IEnumerable <int> rows,
bool scaleInputs, ICovarianceFunction kernel, double lambda = 0.1)
{
var trainingRows = rows.ToArray();
var model = new KernelRidgeRegressionModel(dataset, targetVariable, allowedInputVariables, trainingRows, scaleInputs, kernel, lambda);
try {
int info;
int n = model.trainX.GetLength(0);
alglib.densesolverreport denseSolveRep;
var gram = BuildGramMatrix(model.trainX, lambda, kernel);
var l = new double[n, n];
Array.Copy(gram, l, l.Length);
double[] alpha = new double[n];
double[,] invG;
var y = dataset.GetDoubleValues(targetVariable, trainingRows).ToArray();
for (int i = 0; i < y.Length; i++)
{
y[i] -= model.yOffset;
y[i] *= model.yScale;
}
// cholesky decomposition
var res = alglib.trfac.spdmatrixcholesky(ref l, n, false);
if (res == false) //try lua decomposition if cholesky faild
{
int[] pivots;
var lua = new double[n, n];
Array.Copy(gram, lua, lua.Length);
alglib.rmatrixlu(ref lua, n, n, out pivots);
alglib.rmatrixlusolve(lua, pivots, n, y, out info, out denseSolveRep, out alpha);
if (info != 1)
{
throw new ArgumentException("Could not create model.");
}
alglib.matinvreport rep;
invG = lua; // rename
alglib.rmatrixluinverse(ref invG, pivots, n, out info, out rep);
}
else
{
alglib.spdmatrixcholeskysolve(l, n, false, y, out info, out denseSolveRep, out alpha);
if (info != 1)
{
throw new ArgumentException("Could not create model.");
}
// for LOO-CV we need to build the inverse of the gram matrix
alglib.matinvreport rep;
invG = l; // rename
alglib.spdmatrixcholeskyinverse(ref invG, n, false, out info, out rep);
}
if (info != 1)
{
throw new ArgumentException("Could not invert Gram matrix.");
}
var ssqLooError = 0.0;
for (int i = 0; i < n; i++)
{
var pred_i = Util.ScalarProd(Util.GetRow(gram, i).ToArray(), alpha);
var looPred_i = pred_i - alpha[i] / invG[i, i];
var error = (y[i] - looPred_i) / model.yScale;
ssqLooError += error * error;
}
Array.Copy(alpha, model.alpha, n);
model.LooCvRMSE = Math.Sqrt(ssqLooError / n);
} catch (alglib.alglibexception ae) {
// wrap exception so that calling code doesn't have to know about alglib implementation
throw new ArgumentException("There was a problem in the calculation of the kernel ridge regression model", ae);
}
return(model);
}
