/// <summary>
/// Calculates polynom regression for xData = [0, 1, ... , n] and yData = [y1, y2, ... , yn].
/// </summary>
/// <param name="order">Order of output polynom.</param>
public PolynominalRegression(DenseVector yData, int order)
{
_order = order;
var vandMatrix = new DenseMatrix(yData.Count, order + 1);
for (int i = 0; i < yData.Count; i++)
{
double mult = 1;
for (int j = 0; j < order + 1; j++)
{
vandMatrix[i, j] = mult;
mult *= i;
}
}
_coefs = vandMatrix.TransposeThisAndMultiply(vandMatrix).LU().Solve(TransposeAndMult(vandMatrix, yData));
}
private static Tuple<double, double> Q3_6Simulation(int k)
{
double lambda = Math.Pow(10, k);
//load training set
var trainingData = System.IO.File.ReadLines(@"c:/projects/studies/edx/cs1156x/net/hw6/in.dta").Select(
line => line.Trim().Replace(" ", ",").Replace(" ", ",").Split(',').Select(v => Convert.ToDouble(v)).ToArray()).ToArray();
//load test set
var testData = System.IO.File.ReadLines(@"c:/projects/studies/edx/cs1156x/net/hw6/out.dta").Select(
line => line.Trim().Replace(" ", ",").Replace(" ", ",").Split(',').Select(v => Convert.ToDouble(v)).ToArray()).ToArray();
int N = trainingData.Length;
var Z = new DenseMatrix(N, 8);
var Y = new DenseVector(N);
for (int j = 0; j < N; j++)
{
double x1 = trainingData[j][0], x2 = trainingData[j][1];
Z[j, 0] = 1;
Z[j, 1] = x1;
Z[j, 2] = x2;
Z[j, 3] = x1 * x1;
Z[j, 4] = x2 * x2;
Z[j, 5] = x1 * x2;
Z[j, 6] = Math.Abs(x1 - x2);
Z[j, 7] = Math.Abs(x1 + x2);
Y[j] = trainingData[j][2];
}
var W = Z.TransposeThisAndMultiply(Z).Add(DenseMatrix.Identity(8).Multiply(lambda)).Inverse().TransposeAndMultiply(Z).Multiply(Y);
Func<double, double, double> h = (x1, x2) =>
W[0] + W[1] * x1 + W[2] * x2 + W[3] * x1 * x1 + W[4] * x2 * x2 + W[5] * x1 * x2
+ W[6] * Math.Abs(x1 - x2) + W[7] * Math.Abs(x1 + x2) >= 0 ? 1.0 : -1.0;
double eIn = (trainingData.Count(v => Math.Sign(h(v[0], v[1])) != Math.Sign(v[2])) + 0.0) / trainingData.Length;
double eOut = (testData.Count(v => Math.Sign(h(v[0], v[1])) != Math.Sign(v[2])) + 0.0) / testData.Length;
return Tuple.Create(eIn, eOut);
}
/// <summary>
/// Calculates polynom regression for xData = [x1, x2, ... , xn] and yData = [y1, y2, ... , yn].
/// </summary>
/// <param name="order">Order of output polynom.</param>
public PolynominalRegression(DenseVector xData, DenseVector yData, int order)
{
_order = order;
var vandMatrix = new DenseMatrix(xData.Count, order + 1);
for (int i = 0; i < xData.Count; i++)
{
double mult = 1;
for (int j = 0; j < order + 1; j++)
{
vandMatrix[i, j] = mult;
mult *= xData[i];
}
}
// var vandMatrixT = vandMatrix.Transpose();
// 1 variant:
//_coefs = (vandMatrixT * vandMatrix).Inverse() * vandMatrixT * yData;
// 2 variant:
//_coefs = (vandMatrixT * vandMatrix).LU().Solve(vandMatrixT * yData);
// 3 variant (most fast I think. Possible LU decomposion also can be replaced with one triangular matrix):
_coefs = vandMatrix.TransposeThisAndMultiply(vandMatrix).LU().Solve(TransposeAndMult(vandMatrix, yData));
}
private static Tuple<double, double> RunRegularizedRegression(
svm_problem trainingData,
svm_problem testData,
Func<double, double, double[]> zTransform,
double lambda = 1.0)
{
int zLength = zTransform(0, 0).Length;
int N = trainingData.l;
var Z = new DenseMatrix(N, zLength);
var Y = new DenseVector(N);
for (int j = 0; j < N; j++)
{
Y[j] = trainingData.y[j];
var zj = zTransform(trainingData.x[j][0].value, trainingData.x[j][1].value);
for (int k = 0; k < zLength; k++)
Z[j, k] = zj[k];
}
var W = Z.TransposeThisAndMultiply(Z).Add(DenseMatrix.Identity(zLength).Multiply(lambda)).Inverse().TransposeAndMultiply(Z).Multiply(Y);
Func<double, double, double> h = (x1, x2) => W.DotProduct(DenseVector.OfEnumerable(zTransform(x1, x2))) >= 0 ? 1.0 : -1.0;
Func<svm_problem, double> E = data => (Enumerable.Range(0, data.l).Count(i => Math.Sign(h(data.x[i][0].value, data.x[i][1].value)) !=
Math.Sign(data.y[i])) + 0.0) / data.l;
return Tuple.Create(E(trainingData), E(testData));
}
static Tuple<bool, double, double> RunLloydsRbf(svm_problem training, svm_problem test, int K, double gamma)
{
Func<double[], double[], double> Distance = (x, y) => (x[0] - y[0]) * (x[0] - y[0]) + (x[1] - y[1]) * (x[1] - y[1]);
Random rnd = new Random();
//use LLoyd's to cluster points -----------------------------------------------------------------------------------------------
//pick K points at random as cluster centers
var mu = training.x.Select(v => v.Select(u => u.value).ToArray()).OrderBy(v => rnd.NextDouble()).Take(K).ToArray();
while(true)
{
//cluster points
var clusters = training.x.Select(v => new { x = v, cluster = Enumerable.Range(0, K).OrderBy(cluster =>
Distance(mu[cluster], v.Select(u => u.value).ToArray())).First() }).OrderBy(v => v.cluster).GroupBy(
v => v.cluster, v => v.x, (v, u) => u.ToArray()).ToArray();
//check for empty clusters
if (clusters.Any(v => v.Length == 0))
return Tuple.Create(false, 0.0, 0.0);
//get new mus
var newMu = clusters.Select(v => new[] { v.Sum(u => u[0].value) / v.Length, v.Sum(u => u[1].value) / v.Length }).ToArray();
if (mu.Zip(newMu, (v, u) => v[0] == u[0] && v[1] == u[1]).All(v => v))
break;
mu = newMu;
}
//solve to find W
var Phi = new DenseMatrix(training.l, K + 1);
var Y = new DenseVector(training.l);
for (int i = 0; i < training.l; i++)
{
Y[i] = training.y[i];
Phi[i, 0] = 1;
for (int j = 0; j < K; j++)
Phi[i, j+1] = Math.Exp(-gamma * Distance(training.x[i].Select(v => v.value).ToArray(), mu[j]));
}
var W = Phi.TransposeThisAndMultiply(Phi).Inverse().TransposeAndMultiply(Phi).Multiply(Y);
Func<double[], double> h = x => W[0] + W.Skip(1).Zip(mu, (w, m) => w * Math.Exp(-gamma * Distance(x, m))).Sum();
Func<svm_problem, double> E = data => (Enumerable.Range(0, data.l).Count(i => Math.Sign(h(data.x[i].Select(v => v.value).ToArray())) !=
Math.Sign(data.y[i])) + 0.0) / data.l;
return Tuple.Create(true, E(training), E(test));
}
/// <summary>
/// Non-linear-transformed linear regression simulation for homework Q2 of the
/// 6th week of the CS1156x "Learning From Data" at eDX
/// </summary>
static void RunQ2Simulation()
{
//load training set
var trainingData = System.IO.File.ReadLines(@"c:/projects/studies/edx/cs1156x/net/hw6/in.dta").Select(
line => line.Trim().Replace(" ", ",").Replace(" ", ",").Split(',').Select(v => Convert.ToDouble(v)).ToArray()).ToArray();
//load test set
var testData = System.IO.File.ReadLines(@"c:/projects/studies/edx/cs1156x/net/hw6/out.dta").Select(
line => line.Trim().Replace(" ", ",").Replace(" ", ",").Split(',').Select(v => Convert.ToDouble(v)).ToArray()).ToArray();
int N = trainingData.Length;
var Z = new DenseMatrix(N, 8);
var Y = new DenseVector(N);
for (int j = 0; j < N; j++)
{
double x1 = trainingData[j][0], x2 = trainingData[j][1];
Z[j, 0] = 1;
Z[j, 1] = x1;
Z[j, 2] = x2;
Z[j, 3] = x1 * x1;
Z[j, 4] = x2 * x2;
Z[j, 5] = x1 * x2;
Z[j, 6] = Math.Abs(x1 - x2);
Z[j, 7] = Math.Abs(x1 + x2);
Y[j] = trainingData[j][2];
}
//Z.QR().Solve(DenseMatrix.Identity(Z.RowCount)).Multiply(Y);
var W = Z.TransposeThisAndMultiply(Z).Inverse().TransposeAndMultiply(Z).Multiply(Y);
Func<double, double, double> h = (x1, x2) =>
W[0] + W[1] * x1 + W[2] * x2 + W[3] * x1 * x1 + W[4] * x2 * x2 + W[5] * x1 * x2
+ W[6] * Math.Abs(x1 - x2) + W[7] * Math.Abs(x1 + x2) >= 0 ? 1.0 : -1.0;
double eIn = (trainingData.Count(v => Math.Sign(h(v[0], v[1])) != Math.Sign(v[2])) + 0.0) / trainingData.Length;
double eOut = (testData.Count(v => Math.Sign(h(v[0], v[1])) != Math.Sign(v[2])) + 0.0) / testData.Length;
Console.Out.WriteLine("HW6 Q2:");
Console.Out.WriteLine("\teIn = {0}", eIn);
Console.Out.WriteLine("\teOut = {0}", eOut);
}
