public void AreClose_Azimuth45_AzimuthSum()
{
// Arrange
const int DistanceMax = 3;
const double DistanceStep = 0.1;
var radiuses = new List <double> {
4, 10, 20, 40, 70, 100, 200
};
List <double> distances = getDistances(DistanceStep, DistanceMax);
string dirAzimuthSum = "RadiusDistanceOutput_AzimuthSum";
string dirAzimuth45 = "RadiusDistanceOutput_Azimuth45";
var gp = new GnuPlot();
setLineStyle(gp);
foreach (double radius in radiuses)
{
foreach (double distance in distances)
{
Dictionary <double, double> azim45 = SimpleFormatter.Read(
this.getFileFormat(dirAzimuth45, distance, radius));
Dictionary <double, double> azimSum = SimpleFormatter.Read(
this.getFileFormat(dirAzimuthSum, distance, radius));
gp.HoldOn();
gp.Plot(azim45);
gp.Plot(azimSum);
gp.HoldOff();
AssertHelper.DictionaryAreClose(azim45, azimSum, 0.5);
}
}
}
private static void PlotValidationCurve(double[] x, double[] jtrain, double[] jvc)
{
GnuPlot.HoldOn();
GnuPlot.Set("title \"Validation curve\"");
GnuPlot.Set("xlabel \"Lamda\"");
GnuPlot.Set("key top right box");
GnuPlot.Set("ylabel \"Error\"");
GnuPlot.Set("autoscale xy");
GnuPlot.Plot(x, jtrain, "with lines ls 1 lw 2 lc rgb \"cyan\" title \"Train\" ");
GnuPlot.Plot(x, jvc, "with lines ls 1 lw 2 lc rgb \"orange\" title \"Cross Validation\" ");
GnuPlot.HoldOff();
}
private static void PlotLinearLearningCurve(double[] x, double[] jtrain, double[] jvc)
{
GnuPlot.HoldOn();
GnuPlot.Set("title \"Learning curve for linear regression\"");
GnuPlot.Set("xlabel \"Number of training examples\"");
GnuPlot.Set("key top right box");
GnuPlot.Set("ylabel \"Error\"");
GnuPlot.Set("xr[0:13]");
GnuPlot.Set("yr[0:150]");
GnuPlot.Plot(x, jtrain, "with lines ls 1 lw 2 lc rgb \"cyan\" title \"Train\" ");
GnuPlot.Plot(x, jvc, "with lines ls 1 lw 2 lc rgb \"orange\" title \"Cross Validation\" ");
GnuPlot.HoldOff();
}
private static void Main(string[] args)
{
string fileTrain1 = "approximation_train_1.txt";
string fileTrain2 = "approximation_train_2.txt";
string fileTest = "approximation_test.txt";
string filePathTrain1 = Path.GetFullPath(fileTrain1);
string filePathTrain2 = Path.GetFullPath(fileTrain2);
string filePathTest = Path.GetFullPath(fileTest);
List <double> outputValues = new List <double>();
DataGetter dataGetterTrain1 = new DataGetter(filePathTrain1);
DataGetter dataGetterTrain2 = new DataGetter(filePathTrain2);
DataGetter dataGetterTest = new DataGetter(filePathTest);
double learnRate = 0.1;
double momentum = 0.1;
int numberOfRadialNeurons = 20;
int numberOfEpochs = 300;
//Console.WriteLine("SIEC NEURONOWA RBF WYKORZYSTYWANA DO APROKSYMACJI FUNKJI");
//Console.WriteLine("Prosze wprowadzic wartosc wspolczynnika nauki:");
//learnRate = Convert.ToDouble(Console.ReadLine(), System.Globalization.CultureInfo.InvariantCulture);
//Console.WriteLine("Prosze wprowadzic wartosc wspolczynnika momentum:");
//momentum = Convert.ToDouble(Console.ReadLine(), System.Globalization.CultureInfo.InvariantCulture);
NeuralNetwork neuralNetwork = new NeuralNetwork(dataGetterTrain1.getInputData(), learnRate, momentum, numberOfRadialNeurons);
Console.WriteLine("ZBIOR TRENINGOWY 1: ");
neuralNetwork.Train(numberOfEpochs, dataGetterTrain2);
Console.WriteLine("ZBIOR TRENINGOWY 2: ");
neuralNetwork.Train(numberOfEpochs, dataGetterTrain1);
Console.WriteLine("ZBIOR TESTOWY: ");
outputValues = neuralNetwork.Test(dataGetterTest);
GnuPlot.Set("term wxt 0");
GnuPlot.HoldOn();
GnuPlot.Set("xlabel 'X Values'");
GnuPlot.Set("ylabel 'Y Values'");
GnuPlot.Plot(dataGetterTest.getInputData().ToArray(), dataGetterTest.getExpectedData().ToArray(), "title 'funkcja oryginalna'");
GnuPlot.Plot(dataGetterTest.getInputData().ToArray(), outputValues.ToArray(), "title 'funkcja po aproksymacji'");
GnuPlot.HoldOff();
GnuPlot.Set("term wxt 1");
GnuPlot.Set("xlabel 'Numer epoki'");
GnuPlot.Set("ylabel 'Wartosc bledu'");
GnuPlot.Plot(neuralNetwork.ErrorX.ToArray(), neuralNetwork.ErrorY.ToArray(), "with lines title 'Blad aproksymacji'");
Console.ReadKey();
}
private static (Matrix <double> centroids, Vector <double> idx) RunkMeans(Matrix <double> X, Matrix <double> initial_centroids, int max_iters, bool plot_progress)
{
// Plot the data if we are plotting progress
if (plot_progress)
{
GnuPlot.HoldOn();
}
// Initialize values
int m = X.RowCount;
int n = X.ColumnCount;
int K = initial_centroids.RowCount;
Matrix <double> centroids = initial_centroids;
Matrix <double> previous_centroids = centroids;
Vector <double> idx = Vector <double> .Build.Dense(m);
// Run K-Means
for (int i = 0; i < max_iters; i++)
{
// Output progress
System.Console.WriteLine("K-Means iteration {0}/{1}...", i + 1, max_iters);
// For each example in X, assign it to the closest centroid
idx = FindClosestCentroids(X, centroids);
// Optionally, plot progress here
if (plot_progress)
{
PlotProgresskMeans(X, centroids, previous_centroids, idx, K, i);
previous_centroids = centroids;
//Pause();
}
// Given the memberships, compute new centroids
centroids = ComputeCentroids(X, idx, K);
}
if (plot_progress)
{
GnuPlot.HoldOff();
}
return(centroids, idx);
}
void PlotAverageSpeedAndSize()
{
GnuPlot.Set("xlabel 'Time [Seconds]'");
GnuPlot.Set("ylabel 'Avg. speed'");
GnuPlot.Set("y2label 'Avg. size'");
GnuPlot.Set("ytics nomirror");
GnuPlot.Set("y2tics");
GnuPlot.Set("tics out");
GnuPlot.Set(String.Format("yrange [{0}:{1}]", JumpmanAttributes.MIN_SPEED, JumpmanAttributes.MAX_SPEED));
GnuPlot.Set(String.Format("y2range [{0}:{1}]", JumpmanAttributes.MIN_SIZE, JumpmanAttributes.MAX_SIZE));
//GnuPlot.Set("autoscale y");
//GnuPlot.Set("autoscale y2");
GnuPlot.Set("key left top Left box 3");
GnuPlot.HoldOn();
GnuPlot.Plot(tracker.averageJumpmanAttrTmpPath, "using 1:5 title 'Speed' w lp axes x1y1");
GnuPlot.Plot(tracker.averageJumpmanAttrTmpPath, "using 1:6 title 'Size' w lp axes x1y2");
GnuPlot.HoldOff();
}
void PlotNutritionAndJumpmen()
{
GnuPlot.Set("xlabel 'Time [Seconds]'");
GnuPlot.Set("ylabel 'Total nutrition'");
GnuPlot.Set("y2label 'Amount of jumpmen'");
GnuPlot.Set("ytics nomirror");
GnuPlot.Set("y2tics");
GnuPlot.Set("tics out");
GnuPlot.Unset("yrange");
GnuPlot.Unset("y2range");
//GnuPlot.Set("autoscale y");
//GnuPlot.Set("autoscale y2");
GnuPlot.Set("key left top Left box 3");
GnuPlot.HoldOn();
GnuPlot.Plot(tracker.nutritionTmpPath, "using 1:2 title 'Nutrition' w l axes x1y1");
GnuPlot.Plot(tracker.jumpmanCountTmpPath, "using 1:2 title 'Jumpmen' w lp axes x1y2");
GnuPlot.HoldOff();
}
static void Main(string[] args)
{
if (!System.Console.IsOutputRedirected)
{
System.Console.Clear();
}
CultureInfo.CurrentCulture = CultureInfo.CreateSpecificCulture("en-US");
System.Console.WriteLine("Regularized Logistic Regression ex.2");
System.Console.WriteLine("====================================\n");
var M = Matrix <double> .Build;
var V = Vector <double> .Build;
// Load Data
// The first two columns contains the X values and the third column
// contains the label (y).
Matrix <double> data = DelimitedReader.Read <double>("data\\ex2data2.txt", false, ",", false);
Console.WriteLine(data);
Matrix <double> X = data.SubMatrix(0, data.RowCount, 0, 2);
Vector <double> y = data.Column(2);
System.Console.WriteLine("Features:\n");
System.Console.WriteLine(X);
System.Console.WriteLine("Label:\n");
System.Console.WriteLine(y);
PlotData(X, y);
Pause();
// =========== Part 1: Regularized Logistic Regression ============
// In this part, you are given a dataset with data points that are not
// linearly separable. However, you would still like to use logistic
// regression to classify the data points.
//
// To do so, you introduce more features to use -- in particular, you add
// polynomial features to our data matrix (similar to polynomial
// regression).
//
// Add Polynomial Features
// Note that mapFeature also adds a column of ones for us, so the intercept
// term is handled
X = MapFeature(X.Column(0), X.Column(1));
System.Console.WriteLine("Mapped features:\n");
System.Console.WriteLine(X);
Pause();
// Initialize fitting parameters
Vector <double> initial_theta = V.Dense(X.ColumnCount, 0.0);
// Set regularization parameter lambda to 1
double lambda = 1;
// Compute and display initial cost and gradient for regularized logistic
// regression
LogisticRegression lr = new LogisticRegression(X, y);
lr.Lambda = lambda;
double J = lr.Cost(initial_theta);
Vector <double> grad = lr.Gradient(initial_theta);
System.Console.WriteLine("Cost at initial theta (zeros): {0:f5}\n", J);
System.Console.WriteLine("Expected cost (approx): 0.693\n");
System.Console.WriteLine("Gradient at initial theta (zeros) - first five values only:\n");
System.Console.WriteLine(" {0:f5} \n", grad.SubVector(0, 5));
System.Console.WriteLine("Expected gradients (approx) - first five values only:\n");
System.Console.WriteLine(" 0.0085\n 0.0188\n 0.0001\n 0.0503\n 0.0115\n");
Pause();
// Compute and display cost and gradient
// with all-ones theta and lambda = 10
Vector <double> test_theta = V.Dense(X.ColumnCount, 1.0);
lr.Lambda = 10;
J = lr.Cost(test_theta);
grad = lr.Gradient(test_theta);
System.Console.WriteLine("\nCost at test theta (with lambda = 10): {0:f5}\n", J);
System.Console.WriteLine("Expected cost (approx): 3.16\n");
System.Console.WriteLine("Gradient at test theta - first five values only:\n");
System.Console.WriteLine(" {0:f5} \n", grad.SubVector(0, 5));
System.Console.WriteLine("Expected gradients (approx) - first five values only:\n");
System.Console.WriteLine(" 0.3460\n 0.1614\n 0.1948\n 0.2269\n 0.0922\n");
Pause();
//// ============= Part 2: Regularization and Accuracies =============
// Optional Exercise:
// In this part, you will get to try different values of lambda and
// see how regularization affects the decision coundart
//
// Try the following values of lambda (0, 1, 10, 100).
//
// How does the decision boundary change when you vary lambda? How does
// the training set accuracy vary?
//
// Initialize fitting parameters
initial_theta = V.Dense(X.ColumnCount, 0.0);
// Set regularization parameter lambda to 1 (you should vary this)
lambda = 1;
// Optimize
lr.Lambda = lambda;
var obj = ObjectiveFunction.Gradient(lr.Cost, lr.Gradient);
var solver = new BfgsMinimizer(1e-5, 1e-5, 1e-5, 400);
var result = solver.FindMinimum(obj, initial_theta);
// Plot Boundary
PlotDecisionBoundary(result.MinimizingPoint);
GnuPlot.HoldOff();
// Compute accuracy on our training set
Vector <double> pos = LogisticRegression.Predict(X, result.MinimizingPoint);
Func <double, double> map = delegate(double d){
if (d >= 0.5)
{
return(1);
}
else
{
return(0);
}
};
pos = pos.Map(map);
Vector <double> comp = V.Dense(y.Count);
for (int i = 0; i < y.Count; i++)
{
if (pos[i] == y[i])
{
comp[i] = 1;
}
else
{
comp[i] = 0;
}
}
double accurancy = comp.Mean() * 100;
System.Console.WriteLine("Train Accuracy: {0:f5}\n", accurancy);
System.Console.WriteLine("Expected accuracy (with lambda = 1): 83.1 (approx)\n");
Pause();
}
static void Main(string[] args)
{
if (!System.Console.IsOutputRedirected)
{
System.Console.Clear();
}
CultureInfo.CurrentCulture = CultureInfo.CreateSpecificCulture("en-US");
var M = Matrix <double> .Build;
var V = Vector <double> .Build;
//// =============== Part 1: Loading and Visualizing Data ================
// We start the exercise by first loading and visualizing the dataset.
// The following code will load the dataset into your environment and plot
// the data.
//
System.Console.WriteLine("Loading and Visualizing Data ...\n");
// Load from ex6data1:
// You will have X, y in your environment
Dictionary <string, Matrix <double> > ms = MatlabReader.ReadAll <double>("data\\ex6data1.mat");
Matrix <double> X = ms["X"]; // 51 X 2
Vector <double> y = ms["y"].Column(0); // 51 X 1
// Plot training data
GnuPlot.HoldOn();
PlotData(X, y);
Pause();
//// ==================== Part 2: Training Linear SVM ====================
// The following code will train a linear SVM on the dataset and plot the
// decision boundary learned.
//
System.Console.WriteLine("\nTraining Linear SVM ...\n");
// You should try to change the C value below and see how the decision
// boundary varies (e.g., try C = 1000)
double C = 1.0;
var linearKernel = KernelHelper.LinearKernel();
List <List <double> > libSvmData = ConvertToLibSvmFormat(X, y);
svm_problem prob = ProblemHelper.ReadProblem(libSvmData);
var svc = new C_SVC(prob, linearKernel, C);
PlotBoundary(X, svc);
GnuPlot.HoldOff();
System.Console.WriteLine();
Pause();
//// =============== Part 3: Implementing Gaussian Kernel ===============
// You will now implement the Gaussian kernel to use
// with the SVM. You should complete the code in gaussianKernel.m
//
System.Console.WriteLine("\nEvaluating the Gaussian Kernel ...\n");
double sigma = 2.0;
double sim = GaussianKernel(
V.DenseOfArray(new [] { 1.0, 2, 1 }),
V.DenseOfArray(new [] { 0.0, 4, -1 }),
sigma
);
System.Console.WriteLine("Gaussian Kernel between x1 = [1; 2; 1], x2 = [0; 4; -1], sigma = {0:f6} :\n\t{1:f6}\n(for sigma = 2, this value should be about 0.324652)\n", sigma, sim);
Pause();
//// =============== Part 4: Visualizing Dataset 2 ================
// The following code will load the next dataset into your environment and
// plot the data.
//
System.Console.WriteLine("Loading and Visualizing Data ...\n");
// Load from ex6data2:
// You will have X, y in your environment
ms = MatlabReader.ReadAll <double>("data\\ex6data2.mat");
X = ms["X"]; // 863 X 2
y = ms["y"].Column(0); // 863 X 1
// Plot training data
GnuPlot.HoldOn();
PlotData(X, y);
Pause();
//// ========== Part 5: Training SVM with RBF Kernel (Dataset 2) ==========
// After you have implemented the kernel, we can now use it to train the
// SVM classifier.
//
System.Console.WriteLine("\nTraining SVM with RBF Kernel (this may take 1 to 2 minutes) ...\n");
// SVM Parameters
C = 1;
sigma = 0.1;
double gamma = 1 / (2 * sigma * sigma);
var rbfKernel = KernelHelper.RadialBasisFunctionKernel(gamma);
libSvmData = ConvertToLibSvmFormat(X, y);
prob = ProblemHelper.ReadProblem(libSvmData);
svc = new C_SVC(prob, rbfKernel, C);
PlotBoundary(X, svc);
GnuPlot.HoldOff();
Pause();
double acc = svc.GetCrossValidationAccuracy(10);
System.Console.WriteLine("\nCross Validation Accuracy: {0:f6}\n", acc);
Pause();
//// =============== Part 6: Visualizing Dataset 3 ================
// The following code will load the next dataset into your environment and
// plot the data.
//
System.Console.WriteLine("Loading and Visualizing Data ...\n");
// Load from ex6data2:
// You will have X, y in your environment
ms = MatlabReader.ReadAll <double>("data\\ex6data3.mat");
Matrix <double> Xval;
Vector <double> yval;
X = ms["X"]; // 211 X 2
y = ms["y"].Column(0); // 211 X 1
Xval = ms["Xval"]; // 200 X 2
yval = ms["yval"].Column(0); // 200 X 1
// Plot training data
GnuPlot.HoldOn();
PlotData(X, y);
//// ========== Part 7: Training SVM with RBF Kernel (Dataset 3) ==========
// This is a different dataset that you can use to experiment with. Try
// different values of C and sigma here.
//
(C, sigma) = Dataset3Params(X, y, Xval, yval);
gamma = 1 / (2 * sigma * sigma);
rbfKernel = KernelHelper.RadialBasisFunctionKernel(gamma);
libSvmData = ConvertToLibSvmFormat(X, y);
prob = ProblemHelper.ReadProblem(libSvmData);
svc = new C_SVC(prob, rbfKernel, C);
PlotBoundary(X, svc);
GnuPlot.HoldOff();
Pause();
}
static void Main(string[] args)
{
if (!System.Console.IsOutputRedirected)
{
System.Console.Clear();
}
CultureInfo.CurrentCulture = CultureInfo.CreateSpecificCulture("en-US");
System.Console.WriteLine("Logistic Regression ex.2");
System.Console.WriteLine("========================\n");
var M = Matrix <double> .Build;
var V = Vector <double> .Build;
// Load Data
// The first two columns contains the exam scores and the third column
// contains the label.
Matrix <double> data = DelimitedReader.Read <double>("data\\ex2data1.txt", false, ",", false);
Console.WriteLine(data);
Matrix <double> X = data.SubMatrix(0, data.RowCount, 0, 2);
Vector <double> y = data.Column(2);
System.Console.WriteLine("Features:\n");
System.Console.WriteLine(X);
System.Console.WriteLine("Label:\n");
System.Console.WriteLine(y);
// ==================== Part 1: Plotting ====================
// We start the exercise by first plotting the data to understand the
// the problem we are working with.
System.Console.WriteLine("Plotting data with + indicating (y = 1) examples and o indicating (y = 0) examples.\n");
PlotData(X, y);
GnuPlot.HoldOff();
Pause();
// theta parameters
Vector <double> initial_theta = V.Dense(X.ColumnCount + 1);
// Add intercept term to X
X = X.InsertColumn(0, V.Dense(X.RowCount, 1));
// compute cost
LogisticRegression lr = new LogisticRegression(X, y);
double J = lr.Cost(initial_theta);
Vector <double> grad = lr.Gradient(initial_theta);
System.Console.WriteLine("Cost at initial theta (zeros): {0:f3}\n", J);
System.Console.WriteLine("Expected cost (approx): 0.693\n");
System.Console.WriteLine("Gradient at initial theta (zeros): \n");
System.Console.WriteLine(" {0:f4} \n", grad);
System.Console.WriteLine("Expected gradients (approx):\n -0.1000\n -12.0092\n -11.2628\n");
// Compute and display cost and gradient with non-zero theta
Vector <double> test_theta = V.DenseOfArray(new double[] { -24.0, 0.2, 0.2 });
J = lr.Cost(test_theta);
grad = lr.Gradient(test_theta);
System.Console.WriteLine("\nCost at test theta: {0:f3}\n", J);
System.Console.WriteLine("Expected cost (approx): 0.218\n");
System.Console.WriteLine("Gradient at test theta: \n");
System.Console.WriteLine(" {0:f3} \n", grad);
System.Console.WriteLine("Expected gradients (approx):\n 0.043\n 2.566\n 2.647\n");
Pause();
// ============= Part 3: Optimizing using fmin ================
// In this exercise, I will use fmin function to find the
// optimal parameters theta.
var obj = ObjectiveFunction.Gradient(lr.Cost, lr.Gradient);
var solver = new BfgsMinimizer(1e-5, 1e-5, 1e-5, 1000);
var result = solver.FindMinimum(obj, initial_theta);
System.Console.WriteLine("Cost at theta found by fmin: {0:f5} after {1} iterations\n", result.FunctionInfoAtMinimum.Value, result.Iterations);
System.Console.WriteLine("Expected cost (approx): 0.203\n");
System.Console.WriteLine("theta: \n");
System.Console.WriteLine(result.MinimizingPoint);
System.Console.WriteLine("Expected theta (approx):\n");
System.Console.WriteLine(" -25.161\n 0.206\n 0.201\n");
Pause();
PlotLinearBoundary(X, y, result.MinimizingPoint);
GnuPlot.HoldOff();
// ============== Part 4: Predict and Accuracies ==============
// After learning the parameters, you'll like to use it to predict the outcomes
// on unseen data. In this part, you will use the logistic regression model
// to predict the probability that a student with score 45 on exam 1 and
// score 85 on exam 2 will be admitted.
//
// Furthermore, you will compute the training and test set accuracies of
// our model.
//
// Your task is to complete the code in predict.m
// Predict probability for a student with score 45 on exam 1
// and score 85 on exam 2
double prob = LogisticRegression.Predict(V.DenseOfArray(new [] { 1.0, 45.0, 85.0 }), result.MinimizingPoint);
System.Console.WriteLine("For a student with scores 45 and 85, we predict an admission probability of {0:f5}\n", prob);
System.Console.WriteLine("Expected value: 0.775 +/- 0.002\n\n");
Pause();
// Compute accuracy on our training set
Vector <double> pos = LogisticRegression.Predict(X, result.MinimizingPoint);
Func <double, double> map = delegate(double d){
if (d >= 0.5)
{
return(1);
}
else
{
return(0);
}
};
pos = pos.Map(map);
Vector <double> comp = V.Dense(y.Count);
for (int i = 0; i < y.Count; i++)
{
if (pos[i] == y[i])
{
comp[i] = 1;
}
else
{
comp[i] = 0;
}
}
double accurancy = comp.Mean() * 100;
System.Console.WriteLine("Train Accuracy: {0:f5}\n", accurancy);
System.Console.WriteLine("Expected accuracy (approx): 89.0\n");
System.Console.WriteLine("\n");
}
static void Main(string[] args)
{
if (!System.Console.IsOutputRedirected)
{
System.Console.Clear();
}
CultureInfo.CurrentCulture = CultureInfo.CreateSpecificCulture("en-US");
System.Console.WriteLine("Regularized Linear Regression and Bias v.s. Variance ex.5");
System.Console.WriteLine("=========================================================\n");
var M = Matrix <double> .Build;
var V = Vector <double> .Build;
// =========== Part 1: Loading and Visualizing Data =============
// We start the exercise by first loading and visualizing the dataset.
// The following code will load the dataset into your environment and plot
// the data.
// Load Training Data
System.Console.WriteLine("Loading and Visualizing Data ...\n");
// Load from ex5data1:
// You will have X, y, Xval, yval, Xtest, ytest in your environment
Dictionary <string, Matrix <double> > ms = MatlabReader.ReadAll <double>("data\\ex5data1.mat");
Matrix <double> X = ms["X"];
Vector <double> y = ms["y"].Column(0);
Matrix <double> Xval = ms["Xval"];
Vector <double> yval = ms["yval"].Column(0);
Matrix <double> Xtest = ms["Xtest"];
Vector <double> ytest = ms["ytest"].Column(0);
// m = Number of examples
int m = X.RowCount;
GnuPlot.HoldOn();
PlotData(X.Column(0).ToArray(), y.ToArray());
Pause();
// =========== Part 2: Regularized Linear Regression Cost =============
// You should now implement the cost function for regularized linear
// regression.
Vector <double> theta = V.Dense(2, 1.0);
LinearRegression lr = new LinearRegression(X.InsertColumn(0, V.Dense(m, 1)), y, 1);
double J = lr.Cost(theta);
Vector <double> grad = lr.Gradient(theta);
System.Console.WriteLine("Cost at theta = [1 ; 1]: {0:f6} \n(this value should be about 303.993192)\n", J);
Pause();
// =========== Part 3: Regularized Linear Regression Gradient =============
// You should now implement the gradient for regularized linear
// regression.
System.Console.WriteLine("Gradient at theta = [1 ; 1]: [{0:f6}; {1:f6}] \n(this value should be about [-15.303016; 598.250744])\n", grad[0], grad[1]);
Pause();
// =========== Part 4: Train Linear Regression =============
// Once you have implemented the cost and gradient correctly, the
// trainLinearReg function will use your cost function to train
// regularized linear regression.
//
// Write Up Note: The data is non-linear, so this will not give a great
// fit.
//
// Train linear regression with lambda = 0
lr = new LinearRegression(X.InsertColumn(0, V.Dense(m, 1)), y, 0);
var result = lr.Train();
Vector <double> h = X.InsertColumn(0, V.Dense(m, 1)) * result.MinimizingPoint; // hypothesys
PlotLinearFit(X.Column(0).ToArray(), h.ToArray());
GnuPlot.HoldOff();
Pause();
// =========== Part 5: Learning Curve for Linear Regression =============
// Next, you should implement the learningCurve function.
//
// Write Up Note: Since the model is underfitting the data, we expect to
// see a graph with "high bias" -- Figure 3 in ex5.pdf
//
(Vector <double> error_train, Vector <double> error_val)res;
res = LearningCurve(X.InsertColumn(0, V.Dense(m, 1)), y, Xval.InsertColumn(0, V.Dense(Xval.RowCount, 1)), yval, 0);
PlotLinearLearningCurve(
Generate.LinearRange(1, 1, m),
res.error_train.ToArray(),
res.error_val.ToArray()
);
System.Console.WriteLine("# Training Examples\tTrain Error\tCross Validation Error\n");
for (int i = 0; i < m; i++)
{
System.Console.WriteLine("\t{0,2}\t\t{1:f6}\t{2:f6}", i, res.error_train[i], res.error_val[i]);
}
System.Console.WriteLine();
Pause();
// =========== Part 6: Feature Mapping for Polynomial Regression =============
// One solution to this is to use polynomial regression. You should now
// complete polyFeatures to map each example into its powers
//
int p = 8;
// Map X onto Polynomial Features and Normalize
Matrix <double> X_poly = MapPolyFeatures(X, p);
// normalize
var norm = FeatureNormalize(X_poly);
X_poly = norm.X_norm;
// add one's
X_poly = X_poly.InsertColumn(0, V.Dense(X_poly.RowCount, 1));
// Map X_poly_test and normalize (using mu and sigma)
Matrix <double> X_poly_test = MapPolyFeatures(Xtest, p);
for (int i = 0; i < X_poly_test.ColumnCount; i++)
{
Vector <double> v = X_poly_test.Column(i);
v = v - norm.mu[0, i];
v = v / norm.sigma[0, i];
X_poly_test.SetColumn(i, v);
}
// add one's
X_poly_test = X_poly_test.InsertColumn(0, V.Dense(X_poly_test.RowCount, 1));
// Map X_poly_val and normalize (using mu and sigma)
Matrix <double> X_poly_val = MapPolyFeatures(Xval, p);
for (int i = 0; i < X_poly_val.ColumnCount; i++)
{
Vector <double> v = X_poly_val.Column(i);
v = v - norm.mu[0, i];
v = v / norm.sigma[0, i];
X_poly_val.SetColumn(i, v);
}
// add one's
X_poly_val = X_poly_val.InsertColumn(0, V.Dense(X_poly_val.RowCount, 1));
System.Console.WriteLine("Normalized Training Example 1:\n");
System.Console.WriteLine(X_poly.Row(0));
Pause();
// =========== Part 7: Learning Curve for Polynomial Regression =============
// Now, you will get to experiment with polynomial regression with multiple
// values of lambda. The code below runs polynomial regression with
// lambda = 0. You should try running the code with different values of
// lambda to see how the fit and learning curve change.
//
double lambda = 0;
lr = new LinearRegression(X_poly, y, lambda);
var minRes = lr.Train();
GnuPlot.HoldOn();
GnuPlot.Set("terminal wxt 1");
PlotData(X.Column(0).ToArray(), y.ToArray());
PlotFit(X.Column(0).Minimum(), X.Column(0).Maximum(), norm.mu.Row(0), norm.sigma.Row(0), minRes.MinimizingPoint, p, lambda);
GnuPlot.HoldOff();
// learning curve
GnuPlot.Set("terminal wxt 2");
res = LearningCurve(X_poly, y, X_poly_val, yval, lambda);
PlotLinearLearningCurve(
Generate.LinearRange(1, 1, m),
res.error_train.ToArray(),
res.error_val.ToArray()
);
System.Console.WriteLine("# Training Examples\tTrain Error\tCross Validation Error\n");
for (int i = 0; i < m; i++)
{
System.Console.WriteLine("\t{0,2}\t\t{1:f6}\t{2:f6}", i, res.error_train[i], res.error_val[i]);
}
System.Console.WriteLine();
Pause();
// =========== Part 8: Validation for Selecting Lambda =============
// You will now implement validationCurve to test various values of
// lambda on a validation set. You will then use this to select the
// "best" lambda value.
//
var resVal = ValidationCurve(X_poly, y, X_poly_val, yval);
PlotValidationCurve(resVal.Lamda_vec.ToArray(), resVal.error_train.ToArray(), resVal.error_val.ToArray());
System.Console.WriteLine("# Lambda\tTrain Error\tCross Validation Error\n");
for (int i = 0; i < resVal.error_train.Count; i++)
{
System.Console.WriteLine("\t{0:f4}\t\t{1:f6}\t{2:f6}", resVal.Lamda_vec[i], resVal.error_train[i], resVal.error_val[i]);
}
System.Console.WriteLine();
Pause();
// =========== Part 9: Compute test set error =============
// Compute the test error using the best value of λ you
// found
lambda = 3.0;
lr = new LinearRegression(X_poly, y, lambda);
minRes = lr.Train();
theta = minRes.MinimizingPoint;
h = X_poly_test * theta;
m = X_poly.RowCount;
System.Console.WriteLine("Evaluating test-set:\n");
System.Console.WriteLine("# \tHypothesis\tExpected\tError\n");
for (int i = 0; i < m; i++)
{
System.Console.WriteLine("{0,3}\t{1:f6}\t{2:f6}\t{3:f6}", i + 1, h[i], ytest[i], h[i] - ytest[i]);
}
double mae = (h - ytest).L1Norm(); // Mean Absolute Error
double mse = (h - ytest).L2Norm(); // Mean Squared Error
double rmse = Math.Sqrt(mse); // Root Mean Squared Error
System.Console.WriteLine("\nMAE on test set: {0:F6}", mae);
System.Console.WriteLine("MSE on test set: {0:F6}", mse);
System.Console.WriteLine("RMSE on test set: {0:F6}\n", rmse);
Pause();
GnuPlot.HoldOn();
GnuPlot.Set("terminal wxt 3");
PlotData(Xtest.Column(0).ToArray(), ytest.ToArray());
PlotFit(Xtest.Column(0).Minimum(), Xtest.Column(0).Maximum(), norm.mu.Row(0), norm.sigma.Row(0), theta, p, lambda);
GnuPlot.HoldOff();
Pause();
}
static void Main(string[] args)
{
if (!System.Console.IsOutputRedirected)
{
System.Console.Clear();
}
CultureInfo.CurrentCulture = CultureInfo.CreateSpecificCulture("en-US");
System.Console.WriteLine("Linear Regression ex.1");
System.Console.WriteLine("======================\n");
// ==================== Part 1: Basic Function ====================
var M = Matrix <double> .Build;
var V = Vector <double> .Build;
// load data
Matrix <double> data = DelimitedReader.Read <double>("data\\ex1data1.txt", false, ",", false);
Console.WriteLine(data);
Matrix <double> X = data.Column(0).ToColumnMatrix();
Matrix <double> y = data.Column(1).ToColumnMatrix();
int m = X.RowCount;
// ======================= Part 2: Plotting =======================
System.Console.WriteLine("Plotting data....");
GnuPlot.HoldOn();
PlotData(X.Column(0).ToArray(), y.Column(0).ToArray());
Pause();
// =================== Part 3: Cost and Gradient descent ===================
System.Console.WriteLine("Cost and Gradient descent....");
// Add a column of ones to X
X = X.InsertColumn(0, V.Dense(m, 1));
System.Console.WriteLine(X);
// initialize fitting parameters
Matrix <double> theta = M.Dense(2, 1);
double J = ComputeCost(X, y, theta);
System.Console.WriteLine("With theta = [0 ; 0]\nCost computed = {0:f}\n", J);
System.Console.WriteLine("Expected cost value (approx) 32.07\n");
// initialize fitting parameters
theta[0, 0] = -1; theta[1, 0] = 2;
J = ComputeCost(X, y, theta);
System.Console.WriteLine("With theta = [-1 ; 2]\nCost computed = {0:f}\n", J);
System.Console.WriteLine("Expected cost value (approx) 54.24\n");
// run gradient descent
System.Console.WriteLine("\nRunning Gradient Descent ...\n");
int iterations = 1500;
double alpha = 0.01;
theta = M.Dense(2, 1);
(Matrix <double> theta, Matrix <double> J_history)res;
res = GradientDescent(X, y, theta, alpha, iterations);
theta = res.theta;
// print theta to screen
System.Console.WriteLine("Theta found by gradient descent:\n");
System.Console.WriteLine(theta);
System.Console.WriteLine("Expected theta values (approx)\n");
System.Console.WriteLine(" -3.6303\n 1.1664\n\n");
Matrix <double> h = X * theta; // hypothesys
Matrix <double> x = X.RemoveColumn(0); // remove x0
PlotLinearFit(x.Column(0).ToArray(), h.Column(0).ToArray());
GnuPlot.HoldOff();
var predict1 = M.DenseOfArray(new double[, ] {
{ 1, 3.5 }
}) * theta;
System.Console.WriteLine("For population = 35,000, we predict a profit of {0:F4}", predict1[0, 0] * 10000);
var predict2 = M.DenseOfArray(new double[, ] {
{ 1, 7 }
}) * theta;
System.Console.WriteLine("For population = 70,000, we predict a profit of {0:F4}", predict2[0, 0] * 10000);
Pause();
// ============= Part 4: Visualizing J(theta_0, theta_1) =============
System.Console.WriteLine("Visualizing J(theta_0, theta_1) ...\n");
PlotJ(X, y, theta);
Pause();
}
static void Main(string[] args)
{
if (!System.Console.IsOutputRedirected)
{
System.Console.Clear();
}
CultureInfo.CurrentCulture = CultureInfo.CreateSpecificCulture("en-US");
var M = Matrix <double> .Build;
var V = Vector <double> .Build;
System.Console.WriteLine("Linear Regression ex.1 multiple variables");
System.Console.WriteLine("=========================================\n");
// load data
System.Console.WriteLine("Loading data ...\n");
Matrix <double> data = DelimitedReader.Read <double>("data\\ex1data2.txt", false, ",", false);
Matrix <double> X = data.SubMatrix(0, data.RowCount, 0, 2);
Matrix <double> y = data.SubMatrix(0, data.RowCount, 2, 1);
int m = X.RowCount;
// Print out some data points
System.Console.WriteLine("First 10 examples from the dataset: \n");
var temp = M.DenseOfMatrixArray(new [, ]
{
{
X.SubMatrix(0, 10, 0, 2), y.SubMatrix(0, 10, 0, 1)
}
}
);
Console.WriteLine(temp);
// Scale features and set them to zero mean
System.Console.WriteLine("Normalizing Features ...\n");
(Matrix <double> X_norm, Matrix <double> mu, Matrix <double> sigma)norm_res;
norm_res = FeatureNormalize(X);
System.Console.WriteLine($"mu: {norm_res.mu}");
System.Console.WriteLine($"sigma: {norm_res.sigma}");
System.Console.WriteLine($"X_norm: {norm_res.X_norm}");
// Add intercept term to X
X = norm_res.X_norm;
X = X.InsertColumn(0, V.Dense(X.RowCount, 1));
// Running gradient descent ...
System.Console.WriteLine("Running gradient descent ...\n");
// Choose some alpha value
double alpha = 0.01;
int num_iters = 50;
GnuPlot.HoldOn();
Matrix <double> theta = M.Dense(3, 1);
(Matrix <double> theta, Matrix <double> J_history)res_grad1 = GradientDescentMulti(X, y, theta, alpha, num_iters);
PlotJ(res_grad1.J_history, "{/Symbol a}=" + alpha, "blue");
theta = M.Dense(3, 1);
alpha = 0.03;
(Matrix <double> theta, Matrix <double> J_history)res_grad2 = GradientDescentMulti(X, y, theta, alpha, num_iters);
PlotJ(res_grad2.J_history, "{/Symbol a}=" + alpha, "red");
theta = M.Dense(3, 1);
alpha = 0.1;
(Matrix <double> theta, Matrix <double> J_history)res_grad3 = GradientDescentMulti(X, y, theta, alpha, num_iters);
PlotJ(res_grad3.J_history, "{/Symbol a}=" + alpha, "black");
theta = M.Dense(3, 1);
alpha = 0.3;
(Matrix <double> theta, Matrix <double> J_history)res_grad4 = GradientDescentMulti(X, y, theta, alpha, num_iters);
PlotJ(res_grad4.J_history, "{/Symbol a}=" + alpha, "green");
GnuPlot.HoldOff();
// Display gradient descent's result
System.Console.WriteLine("Theta computed from gradient descent: \n");
System.Console.WriteLine(res_grad4.theta);
System.Console.WriteLine("\n");
// Estimate the price of a 1650 sq-ft, 3 br house
// Recall that the first column of X is all-ones. Thus, it does
// not need to be normalized.
double x1 = 1650;
double x2 = 3;
x1 = (x1 - norm_res.mu[0, 0]) / norm_res.sigma[0, 0];
x2 = (x2 - norm_res.mu[0, 1]) / norm_res.sigma[0, 1];
Matrix <double> K = Matrix <double> .Build.DenseOfRowArrays(new double[] { 1, x1, x2 });
System.Console.WriteLine("K is:");
System.Console.WriteLine(K);
double price = (K * res_grad4.theta)[0, 0];
System.Console.WriteLine("Predicted price of a 1650 sq-ft, 3 br house (using gradient descent):\n ${0:f}\n", price);
Pause();
// =================================================================
// NORMAL EQUATION
// =================================================================
System.Console.WriteLine("Solving with normal equations...\n");
// load data
System.Console.WriteLine("Loading data ...\n");
data = DelimitedReader.Read <double>("data\\ex1data2.txt", false, ",", false);
X = data.SubMatrix(0, data.RowCount, 0, 2);
y = data.SubMatrix(0, data.RowCount, 2, 1);
m = X.RowCount;
// Add intercept term to X
X = X.InsertColumn(0, V.Dense(X.RowCount, 1));
// Calculate the parameters from the normal equation
theta = normalEqn(X, y);
System.Console.WriteLine("Theta computed from the normal equations: \n");
System.Console.WriteLine(theta);
System.Console.WriteLine("\n");
K = Matrix <double> .Build.DenseOfRowArrays(new double[] { 1, 1650, 3 });
price = (K * theta)[0, 0];
System.Console.WriteLine("Predicted price of a 1650 sq-ft, 3 br house (using normal equation):\n ${0:f}\n", price);
Pause();
}
