/// <summary>
/// Find vector x that minimizes the function f(x) using the Broyden–Fletcher–Goldfarb–Shanno (BFGS) algorithm.
/// For more options and diagnostics consider to use <see cref="BfgsMinimizer"/> directly.
/// An alternative routine using conjugate gradients (CG) is available in <see cref="ConjugateGradientMinimizer"/>.
/// </summary>
public static Vector <double> OfFunctionGradient(Func <Vector <double>, Tuple <double, Vector <double> > > functionGradient, Vector <double> initialGuess, double gradientTolerance = 1e-5, double parameterTolerance = 1e-5, double functionProgressTolerance = 1e-5, int maxIterations = 1000)
{
var objective = ObjectiveFunction.Gradient(functionGradient);
var algorithm = new BfgsMinimizer(gradientTolerance, parameterTolerance, functionProgressTolerance, maxIterations);
var result = algorithm.FindMinimum(objective, initialGuess);
return(result.MinimizingPoint);
}
/// <summary>
/// Find vector x that minimizes the function f(x), constrained within bounds, using the Broyden–Fletcher–Goldfarb–Shanno Bounded (BFGS-B) algorithm.
/// For more options and diagnostics consider to use <see cref="BfgsBMinimizer"/> directly.
/// </summary>
public static Vector <double> OfFunctionGradientConstrained(Func <Vector <double>, double> function, Func <Vector <double>, Vector <double> > gradient, Vector <double> lowerBound, Vector <double> upperBound, Vector <double> initialGuess, double gradientTolerance = 1e-5, double parameterTolerance = 1e-5, double functionProgressTolerance = 1e-5, int maxIterations = 1000)
{
var objective = ObjectiveFunction.Gradient(function, gradient);
var algorithm = new BfgsBMinimizer(gradientTolerance, parameterTolerance, functionProgressTolerance, maxIterations);
var result = algorithm.FindMinimum(objective, lowerBound, upperBound, initialGuess);
return(result.MinimizingPoint);
}
public void FindMinimum_BigRosenbrock_Hard()
{
var obj = ObjectiveFunction.Gradient(BigRosenbrockFunction.Value, BigRosenbrockFunction.Gradient);
var solver = new LimitedMemoryBfgsMinimizer(1e-5, 1e-5, 1e-5, 5, 1000);
var result = solver.FindMinimum(obj, new DenseVector(new[] { -1.2 * 100.0, 1.0 * 100.0 }));
Assert.That(Math.Abs(result.MinimizingPoint[0] - BigRosenbrockFunction.Minimum[0]), Is.LessThan(1e-3));
Assert.That(Math.Abs(result.MinimizingPoint[1] - BigRosenbrockFunction.Minimum[1]), Is.LessThan(1e-3));
}
public void FindMinimum_Rosenbrock_Overton()
{
var obj = ObjectiveFunction.Gradient(RosenbrockFunction.Value, RosenbrockFunction.Gradient);
var solver = new LimitedMemoryBfgsMinimizer(1e-5, 1e-5, 1e-5, 5, 100);
var result = solver.FindMinimum(obj, new DenseVector(new[] { -0.9, -0.5 }));
Assert.That(Math.Abs(result.MinimizingPoint[0] - RosenbrockFunction.Minimum[0]), Is.LessThan(1e-3));
Assert.That(Math.Abs(result.MinimizingPoint[1] - RosenbrockFunction.Minimum[1]), Is.LessThan(1e-3));
}
public void FindMinimum_Rosenbrock_Hard()
{
var obj = ObjectiveFunction.Gradient(RosenbrockFunction.Value, RosenbrockFunction.Gradient);
var solver = new ConjugateGradientMinimizer(1e-5, 1000);
var result = solver.FindMinimum(obj, new DenseVector(new[] { -1.2, 1.0 }));
Assert.That(Math.Abs(result.MinimizingPoint[0] - 1.0), Is.LessThan(1e-3));
Assert.That(Math.Abs(result.MinimizingPoint[1] - 1.0), Is.LessThan(1e-3));
}
public MinimizationResult Train()
{
Vector <double> theta = Vector <double> .Build.Dense(X.ColumnCount);
LinearRegression lr = new LinearRegression(this.X, this.y, this.Lambda);
var obj = ObjectiveFunction.Gradient(lr.Cost, lr.Gradient);
var solver = new BfgsMinimizer(1e-5, 1e-5, 1e-5, 200);
MinimizationResult result = solver.FindMinimum(obj, theta);
return(result);
}
public void OptimizeLambda(LLNAModel model, CorpusDocument doc)
{
int i = 0, n = model.K - 1, iter = 0;
double fOld = 0;
var bundle = new Bundle(model.K, doc, model, this);
var x = Vector <double> .Build.Dense(model.K - 1);
for (i = 0; i < model.K - 1; i++)
{
x[i] = Lambda[i];
}
/*
* var iterator = new MultidimensionalIterator(n);
* var fdf = new MultidimensionalFunctionFDF
* {
* F = (vector) =>
* {
* return ComputeFunction(vector, bundle);
* },
* DF = (vector) =>
* {
* return ComputeGradient(vector, bundle);
* },
* N = n
* };
* var multidimensionalMinimizer = new MultidimensionalMinimizer(n, iterator, fdf);
* multidimensionalMinimizer.Set(x, 0.01, 1e-3);
* do
* {
* iter++;
* fOld = multidimensionalMinimizer.F;
* multidimensionalMinimizer.Iterate();
* }
* while (true);
*/
var obj = ObjectiveFunction.Gradient((vector) => {
return(ComputeFunction(vector, bundle));
}, (vector) =>
{
return(ComputeGradient(vector, bundle));
});
var solver = new ConjugateGradientMinimizer(1e-8, 5000);
var result = solver.FindMinimum(obj, x);
for (i = 0; i < model.K - 1; i++)
{
Lambda[i] = result.MinimizingPoint[i];
}
Lambda[i] = 0;
}
public void FindMinimum_Rosenbrock_Easy_OneBoundary()
{
var obj = ObjectiveFunction.Gradient(RosenbrockFunction.Value, RosenbrockFunction.Gradient);
var solver = new BfgsBMinimizer(1e-5, 1e-5, 1e-5, maximumIterations: 1000);
var lowerBound = new DenseVector(new[] { 1.0, -5.0 });
var upperBound = new DenseVector(new[] { 5.0, 5.0 });
var initialGuess = new DenseVector(new[] { 1.2, 1.2 });
var result = solver.FindMinimum(obj, lowerBound, upperBound, initialGuess);
Assert.That(Math.Abs(result.MinimizingPoint[0] - RosenbrockFunction.Minimum[0]), Is.LessThan(1e-3));
Assert.That(Math.Abs(result.MinimizingPoint[1] - RosenbrockFunction.Minimum[1]), Is.LessThan(1e-3));
}
public void FindMinimum_Rosenbrock_MinimumLesserOrEqualToUpperBoundary()
{
var obj = ObjectiveFunction.Gradient(RosenbrockFunction.Value, RosenbrockFunction.Gradient);
var solver = new BfgsBMinimizer(1e-5, 1e-5, 1e-5, maximumIterations: 1000);
var lowerBound = new DenseVector(new[] { -2.0, -2.0 });
var upperBound = new DenseVector(new[] { 0.5, 0.5 });
var initialGuess = new DenseVector(new[] { -0.9, -0.5 });
var result = solver.FindMinimum(obj, lowerBound, upperBound, initialGuess);
Assert.LessOrEqual(result.MinimizingPoint[0], upperBound[0]);
Assert.LessOrEqual(result.MinimizingPoint[1], upperBound[1]);
}
public void FindMinimum_Quadratic_TwoBoundaries()
{
var obj = ObjectiveFunction.Gradient(
x => x[0] * x[0] + x[1] * x[1],
x => new DenseVector(new[] { 2 * x[0], 2 * x[1] })
);
var solver = new BfgsBMinimizer(1e-5, 1e-5, 1e-5, maximumIterations: 1000);
var lowerBound = new DenseVector(new[] { 1.0, 1.0 });
var upperBound = new DenseVector(new[] { 2.0, 2.0 });
var initialGuess = new DenseVector(new[] { 1.5, 1.5 });
var result = solver.FindMinimum(obj, lowerBound, upperBound, initialGuess);
Assert.That(Math.Abs(result.MinimizingPoint[0] - 1.0), Is.LessThan(1e-3));
Assert.That(Math.Abs(result.MinimizingPoint[1] - 1.0), Is.LessThan(1e-3));
}
/// <summary>
/// 运用梯度下降法解决问题
/// </summary>
/// <returns></returns>
public void LeastSquareSolver()
{
var len = _xArr.Length;
var xMat = new DenseMatrix(2, len);
var yMat = new DenseMatrix(1, len);
for (var i = 0; i < len; i++)
{
xMat[0, i] = 1;
xMat[1, i] = _xArr[i];
}
for (var i = 0; i < len; i++)
{
yMat[0, i] = _yArr[i];
}
Vector <double> dLeastSquareLoss(Vector <double> theta)
{
/*
*
*/
var theta0 = 0.0;
var theta1 = 0.0;
var sum = 0.0;
//输入猜测的theta，输出当前点的斜率
for (var i = 0; i < len; i++)
{
var y_pred = theta[0] + theta[1] * _xArr[i];
theta0 += y_pred - _yArr[i];
theta1 += (y_pred - _yArr[i]) * _xArr[i];
}
return(new DenseVector(new[] { theta0 / len, theta1 / len }));
}
var solver = new BfgsMinimizer(1e-8, 1e-8, 1e-8, 10000);
var f = new Func <Vector <double>, double>(LeastSquareLoss);
var g = new Func <Vector <double>, Vector <double> >(dLeastSquareLoss);
var obj = ObjectiveFunction.Gradient(f, g);
var r1 = solver.FindMinimum(obj, new DenseVector(new [] { 0.0, 0.0 }));
Console.WriteLine(r1.MinimizingPoint);
}
static void Main(string[] args)
{
if (!System.Console.IsOutputRedirected)
{
System.Console.Clear();
}
CultureInfo.CurrentCulture = CultureInfo.CreateSpecificCulture("en-US");
System.Console.WriteLine("Regularized Logistic Regression ex.2");
System.Console.WriteLine("====================================\n");
var M = Matrix <double> .Build;
var V = Vector <double> .Build;
// Load Data
// The first two columns contains the X values and the third column
// contains the label (y).
Matrix <double> data = DelimitedReader.Read <double>("data\\ex2data2.txt", false, ",", false);
Console.WriteLine(data);
Matrix <double> X = data.SubMatrix(0, data.RowCount, 0, 2);
Vector <double> y = data.Column(2);
System.Console.WriteLine("Features:\n");
System.Console.WriteLine(X);
System.Console.WriteLine("Label:\n");
System.Console.WriteLine(y);
PlotData(X, y);
Pause();
// =========== Part 1: Regularized Logistic Regression ============
// In this part, you are given a dataset with data points that are not
// linearly separable. However, you would still like to use logistic
// regression to classify the data points.
//
// To do so, you introduce more features to use -- in particular, you add
// polynomial features to our data matrix (similar to polynomial
// regression).
//
// Add Polynomial Features
// Note that mapFeature also adds a column of ones for us, so the intercept
// term is handled
X = MapFeature(X.Column(0), X.Column(1));
System.Console.WriteLine("Mapped features:\n");
System.Console.WriteLine(X);
Pause();
// Initialize fitting parameters
Vector <double> initial_theta = V.Dense(X.ColumnCount, 0.0);
// Set regularization parameter lambda to 1
double lambda = 1;
// Compute and display initial cost and gradient for regularized logistic
// regression
LogisticRegression lr = new LogisticRegression(X, y);
lr.Lambda = lambda;
double J = lr.Cost(initial_theta);
Vector <double> grad = lr.Gradient(initial_theta);
System.Console.WriteLine("Cost at initial theta (zeros): {0:f5}\n", J);
System.Console.WriteLine("Expected cost (approx): 0.693\n");
System.Console.WriteLine("Gradient at initial theta (zeros) - first five values only:\n");
System.Console.WriteLine(" {0:f5} \n", grad.SubVector(0, 5));
System.Console.WriteLine("Expected gradients (approx) - first five values only:\n");
System.Console.WriteLine(" 0.0085\n 0.0188\n 0.0001\n 0.0503\n 0.0115\n");
Pause();
// Compute and display cost and gradient
// with all-ones theta and lambda = 10
Vector <double> test_theta = V.Dense(X.ColumnCount, 1.0);
lr.Lambda = 10;
J = lr.Cost(test_theta);
grad = lr.Gradient(test_theta);
System.Console.WriteLine("\nCost at test theta (with lambda = 10): {0:f5}\n", J);
System.Console.WriteLine("Expected cost (approx): 3.16\n");
System.Console.WriteLine("Gradient at test theta - first five values only:\n");
System.Console.WriteLine(" {0:f5} \n", grad.SubVector(0, 5));
System.Console.WriteLine("Expected gradients (approx) - first five values only:\n");
System.Console.WriteLine(" 0.3460\n 0.1614\n 0.1948\n 0.2269\n 0.0922\n");
Pause();
//// ============= Part 2: Regularization and Accuracies =============
// Optional Exercise:
// In this part, you will get to try different values of lambda and
// see how regularization affects the decision coundart
//
// Try the following values of lambda (0, 1, 10, 100).
//
// How does the decision boundary change when you vary lambda? How does
// the training set accuracy vary?
//
// Initialize fitting parameters
initial_theta = V.Dense(X.ColumnCount, 0.0);
// Set regularization parameter lambda to 1 (you should vary this)
lambda = 1;
// Optimize
lr.Lambda = lambda;
var obj = ObjectiveFunction.Gradient(lr.Cost, lr.Gradient);
var solver = new BfgsMinimizer(1e-5, 1e-5, 1e-5, 400);
var result = solver.FindMinimum(obj, initial_theta);
// Plot Boundary
PlotDecisionBoundary(result.MinimizingPoint);
GnuPlot.HoldOff();
// Compute accuracy on our training set
Vector <double> pos = LogisticRegression.Predict(X, result.MinimizingPoint);
Func <double, double> map = delegate(double d){
if (d >= 0.5)
{
return(1);
}
else
{
return(0);
}
};
pos = pos.Map(map);
Vector <double> comp = V.Dense(y.Count);
for (int i = 0; i < y.Count; i++)
{
if (pos[i] == y[i])
{
comp[i] = 1;
}
else
{
comp[i] = 0;
}
}
double accurancy = comp.Mean() * 100;
System.Console.WriteLine("Train Accuracy: {0:f5}\n", accurancy);
System.Console.WriteLine("Expected accuracy (with lambda = 1): 83.1 (approx)\n");
Pause();
}
static void Main(string[] args)
{
if (!System.Console.IsOutputRedirected)
{
System.Console.Clear();
}
CultureInfo.CurrentCulture = CultureInfo.CreateSpecificCulture("en-US");
System.Console.WriteLine("Multi-class Classification and Neural Networks ex.4");
System.Console.WriteLine("===================================================\n");
var M = Matrix <double> .Build;
var V = Vector <double> .Build;
// Setup the parameters you will use for this exercise
int input_layer_size = 400; // 20x20 Input Images of Digits
int hidden_layer_size = 25; // 25 hidden units
int num_labels = 10; // 10 labels, from 1 to 10
// (note that we have mapped "0" to label 10)
// =========== Part 1: Loading and Visualizing Data =============
// We start the exercise by first loading and visualizing the dataset.
// You will be working with a dataset that contains handwritten digits.
//
// read all matrices of a file by name into a dictionary
Dictionary <string, Matrix <double> > ms = MatlabReader.ReadAll <double>("data\\ex3data1.mat");
Matrix <double> X = ms["X"];
Vector <double> y = ms["y"].Column(0);
// get a casual sequence of 100 int numbers
var srs = new MathNet.Numerics.Random.SystemRandomSource();
var seq = srs.NextInt32Sequence(0, 5000).Take(100).ToList();
// Randomly select 100 data points to display
Vector <double>[] sel = new Vector <double> [100];
int idx = 0;
Vector <double> v = V.Dense(400);
foreach (int i in seq)
{
sel[idx++] = X.Row(i);
}
// display
DisplayData(sel);
Pause();
// ================ Part 2: Loading Parameters ================
// In this part of the exercise, we load some pre-initialized
// neural network parameters.
System.Console.WriteLine("\nLoading Saved Neural Network Parameters ...\n");
// read all matrices of a file by name into a dictionary
Dictionary <string, Matrix <double> > mr = MatlabReader.ReadAll <double>("data\\ex3weights.mat");
Matrix <double> theta1 = mr["Theta1"]; // 25 X 401
Matrix <double> theta2 = mr["Theta2"]; // 10 X 26
// Unroll parameters
Vector <double> nn_params = NeuralNetwork.UnrollParameters(theta1, theta2);
Pause();
// ================ Part 3: Compute Cost (Feedforward) ================
// To the neural network, you should first start by implementing the
// feedforward part of the neural network that returns the cost only. You
// should complete the code in nnCostFunction.m to return cost. After
// implementing the feedforward to compute the cost, you can verify that
// your implementation is correct by verifying that you get the same cost
// as us for the fixed debugging parameters.
//
// We suggest implementing the feedforward cost *without* regularization
// first so that it will be easier for you to debug. Later, in part 4, you
// will get to implement the regularized cost.
System.Console.WriteLine("\nFeedforward Using Neural Network ...\n");
// Weight regularization parameter (we set this to 0 here).
NeuralNetwork nn = new NeuralNetwork(X, y, input_layer_size, hidden_layer_size, num_labels);
nn.Lambda = 0.0;
double J = nn.Cost(nn_params);
System.Console.WriteLine("Cost at parameters (loaded from ex4weights): {0:f6}\n(this value should be about 0.287629)\n", J);
Pause();
// =============== Part 4: Implement Regularization ===============
// Once your cost function implementation is correct, you should now
// continue to implement the regularization with the cost.
//
System.Console.WriteLine("\nChecking Cost Function (w/ Regularization) ... \n");
// Weight regularization parameter (we set this to 1 here).
nn.Lambda = 1.0;
J = nn.Cost(nn_params);
System.Console.WriteLine("Cost at parameters (loaded from ex4weights): {0:f6} \n(this value should be about 0.383770)\n", J);
Pause();
// ================ Part 5: Sigmoid Gradient ================
// Before you start implementing the neural network, you will first
// implement the gradient for the sigmoid function. You should complete the
// code in the sigmoidGradient.m file.
System.Console.WriteLine("\nEvaluating sigmoid gradient...\n");
var g = nn.SigmoidGradient(V.DenseOfArray(new[] { -1.0, -0.5, 0, 0.5, 1 }));
System.Console.WriteLine("Sigmoid gradient evaluated at [-1 -0.5 0 0.5 1]:\n ");
System.Console.WriteLine("{0:f5} ", g);
System.Console.WriteLine("\n\n");
Pause();
// ================ Part 6: Initializing Pameters ================
// In this part of the exercise, you will be starting to implment a two
// layer neural network that classifies digits. You will start by
// implementing a function to initialize the weights of the neural network
// (randInitializeWeights.m)
System.Console.WriteLine("\nInitializing Neural Network Parameters ...\n");
Matrix <double> initial_Theta1 = RandInitializeWeights(input_layer_size + 1, hidden_layer_size);
Matrix <double> initial_Theta2 = RandInitializeWeights(hidden_layer_size + 1, num_labels);
// Unroll parameters
Vector <double> initial_nn_params = NeuralNetwork.UnrollParameters(initial_Theta1, initial_Theta2);
Pause();
// =============== Part 7: Implement Backpropagation ===============
// Once your cost matches up with ours, you should proceed to implement the
// backpropagation algorithm for the neural network. You should add to the
// code you've written in nnCostFunction.m to return the partial
// derivatives of the parameters.
System.Console.WriteLine("\nChecking Backpropagation... \n");
CheckGradient(0);
Pause();
// =============== Part 8: Implement Regularization ===============
// Once your backpropagation implementation is correct, you should now
// continue to implement the regularization with the cost and gradient.
System.Console.WriteLine("\nChecking Backpropagation (w/ Regularization) ... \n");
// Check gradients by running checkNNGradients
double lambda = 3;
CheckGradient(lambda);
// Also output the costFunction debugging values
nn.Lambda = lambda;
double debug_J = nn.Cost(nn_params);
System.Console.WriteLine("\n\nCost at (fixed) debugging parameters (w/ lambda = {0:f1}): {1:f6} " +
"\n(for lambda = 3, this value should be about 0.576051)\n\n", lambda, debug_J);
Pause();
// =================== Part 8: Training NN ===================
// You have now implemented all the code necessary to train a neural
// network. To train your neural network, we will now use "fmincg", which
// is a function which works similarly to "fminunc". Recall that these
// advanced optimizers are able to train our cost functions efficiently as
// long as we provide them with the gradient computations.
System.Console.WriteLine("\nTraining Neural Network... \n");
// After you have completed the assignment, change the MaxIter to a larger
// value to see how more training helps.
int maxIter = 40;
// You should also try different values of lambda
lambda = 1;
nn.Lambda = lambda;
var obj = ObjectiveFunction.Gradient(nn.Cost, nn.Gradient);
var solver = new LimitedMemoryBfgsMinimizer(1e-5, 1e-5, 1e-5, 5, maxIter);
var result = solver.FindMinimum(obj, initial_nn_params);
System.Console.WriteLine("Reason For Exit: {0}", result.ReasonForExit);
System.Console.WriteLine("Iterations: {0}", result.Iterations);
System.Console.WriteLine("Cost: {0:e}", result.FunctionInfoAtMinimum.Value);
Pause();
// ================= Part 10: Implement Predict =================
// After training the neural network, we would like to use it to predict
// the labels. You will now implement the "predict" function to use the
// neural network to predict the labels of the training set. This lets
// you compute the training set accuracy.
Vector <double> pred = nn.Predict(result.MinimizingPoint, X);
Vector <double> comp = V.Dense(y.Count);
for (int i = 0; i < y.Count; i++)
{
if (pred[i] == y[i])
{
comp[i] = 1;
}
else
{
comp[i] = 0;
}
}
double accuracy = comp.Mean() * 100;
System.Console.WriteLine("\nTraining Set Accuracy: {0:f5}\n", accuracy);
Pause();
}
static void Main(string[] args)
{
if (!System.Console.IsOutputRedirected)
{
System.Console.Clear();
}
CultureInfo.CurrentCulture = CultureInfo.CreateSpecificCulture("en-US");
System.Console.WriteLine("Logistic Regression ex.2");
System.Console.WriteLine("========================\n");
var M = Matrix <double> .Build;
var V = Vector <double> .Build;
// Load Data
// The first two columns contains the exam scores and the third column
// contains the label.
Matrix <double> data = DelimitedReader.Read <double>("data\\ex2data1.txt", false, ",", false);
Console.WriteLine(data);
Matrix <double> X = data.SubMatrix(0, data.RowCount, 0, 2);
Vector <double> y = data.Column(2);
System.Console.WriteLine("Features:\n");
System.Console.WriteLine(X);
System.Console.WriteLine("Label:\n");
System.Console.WriteLine(y);
// ==================== Part 1: Plotting ====================
// We start the exercise by first plotting the data to understand the
// the problem we are working with.
System.Console.WriteLine("Plotting data with + indicating (y = 1) examples and o indicating (y = 0) examples.\n");
PlotData(X, y);
GnuPlot.HoldOff();
Pause();
// theta parameters
Vector <double> initial_theta = V.Dense(X.ColumnCount + 1);
// Add intercept term to X
X = X.InsertColumn(0, V.Dense(X.RowCount, 1));
// compute cost
LogisticRegression lr = new LogisticRegression(X, y);
double J = lr.Cost(initial_theta);
Vector <double> grad = lr.Gradient(initial_theta);
System.Console.WriteLine("Cost at initial theta (zeros): {0:f3}\n", J);
System.Console.WriteLine("Expected cost (approx): 0.693\n");
System.Console.WriteLine("Gradient at initial theta (zeros): \n");
System.Console.WriteLine(" {0:f4} \n", grad);
System.Console.WriteLine("Expected gradients (approx):\n -0.1000\n -12.0092\n -11.2628\n");
// Compute and display cost and gradient with non-zero theta
Vector <double> test_theta = V.DenseOfArray(new double[] { -24.0, 0.2, 0.2 });
J = lr.Cost(test_theta);
grad = lr.Gradient(test_theta);
System.Console.WriteLine("\nCost at test theta: {0:f3}\n", J);
System.Console.WriteLine("Expected cost (approx): 0.218\n");
System.Console.WriteLine("Gradient at test theta: \n");
System.Console.WriteLine(" {0:f3} \n", grad);
System.Console.WriteLine("Expected gradients (approx):\n 0.043\n 2.566\n 2.647\n");
Pause();
// ============= Part 3: Optimizing using fmin ================
// In this exercise, I will use fmin function to find the
// optimal parameters theta.
var obj = ObjectiveFunction.Gradient(lr.Cost, lr.Gradient);
var solver = new BfgsMinimizer(1e-5, 1e-5, 1e-5, 1000);
var result = solver.FindMinimum(obj, initial_theta);
System.Console.WriteLine("Cost at theta found by fmin: {0:f5} after {1} iterations\n", result.FunctionInfoAtMinimum.Value, result.Iterations);
System.Console.WriteLine("Expected cost (approx): 0.203\n");
System.Console.WriteLine("theta: \n");
System.Console.WriteLine(result.MinimizingPoint);
System.Console.WriteLine("Expected theta (approx):\n");
System.Console.WriteLine(" -25.161\n 0.206\n 0.201\n");
Pause();
PlotLinearBoundary(X, y, result.MinimizingPoint);
GnuPlot.HoldOff();
// ============== Part 4: Predict and Accuracies ==============
// After learning the parameters, you'll like to use it to predict the outcomes
// on unseen data. In this part, you will use the logistic regression model
// to predict the probability that a student with score 45 on exam 1 and
// score 85 on exam 2 will be admitted.
//
// Furthermore, you will compute the training and test set accuracies of
// our model.
//
// Your task is to complete the code in predict.m
// Predict probability for a student with score 45 on exam 1
// and score 85 on exam 2
double prob = LogisticRegression.Predict(V.DenseOfArray(new [] { 1.0, 45.0, 85.0 }), result.MinimizingPoint);
System.Console.WriteLine("For a student with scores 45 and 85, we predict an admission probability of {0:f5}\n", prob);
System.Console.WriteLine("Expected value: 0.775 +/- 0.002\n\n");
Pause();
// Compute accuracy on our training set
Vector <double> pos = LogisticRegression.Predict(X, result.MinimizingPoint);
Func <double, double> map = delegate(double d){
if (d >= 0.5)
{
return(1);
}
else
{
return(0);
}
};
pos = pos.Map(map);
Vector <double> comp = V.Dense(y.Count);
for (int i = 0; i < y.Count; i++)
{
if (pos[i] == y[i])
{
comp[i] = 1;
}
else
{
comp[i] = 0;
}
}
double accurancy = comp.Mean() * 100;
System.Console.WriteLine("Train Accuracy: {0:f5}\n", accurancy);
System.Console.WriteLine("Expected accuracy (approx): 89.0\n");
System.Console.WriteLine("\n");
}
// BFGF Minimizer
public static Value Argmin(Value function, Value initial, Value tolerance, Netlist netlist, Style style, int s)
{
if (!(initial is ListValue <Value>))
{
throw new Error("argmin: expecting a list for second argument");
}
Vector <double> initialGuess = CreateVector.Dense((initial as ListValue <Value>).ToDoubleArray("argmin: expecting a list of numbers for second argument"));
if (!(tolerance is NumberValue))
{
throw new Error("argmin: expecting a number for third argument");
}
double toler = (tolerance as NumberValue).value;
if (!(function is FunctionValue))
{
throw new Error("argmin: expecting a function as first argument");
}
FunctionValue closure = function as FunctionValue;
if (closure.parameters.parameters.Count != 1)
{
throw new Error("argmin: initial values and function parameters have different lengths");
}
IObjectiveFunction objectiveFunction = ObjectiveFunction.Gradient(
(Vector <double> objParameters) => {
const string badResult = "argmin: objective function should return a list with a number (cost) and a list of numbers (partial derivatives of cost)";
List <Value> parameters = new List <Value>(); foreach (double parameter in objParameters)
{
parameters.Add(new NumberValue(parameter));
}
ListValue <Value> arg1 = new ListValue <Value>(parameters);
List <Value> arguments = new List <Value>(); arguments.Add(arg1);
bool autoContinue = netlist.autoContinue; netlist.autoContinue = true;
Value result = closure.ApplyReject(arguments, netlist, style, s);
if (result == null)
{
throw new Error(badResult);
}
netlist.autoContinue = autoContinue;
if (!(result is ListValue <Value>))
{
throw new Error(badResult);
}
List <Value> results = (result as ListValue <Value>).elements;
if (results.Count != 2 || !(results[0] is NumberValue) || !(results[1] is ListValue <Value>))
{
throw new Error(badResult);
}
double cost = (results[0] as NumberValue).value;
ListValue <Value> gradients = results[1] as ListValue <Value>;
KGui.gui.GuiOutputAppendText("argmin: parameters=" + arg1.Format(style) + " => cost=" + style.FormatDouble(cost) + ", gradients=" + results[1].Format(style) + Environment.NewLine);
return(new Tuple <double, Vector <double> >(cost, CreateVector.Dense(gradients.ToDoubleArray(badResult))));
});
try {
BfgsMinimizer minimizer = new BfgsMinimizer(toler, toler, toler);
MinimizationResult result = minimizer.FindMinimum(objectiveFunction, initialGuess);
if (result.ReasonForExit == ExitCondition.Converged || result.ReasonForExit == ExitCondition.AbsoluteGradient || result.ReasonForExit == ExitCondition.RelativeGradient)
{
List <Value> elements = new List <Value>();
for (int i = 0; i < result.MinimizingPoint.Count; i++)
{
elements.Add(new NumberValue(result.MinimizingPoint[i]));
}
ListValue <Value> list = new ListValue <Value>(elements);
KGui.gui.GuiOutputAppendText("argmin: converged with parameters " + list.Format(style) + " and reason '" + result.ReasonForExit + "'" + Environment.NewLine);
return(list);
}
else
{
throw new Error("reason '" + result.ReasonForExit.ToString() + "'");
}
} catch (Exception e) { throw new Error("argmin ended: " + ((e.InnerException == null) ? e.Message : e.InnerException.Message)); } // somehow we need to recatch the inner exception coming from CostAndGradient
}