public void FindMinimum_BigRosenbrock_Hard() { var obj = ObjectiveFunction.Gradient(BigRosenbrockFunction.Value, BigRosenbrockFunction.Gradient); var solver = new LimitedMemoryBfgsMinimizer(1e-5, 1e-5, 1e-5, 5, 1000); var result = solver.FindMinimum(obj, new DenseVector(new[] { -1.2 * 100.0, 1.0 * 100.0 })); Assert.That(Math.Abs(result.MinimizingPoint[0] - BigRosenbrockFunction.Minimum[0]), Is.LessThan(1e-3)); Assert.That(Math.Abs(result.MinimizingPoint[1] - BigRosenbrockFunction.Minimum[1]), Is.LessThan(1e-3)); }
public void FindMinimum_Rosenbrock_Overton() { var obj = ObjectiveFunction.Gradient(RosenbrockFunction.Value, RosenbrockFunction.Gradient); var solver = new LimitedMemoryBfgsMinimizer(1e-5, 1e-5, 1e-5, 5, 100); var result = solver.FindMinimum(obj, new DenseVector(new[] { -0.9, -0.5 })); Assert.That(Math.Abs(result.MinimizingPoint[0] - RosenbrockFunction.Minimum[0]), Is.LessThan(1e-3)); Assert.That(Math.Abs(result.MinimizingPoint[1] - RosenbrockFunction.Minimum[1]), Is.LessThan(1e-3)); }
public void Thurber_LBfgs_Dif() { var obj = ObjectiveFunction.NonlinearFunction(ThurberModel, ThurberX, ThurberY, accuracyOrder: 6); var solver = new LimitedMemoryBfgsMinimizer(1e-10, 1e-10, 1e-10, 1000); var result = solver.FindMinimum(obj, ThurberStart); for (int i = 0; i < result.MinimizingPoint.Count; i++) { AssertHelpers.AlmostEqualRelative(ThurberPbest[i], result.MinimizingPoint[i], 6); } }
public void Rat43_LBfgs_Dif() { var obj = ObjectiveFunction.NonlinearFunction(Rat43Model, Rat43X, Rat43Y, accuracyOrder: 6); var solver = new LimitedMemoryBfgsMinimizer(1e-10, 1e-10, 1e-10, 1000); var result = solver.FindMinimum(obj, Rat43Start2); for (int i = 0; i < result.MinimizingPoint.Count; i++) { AssertHelpers.AlmostEqualRelative(Rat43Pbest[i], result.MinimizingPoint[i], 2); } }
public void Mgh_Tests(TestFunctions.TestCase test_case) { var obj = new MghObjectiveFunction(test_case.Function, true, true); var solver = new LimitedMemoryBfgsMinimizer(1e-8, 1e-8, 1e-8, 5, 1000); var result = solver.FindMinimum(obj, test_case.InitialGuess); if (test_case.MinimizingPoint != null) { Assert.That((result.MinimizingPoint - test_case.MinimizingPoint).L2Norm(), Is.LessThan(1e-3)); } var val1 = result.FunctionInfoAtMinimum.Value; var val2 = test_case.MinimalValue; var abs_min = Math.Min(Math.Abs(val1), Math.Abs(val2)); var abs_err = Math.Abs(val1 - val2); var rel_err = abs_err / abs_min; var success = (abs_min <= 1 && abs_err < 1e-3) || (abs_min > 1 && rel_err < 1e-3); Assert.That(success, "Minimal function value is not as expected."); }
static void Main(string[] args) { if (!System.Console.IsOutputRedirected) { System.Console.Clear(); } CultureInfo.CurrentCulture = CultureInfo.CreateSpecificCulture("en-US"); System.Console.WriteLine("Multi-class Classification and Neural Networks ex.4"); System.Console.WriteLine("===================================================\n"); var M = Matrix <double> .Build; var V = Vector <double> .Build; // Setup the parameters you will use for this exercise int input_layer_size = 400; // 20x20 Input Images of Digits int hidden_layer_size = 25; // 25 hidden units int num_labels = 10; // 10 labels, from 1 to 10 // (note that we have mapped "0" to label 10) // =========== Part 1: Loading and Visualizing Data ============= // We start the exercise by first loading and visualizing the dataset. // You will be working with a dataset that contains handwritten digits. // // read all matrices of a file by name into a dictionary Dictionary <string, Matrix <double> > ms = MatlabReader.ReadAll <double>("data\\ex3data1.mat"); Matrix <double> X = ms["X"]; Vector <double> y = ms["y"].Column(0); // get a casual sequence of 100 int numbers var srs = new MathNet.Numerics.Random.SystemRandomSource(); var seq = srs.NextInt32Sequence(0, 5000).Take(100).ToList(); // Randomly select 100 data points to display Vector <double>[] sel = new Vector <double> [100]; int idx = 0; Vector <double> v = V.Dense(400); foreach (int i in seq) { sel[idx++] = X.Row(i); } // display DisplayData(sel); Pause(); // ================ Part 2: Loading Parameters ================ // In this part of the exercise, we load some pre-initialized // neural network parameters. System.Console.WriteLine("\nLoading Saved Neural Network Parameters ...\n"); // read all matrices of a file by name into a dictionary Dictionary <string, Matrix <double> > mr = MatlabReader.ReadAll <double>("data\\ex3weights.mat"); Matrix <double> theta1 = mr["Theta1"]; // 25 X 401 Matrix <double> theta2 = mr["Theta2"]; // 10 X 26 // Unroll parameters Vector <double> nn_params = NeuralNetwork.UnrollParameters(theta1, theta2); Pause(); // ================ Part 3: Compute Cost (Feedforward) ================ // To the neural network, you should first start by implementing the // feedforward part of the neural network that returns the cost only. You // should complete the code in nnCostFunction.m to return cost. After // implementing the feedforward to compute the cost, you can verify that // your implementation is correct by verifying that you get the same cost // as us for the fixed debugging parameters. // // We suggest implementing the feedforward cost *without* regularization // first so that it will be easier for you to debug. Later, in part 4, you // will get to implement the regularized cost. System.Console.WriteLine("\nFeedforward Using Neural Network ...\n"); // Weight regularization parameter (we set this to 0 here). NeuralNetwork nn = new NeuralNetwork(X, y, input_layer_size, hidden_layer_size, num_labels); nn.Lambda = 0.0; double J = nn.Cost(nn_params); System.Console.WriteLine("Cost at parameters (loaded from ex4weights): {0:f6}\n(this value should be about 0.287629)\n", J); Pause(); // =============== Part 4: Implement Regularization =============== // Once your cost function implementation is correct, you should now // continue to implement the regularization with the cost. // System.Console.WriteLine("\nChecking Cost Function (w/ Regularization) ... \n"); // Weight regularization parameter (we set this to 1 here). nn.Lambda = 1.0; J = nn.Cost(nn_params); System.Console.WriteLine("Cost at parameters (loaded from ex4weights): {0:f6} \n(this value should be about 0.383770)\n", J); Pause(); // ================ Part 5: Sigmoid Gradient ================ // Before you start implementing the neural network, you will first // implement the gradient for the sigmoid function. You should complete the // code in the sigmoidGradient.m file. System.Console.WriteLine("\nEvaluating sigmoid gradient...\n"); var g = nn.SigmoidGradient(V.DenseOfArray(new[] { -1.0, -0.5, 0, 0.5, 1 })); System.Console.WriteLine("Sigmoid gradient evaluated at [-1 -0.5 0 0.5 1]:\n "); System.Console.WriteLine("{0:f5} ", g); System.Console.WriteLine("\n\n"); Pause(); // ================ Part 6: Initializing Pameters ================ // In this part of the exercise, you will be starting to implment a two // layer neural network that classifies digits. You will start by // implementing a function to initialize the weights of the neural network // (randInitializeWeights.m) System.Console.WriteLine("\nInitializing Neural Network Parameters ...\n"); Matrix <double> initial_Theta1 = RandInitializeWeights(input_layer_size + 1, hidden_layer_size); Matrix <double> initial_Theta2 = RandInitializeWeights(hidden_layer_size + 1, num_labels); // Unroll parameters Vector <double> initial_nn_params = NeuralNetwork.UnrollParameters(initial_Theta1, initial_Theta2); Pause(); // =============== Part 7: Implement Backpropagation =============== // Once your cost matches up with ours, you should proceed to implement the // backpropagation algorithm for the neural network. You should add to the // code you've written in nnCostFunction.m to return the partial // derivatives of the parameters. System.Console.WriteLine("\nChecking Backpropagation... \n"); CheckGradient(0); Pause(); // =============== Part 8: Implement Regularization =============== // Once your backpropagation implementation is correct, you should now // continue to implement the regularization with the cost and gradient. System.Console.WriteLine("\nChecking Backpropagation (w/ Regularization) ... \n"); // Check gradients by running checkNNGradients double lambda = 3; CheckGradient(lambda); // Also output the costFunction debugging values nn.Lambda = lambda; double debug_J = nn.Cost(nn_params); System.Console.WriteLine("\n\nCost at (fixed) debugging parameters (w/ lambda = {0:f1}): {1:f6} " + "\n(for lambda = 3, this value should be about 0.576051)\n\n", lambda, debug_J); Pause(); // =================== Part 8: Training NN =================== // You have now implemented all the code necessary to train a neural // network. To train your neural network, we will now use "fmincg", which // is a function which works similarly to "fminunc". Recall that these // advanced optimizers are able to train our cost functions efficiently as // long as we provide them with the gradient computations. System.Console.WriteLine("\nTraining Neural Network... \n"); // After you have completed the assignment, change the MaxIter to a larger // value to see how more training helps. int maxIter = 40; // You should also try different values of lambda lambda = 1; nn.Lambda = lambda; var obj = ObjectiveFunction.Gradient(nn.Cost, nn.Gradient); var solver = new LimitedMemoryBfgsMinimizer(1e-5, 1e-5, 1e-5, 5, maxIter); var result = solver.FindMinimum(obj, initial_nn_params); System.Console.WriteLine("Reason For Exit: {0}", result.ReasonForExit); System.Console.WriteLine("Iterations: {0}", result.Iterations); System.Console.WriteLine("Cost: {0:e}", result.FunctionInfoAtMinimum.Value); Pause(); // ================= Part 10: Implement Predict ================= // After training the neural network, we would like to use it to predict // the labels. You will now implement the "predict" function to use the // neural network to predict the labels of the training set. This lets // you compute the training set accuracy. Vector <double> pred = nn.Predict(result.MinimizingPoint, X); Vector <double> comp = V.Dense(y.Count); for (int i = 0; i < y.Count; i++) { if (pred[i] == y[i]) { comp[i] = 1; } else { comp[i] = 0; } } double accuracy = comp.Mean() * 100; System.Console.WriteLine("\nTraining Set Accuracy: {0:f5}\n", accuracy); Pause(); }