[Test] public void CompareMultiLayerPerceptron() { var model = MultiLayerPerceptronModel(); var ctx = Context.GpuContext(0); var memMb = ctx.ToGpuContext().Gpu.Device.TotalMemory / 1024.0 / 1024.0; if (memMb < 4096.0) { Assert.Inconclusive("Need more Gpu memory."); } var opt = new GradientDescentOptimizer(ctx, model.Loss.Loss, 0.00008); // now we need to initalize the parameters for the optimizer opt.Initalize(); // load mnist data var mnist = new MNIST(); var batcher = new Batcher(ctx, mnist.TrainImages, mnist.TrainLabels); var timer = Stopwatch.StartNew(); for (var i = 0; i < 1; ++i) { batcher.Next(5000, opt, model.Images, model.Labels); opt.Forward(); opt.Backward(); opt.Optimize(); } timer.Stop(); Console.WriteLine(timer.Elapsed); timer.Restart(); for (var i = 0; i < 5; ++i) { batcher.Next(10000, opt, model.Images, model.Labels); opt.Forward(); opt.Backward(); opt.Optimize(); } ctx.ToGpuContext().Stream.Synchronize(); timer.Stop(); Console.WriteLine(timer.Elapsed); timer.Restart(); PrintResult(opt, model, mnist.TestImages, mnist.TestLabels); timer.Stop(); Console.WriteLine(timer.Elapsed); CleanMem_(); }
[Test] public void CompareConvolutionalNeuralNetwork() { var model = ConvolutionalNeuralNetworkModel(); var ctx = Context.GpuContext(0); var memMb = ctx.ToGpuContext().Gpu.Device.TotalMemory / 1024.0 / 1024.0; if (memMb < 4096.0) { Assert.Inconclusive("Need more Gpu memory."); } var opt = new GradientDescentOptimizer(ctx, model.Loss.Loss, 0.000008); opt.Initalize(); var mnist = new MNIST(); var batcher = new Batcher(ctx, mnist.TrainImages, mnist.TrainLabels); var timer = Stopwatch.StartNew(); for (var i = 0; i < 2; ++i) { batcher.Next(2500, opt, model.Images, model.Labels); opt.Forward(); opt.Backward(); opt.Optimize(); } timer.Stop(); Console.WriteLine(timer.Elapsed); timer.Restart(); for (var i = 0; i < 20; ++i) { batcher.Next(2500, opt, model.Images, model.Labels); opt.Forward(); opt.Backward(); opt.Optimize(); } ctx.ToGpuContext().Stream.Synchronize(); timer.Stop(); Console.WriteLine(timer.Elapsed); timer.Restart(); PrintResult(opt, model, mnist.TestImages, mnist.TestLabels); timer.Stop(); Console.WriteLine(timer.Elapsed); CleanMem_(); }
[Test] public void MultinomialRegression() { CleanMem_(); const long batchSize = 1000L; const long epochs = 3; var model = MultinomialRegressionModel(); var ctx = Context.GpuContext(0); var opt = new GradientDescentOptimizer(ctx, model.Loss.Loss, 0.0005); opt.Initalize(); var mnist = new MNIST(); var batcher = new Batcher(ctx, mnist.TrainImages, mnist.TrainLabels); for (var e = 1; e <= epochs; ++e) { var i = 0; while (batcher.Next(batchSize, opt, model.Images, model.Labels)) { i++; opt.Forward(); opt.Backward(); opt.Optimize(); if ((i % 10 == 0) || ((i == 1) && (e == 1))) { PrintStatus(e, i, opt, model, mnist.ValidationImages, mnist.ValidationLabels); } } } PrintResult(opt, model, mnist.TestImages, mnist.TestLabels); CleanMem_(); }
public void GradientDescentOptimizerReturnsValueCloseToOptimumOneParameter() { var expected = 42.0; var parameterSettings = new[] { new ParameterSetting(0, 50, 1, 30) }; Func <double[], double> costFunc = parameters => Math.Abs(expected - parameters[0]); var actual = GradientDescentOptimizer.Optimize(costFunc, parameterSettings, parameterSettings[0].StepSize); Assert.That(actual.Parameters[0], Is.EqualTo(expected).Within(parameterSettings[0].StepSize)); }
public void GradientDescentOptimizerHandlesConstantCostFunction() { var parameterSettings = new[] { new ParameterSetting(0, 50, 1, 30) }; Func <double[], double> costFunc = parameters => 0; OptimizationResult actual = null; Assert.That(() => actual = GradientDescentOptimizer.Optimize(costFunc, parameterSettings, parameterSettings[0].StepSize), Throws.Nothing); Assert.That(actual, Is.Not.Null); Assert.That(actual.Parameters[0], Is.Not.NaN); }
//runs the backbone of the network (forward and backward prop and other stuff) [Test] public void LinearNeuron(Datas data) { CleanMem_(); const long BatchSize = 1000L; const long Epoch = 5; var model = LinearModel(); var ctx = Context.GpuContext(0); //makes sure gpu has 2 or more GB of ram var memMB = ctx.ToGpuContext().Gpu.Device.TotalMemory / 1024.0 / 1024.0; if (memMB < 4096.0) { Assert.Inconclusive("Need more gpu mem"); } var opt = new GradientDescentOptimizer(ctx, model.Loss.Loss, 0.00005); opt.Initalize(); var batcher = new Batcher(ctx, data.TrainText, data.TrainStory); for (var e = 1; e < Epoch; e++) { int i = 0; while (batcher.Next(BatchSize, opt, model.Text, model.Story)) { i++; opt.Forward(); opt.Backward(); opt.Optimize(); if ((i % 10 == 0) || (i == 1 && e == 1)) { PrintStatus(e, i, opt, model, data.TrainText, data.TrainStory); } } } //PrintResult(opt, model, data.TestText, data.TestStory); //Need to make some place to dump the weights and biases, but not exactly sure how... maybe just have it write a story right away? CleanMem_(); }
public static void SimpleLogisticRegression() { //const int N = 8; //const int D = 5; //const int P = 3; //const double learn = 0.001; const int N = 100; const int D = 784; const int P = 10; const double learn = 0.00005; var input = Variable <double>(); var label = Variable <double>(); var weights = Parameter(0.01 * RandomUniform <double>(Shape.Create(D, P))); var pred = Dot(input, weights); var loss = L2Loss(pred, label); var ctx = Context.GpuContext(0); var opt = new GradientDescentOptimizer(ctx, loss, learn); // set some data var inputData = new double[N, D]; var matA = new double[D, P]; var matB = new double[N, P]; NormalRandomArray(inputData); NormalRandomArray(matA); NormalRandomArray(matB); var labelData = Dot(inputData, matA).Add(matB.Mul(0.1)); opt.AssignTensor(input, inputData.AsTensor()); opt.AssignTensor(label, labelData.AsTensor()); opt.Initalize(); for (var i = 0; i < 800; ++i) { opt.Forward(); opt.Backward(); opt.Optimize(); if (i % 20 == 0) { Console.WriteLine($"loss = {opt.GetTensor(loss).ToScalar()}"); } } }
public void GradientDescentOptimizerReturnsValueCloseToOptimumThreeParameters() { var expected = new double[] { 1, 13, -4 }; var parameterSettings = new[] { new ParameterSetting(0, 10, 0.1, Double.NaN), new ParameterSetting(5, 20, 0.5, 19), new ParameterSetting(-10, 10, 0.2, 1.1) }; Func <double[], double> costFunc = parameters => Math.Abs(expected[0] - parameters[0]) + Math.Abs(expected[1] - parameters[1]) + Math.Abs(expected[2] - parameters[2]); var actual = GradientDescentOptimizer.Optimize(costFunc, parameterSettings, parameterSettings.Sum(p => p.StepSize)); for (int p = 0; p < expected.Length; p++) { Assert.That(actual.Parameters[p], Is.EqualTo(expected[p]).Within(parameterSettings[p].StepSize)); } }
public void EstimateAminoAcidHelixAffinity() { var annotatedSequencesFile = @"G:\Projects\HumanGenome\fullPdbSequencesHelixMarked.txt"; var annotatedSequences = ParseHelixSequences(annotatedSequencesFile); //var aminoAcidPairs = GetAminoAcidPairs(annotatedSequences); //var leastCommonPair = aminoAcidPairs.OrderBy(kvp => kvp.Value).First(); Func <double[], double> costFunc = parameters => HelixSequenceCostFunc(parameters, annotatedSequences); var parameterSettings = GeneratePairwiseAminoAcidParameters(); var randomizedStartValueIterations = 100; //Parallel.For(0L, randomizedStartValueIterations, idx => for (int idx = 0; idx < randomizedStartValueIterations; idx++) { RandomizeStartValues(parameterSettings, 2); var optimizationResult = GradientDescentOptimizer.Optimize(costFunc, parameterSettings, double.NegativeInfinity); WriteOptimizationResult(@"G:\Projects\HumanGenome\helixAffinityOptimizationResults.dat", optimizationResult); } }