public static BackpropAlgorithm CreateMainColorsDemo1() { Console.WriteLine("init CreateMainColorsDemo1"); var activation = Activation.ReLU; var net = new ConvNet(3, 48) { IsTraining = true }; net.AddLayer(new FlattenLayer(outputDim: 128, activation: activation)); net.AddLayer(new FlattenLayer(outputDim: 128, activation: activation)); net.AddLayer(new DenseLayer(outputDim: 12, activation: activation)); net._Build(); net.RandomizeParameters(seed: 0); var lrate = 1.1D; var alg = new BackpropAlgorithm(net) { EpochCount = 500, LearningRate = lrate, BatchSize = 8, UseBatchParallelization = true, MaxBatchThreadCount = 8, LossFunction = Loss.Euclidean, Optimizer = Optimizer.Adadelta, Regularizator = Regularizator.L2(0.0001D), LearningRateScheduler = LearningRateScheduler.DropBased(lrate, 5, 0.5D) }; alg.Build(); return(alg); }
public static ConvNet SimpleLinearNetworkWithDropout(double drate, int dseed) { var net = new ConvNet(1, 1, activation: new Mocks.LinearActivation()); net.IsTraining = true; var layer1 = new DenseLayer(1); net.AddLayer(layer1); var layer2 = new DenseLayer(1); net.AddLayer(layer2); var layer3 = new DropoutLayer(drate, dseed); layer3.Mask = new bool[1][, ] { new bool[, ] { { true } } }; net.AddLayer(layer3); var layer4 = new DenseLayer(1); net.AddLayer(layer4); net._Build(); layer1.Weights[1] = 1; layer1.Weights[0] = 3; layer2.Weights[1] = -1; layer2.Weights[0] = 1; layer4.Weights[1] = 2; layer4.Weights[0] = -1; return(net); }
public static ConvNet SimpleLinearNetwork(IActivationFunction activation = null) { activation = activation ?? new Mocks.LinearActivation(); var net = new ConvNet(1, 1, activation: activation); net.IsTraining = true; var layer1 = new DenseLayer(1); net.AddLayer(layer1); var layer2 = new DenseLayer(1); net.AddLayer(layer2); var layer3 = new DenseLayer(1); net.AddLayer(layer3); net._Build(); layer1.Weights[1] = 1; layer1.Weights[0] = 3; layer2.Weights[1] = -1; layer2.Weights[0] = 1; layer3.Weights[1] = 2; layer3.Weights[0] = -1; return(net); }
public static BackpropAlgorithm CreateMNISTHardDemo(ClassifiedSample <double[][, ]> training) { Console.WriteLine("init CreateMNISTHardDemo"); var activation = Activation.ReLU; var net = new ConvNet(1, 28) { IsTraining = true }; net.AddLayer(new ConvLayer(outputDepth: 32, windowSize: 3, activation: activation)); net.AddLayer(new ConvLayer(outputDepth: 64, windowSize: 3, activation: activation)); net.AddLayer(new MaxPoolingLayer(windowSize: 2, stride: 2)); net.AddLayer(new DropoutLayer(0.25)); net.AddLayer(new FlattenLayer(outputDim: 128, activation: activation)); net.AddLayer(new DropoutLayer(0.5)); net.AddLayer(new FlattenLayer(outputDim: 10, activation: Activation.Logistic(1))); net._Build(); net.RandomizeParameters(seed: 0); var lrate = 0.005D; var alg = new BackpropAlgorithm(training, net) { LossFunction = Loss.Euclidean, EpochCount = 50, LearningRate = lrate, BatchSize = 1, LearningRateScheduler = LearningRateScheduler.Constant(lrate) }; return(alg); }
/// <summary> /// /// </summary> public static BackpropAlgorithm CreateMNISTSimpleDemoWithBatching(ClassifiedSample <double[][, ]> training) { Console.WriteLine("init CreateMNISTSimpleDemoWithBatching"); var activation = Activation.ReLU; var net = new ConvNet(1, 28) { IsTraining = true }; net.AddLayer(new ConvLayer(outputDepth: 8, windowSize: 5)); net.AddLayer(new MaxPoolingLayer(windowSize: 2, stride: 2, activation: activation)); net.AddLayer(new ConvLayer(outputDepth: 18, windowSize: 5)); net.AddLayer(new MaxPoolingLayer(windowSize: 2, stride: 2, activation: activation)); net.AddLayer(new FlattenLayer(outputDim: 10, activation: activation)); net._Build(); net.RandomizeParameters(seed: 0); var lrate = 0.0001D; var alg = new BackpropAlgorithm(training, net) { EpochCount = 50, LearningRate = lrate, BatchSize = 8, UseBatchParallelization = true, MaxBatchThreadCount = 8, LossFunction = Loss.Euclidean, Optimizer = Optimizer.RMSProp, LearningRateScheduler = LearningRateScheduler.DropBased(lrate, 5, 0.5D) }; return(alg); }
/// <summary> /// Error: 19.1 /// </summary> public static BackpropAlgorithm CreateKaggleCatOrDogDemo_Pretrained() { Console.WriteLine("init CreateKaggleCatOrDogDemo_Pretrained"); ConvNet net; var assembly = Assembly.GetExecutingAssembly(); using (var stream = assembly.GetManifestResourceStream("ML.DeepTests.Pretrained.cn_e16_p37.65.mld")) { net = ConvNet.Deserialize(stream); net.IsTraining = true; } var lrate = 0.01D; var alg = new BackpropAlgorithm(net) { LossFunction = Loss.CrossEntropySoftMax, EpochCount = 500, LearningRate = lrate, BatchSize = 4, UseBatchParallelization = true, MaxBatchThreadCount = 8, Optimizer = Optimizer.Adadelta, Regularizator = Regularizator.L2(0.001D), LearningRateScheduler = LearningRateScheduler.DropBased(lrate, 5, 0.5D) }; alg.Build(); return(alg); }
public void Gradient_1ConvLayer_1Iter_Euclidean() { // arrange var net = new ConvNet(3, 1, 1) { IsTraining = true }; net.AddLayer(new ConvLayer(outputDepth: 2, windowSize: 1, activation: Activation.Atan)); net._Build(); net.RandomizeParameters(seed: 0); var point1 = RandomPoint(3, 1, 1); var point2 = RandomPoint(3, 1, 1); // just for 2 dim output var sample = new ClassifiedSample <double[][, ]>(); sample[point1] = CLASSES[0]; sample[point2] = CLASSES[1]; var alg = new BackpropAlgorithm(net) { LearningRate = 0.1D, LossFunction = Loss.Euclidean }; alg.Build(); // act alg.RunIteration(point1, EXPECTED[0]); // assert AssertNetGradient(alg, point1, EXPECTED[0]); }
private ML.DeepMethods.Algorithms.BackpropAlgorithm createCNNAlg_NN_ForTest() { var cnn = new ConvNet(2, 1) { IsTraining = true }; cnn.AddLayer(new DenseLayer(15, activation: Activation.Logistic(1))); cnn.AddLayer(new MaxPoolingLayer(1, 1)); //cnn.AddLayer(new _ActivationLayer(Activation.Logistic(1))); cnn.AddLayer(new DropoutLayer(0.1)); cnn.AddLayer(new FlattenLayer(3, activation: Activation.Logistic(1))); //cnn.AddLayer(new _ActivationLayer(Activation.Logistic(1))); cnn.AddLayer(new MaxPoolingLayer(1, 1)); cnn._Build(); cnn.RandomizeParameters(0); var sample = new ClassifiedSample <double[][, ]>(); foreach (var obj in Data.TrainingSample) { var data = obj.Key; var key = new double[data.Length][, ]; for (int i = 0; i < data.Length; i++) { key[i] = new double[1, 1]; } for (int i = 0; i < data.Length; i++) { key[i][0, 0] = data[i]; } sample[key] = obj.Value; } var alg = new ML.DeepMethods.Algorithms.BackpropAlgorithm(sample, cnn); alg.EpochCount = 6000; alg.LearningRate = 0.01D; alg.BatchSize = 1; alg.LossFunction = Loss.Euclidean; int epoch = 0; alg.EpochEndedEvent += (o, e) => { if (epoch++ % 300 != 0) { return; } Console.WriteLine("----------------Epoch #: {0}", epoch); Console.WriteLine("L:\t{0}", alg.LossValue); Console.WriteLine("DL:\t{0}", alg.LossDelta); Console.WriteLine("DW:\t{0}", alg.Step2); }; return(alg); }
private void testSerDeser() { var activation = Activation.ReLU; var net = new ConvNet(1, 28) { IsTraining = true }; net.AddLayer(new ConvLayer(outputDepth: 8, windowSize: 5)); net.AddLayer(new MaxPoolingLayer(windowSize: 2, stride: 2, activation: activation)); net.AddLayer(new ConvLayer(outputDepth: 18, windowSize: 5)); net.AddLayer(new MaxPoolingLayer(windowSize: 2, stride: 2, activation: activation)); net.AddLayer(new FlattenLayer(outputDim: 10, activation: activation)); net._Build(); net.RandomizeParameters(seed: 0); var path = @"F:\Work\AgniCore\_testftp\cn_e5_p53.88.mld"; //using (var file = System.IO.File.Open(path, System.IO.FileMode.Create, System.IO.FileAccess.Write)) // net.Serialize(file); using (var file = System.IO.File.Open(path, System.IO.FileMode.Open, System.IO.FileAccess.Read)) { var res = ConvNet.Deserialize(file); } }
protected override void Test() { ConvNet lenet1; var fpath = Path.Combine(OutputPath, "cn_e26_p0.06.mld"); //var fpath = @"F:\Work\git\ML\solution\ML.DigitsDemo\lenet1.mld"; using (var stream = File.Open(fpath, FileMode.Open)) { lenet1 = ConvNet.Deserialize(stream); } var alg = new BackpropAlgorithm(lenet1); var fout = Path.Combine(SrcPath, "result1.csv"); using (var file = File.Open(fout, FileMode.Create, FileAccess.Write)) using (var writer = new StreamWriter(file)) { writer.WriteLine("ImageId,Label"); int num = 1; foreach (var data in m_Test) { var mark = alg.Predict(data); var idx = GeneralUtils.ArgMax(mark); var cls = m_Classes[idx]; writer.WriteLine("{0},{1}", num++, (int)cls.Value); } writer.Flush(); } }
protected override void Test() { ConvNet lenet1; var fpath = Path.Combine(ResultsFolder, "cn_e26_p0.06.mld"); //var fpath = @"F:\Work\git\ML\solution\ML.DigitsDemo\lenet1.mld"; using (var stream = File.Open(fpath, FileMode.Open)) { lenet1 = ConvNet.Deserialize(stream); } var alg = new BackpropAlgorithm(m_Training, lenet1); var fout = Path.Combine(MnistSrc, "result1.csv"); using (var file = File.Open(fout, FileMode.Create, FileAccess.Write)) using (var writer = new StreamWriter(file)) { writer.WriteLine("ImageId,Label"); int num = 1; foreach (var data in m_Test) { var cls = alg.Classify(data); writer.WriteLine("{0},{1}", num++, (int)cls.Value); } writer.Flush(); } }
public static BackpropAlgorithm CreateKaggleCatOrDogFiltersDemo1_Pretrained(string fpath) { Console.WriteLine("init CreateKaggleCatOrDogFiltersDemo1_Pretrained"); ConvNet net; var assembly = Assembly.GetExecutingAssembly(); using (var stream = System.IO.File.Open(fpath, System.IO.FileMode.Open, System.IO.FileAccess.Read)) { net = ConvNet.Deserialize(stream); net.IsTraining = true; } var lrate = 0.001D; var alg = new BackpropAlgorithm(net) { LossFunction = Loss.CrossEntropySoftMax, EpochCount = 500, LearningRate = lrate, BatchSize = 8, UseBatchParallelization = true, MaxBatchThreadCount = 8, Optimizer = Optimizer.Adadelta, Regularizator = Regularizator.L2(0.001D), LearningRateScheduler = LearningRateScheduler.DropBased(lrate, 5, 0.5D) }; alg.Build(); return(alg); }
public BackpropAlgorithm(ConvNet net) : base(net) { m_EpochCount = DFT_EPOCH_COUNT; m_LearningRate = DFT_LEARNING_RATE; m_Stop = DTF_STOP_CRITERIA; m_BatchSize = DFT_BATCH_SIZE; m_MaxBatchThreadCount = DFT_BATCH_THREAD_COUNT; }
private void initNet() { var assembly = Assembly.GetExecutingAssembly(); using (var stream = assembly.GetManifestResourceStream("ML.DigitsDemo.mnist.mld")) { m_Network = ConvNet.Deserialize(stream); } }
protected ConvNetAlgorithmBase(ConvNet net) { if (net == null) { throw new MLException("Network can not be null"); } m_Net = net; }
protected ConvNetAlgorithmBase(ClassifiedSample <double[][, ]> trainingSample, ConvNet net) : base(trainingSample) { if (net == null) { throw new MLException("Network can not be null"); } m_Net = net; }
void run_optimization(ConvNet conv_net, OptimizerV2 optimizer, Tensor x, Tensor y) { using var g = tf.GradientTape(); var pred = conv_net.Apply(x, training: true); var loss = cross_entropy_loss(pred, y); // Compute gradients. var gradients = g.gradient(loss, conv_net.trainable_variables); // Update W and b following gradients. optimizer.apply_gradients(zip(gradients, conv_net.trainable_variables.Select(x => x as ResourceVariable))); }
public void Gradient_DifferentLayers_1Iter_CrossEntropy_Regularization() { // arrange var activation = Activation.ReLU; var net = new ConvNet(1, 5) { IsTraining = true }; net.AddLayer(new ConvLayer(outputDepth: 2, windowSize: 3, padding: 1)); net.AddLayer(new MaxPoolingLayer(windowSize: 3, stride: 2, activation: Activation.Exp)); net.AddLayer(new ActivationLayer(activation: Activation.Tanh)); net.AddLayer(new FlattenLayer(outputDim: 10, activation: activation)); net.AddLayer(new DropoutLayer(rate: 0.5D)); net.AddLayer(new DenseLayer(outputDim: 3, activation: Activation.Exp)); net._Build(); net.RandomizeParameters(seed: 0); var sample = new ClassifiedSample <double[][, ]>(); for (int i = 0; i < 3; i++) { var point = RandomPoint(1, 5, 5); sample[point] = new Class(i.ToString(), i); } var regularizator = Regularizator.Composite(Regularizator.L1(0.1D), Regularizator.L2(0.3D)); var alg = new BackpropAlgorithm(net) { LearningRate = 0.1D, LossFunction = Loss.CrossEntropySoftMax, Regularizator = regularizator }; alg.Build(); // act var data = sample.First(); var expected = new double[3] { 1.0D, 0.0D, 0.0D }; alg.RunIteration(data.Key, expected); regularizator.Apply(alg.Gradient, alg.Net.Weights); ((DropoutLayer)alg.Net[4]).ApplyCustomMask = true; // assert AssertNetGradient(alg, data.Key, expected); }
private void initNet() { try { var assembly = Assembly.GetExecutingAssembly(); using (var stream = assembly.GetManifestResourceStream("ML.MainColorDemo.net.mld")) { m_Network = ConvNet.Deserialize(stream); m_Network.IsTraining = false; } } catch (Exception error) { MessageBox.Show("Error while CNN deserialize: " + error.Message); } }
public void Gradient_MNISTSimple_1Iter() { // arrange var activation = Activation.ReLU; var net = new ConvNet(1, 14) { IsTraining = true }; net.AddLayer(new ConvLayer(outputDepth: 4, windowSize: 5)); net.AddLayer(new MaxPoolingLayer(windowSize: 2, stride: 2, activation: activation)); net.AddLayer(new ConvLayer(outputDepth: 8, windowSize: 5)); net.AddLayer(new MaxPoolingLayer(windowSize: 2, stride: 2, activation: activation)); net.AddLayer(new FlattenLayer(outputDim: 10, activation: activation)); net._Build(); Randomize(net.Weights, -1.0D, 1.0D); var sample = new ClassifiedSample <double[][, ]>(); for (int i = 0; i < 10; i++) { var point = RandomPoint(1, 14, 14); sample[point] = new Class(i.ToString(), i); } var alg = new BackpropAlgorithm(net) { LearningRate = 0.005D, LossFunction = Loss.Euclidean }; alg.Build(); // act var data = sample.First(); var expected = new double[10] { 1.0D, 0.0D, 0.0D, 0.0D, 0.0D, 0.0D, 0.0D, 0.0D, 0.0D, 0.0D }; alg.RunIteration(data.Key, expected); // assert AssertNetGradient(alg, data.Key, expected); }
/// <summary> /// Error 21.65 /// </summary> public static BackpropAlgorithm CreateCIFAR10Trunc2ClassesDemo2_SEALED() { Console.WriteLine("init CreateCIFAR10Trunc2ClassesDemo2"); var activation = Activation.ReLU; var net = new ConvNet(3, 32) { IsTraining = true }; net.AddLayer(new ConvLayer(outputDepth: 16, windowSize: 3, padding: 1, activation: activation)); net.AddLayer(new ConvLayer(outputDepth: 16, windowSize: 3, padding: 1, activation: activation)); net.AddLayer(new MaxPoolingLayer(windowSize: 3, stride: 2)); net.AddLayer(new DropoutLayer(0.25)); net.AddLayer(new ConvLayer(outputDepth: 32, windowSize: 3, padding: 1, activation: activation)); net.AddLayer(new ConvLayer(outputDepth: 32, windowSize: 3, padding: 1, activation: activation)); net.AddLayer(new MaxPoolingLayer(windowSize: 3, stride: 2)); net.AddLayer(new DropoutLayer(0.25)); net.AddLayer(new FlattenLayer(outputDim: 256, activation: activation)); net.AddLayer(new DropoutLayer(0.5)); net.AddLayer(new DenseLayer(outputDim: 2, activation: Activation.Exp)); net._Build(); net.RandomizeParameters(seed: 0); var lrate = 0.01D; var alg = new BackpropAlgorithm(net) { LossFunction = Loss.CrossEntropySoftMax, EpochCount = 500, LearningRate = lrate, BatchSize = 4, UseBatchParallelization = true, MaxBatchThreadCount = 8, Optimizer = Optimizer.Adadelta, Regularizator = Regularizator.L2(0.001D), LearningRateScheduler = LearningRateScheduler.DropBased(lrate, 5, 0.5D) }; alg.Build(); return(alg); }
private ML.DeepMethods.Algorithms.BackpropAlgorithm createCNNAlg_NN_ForTest() { var cnn = new ConvNet(6, 1) { IsTraining = true }; cnn.AddLayer(new DenseLayer(15, activation: Activation.Logistic(1))); cnn.AddLayer(new DenseLayer(15, activation: Activation.Logistic(1))); cnn.AddLayer(new DenseLayer(15, activation: Activation.Logistic(1))); //cnn.AddLayer(new MaxPoolingLayer(1, 1)); //cnn.AddLayer(new _ActivationLayer(Activation.Logistic(1))); //cnn.AddLayer(new DropoutLayer(0.1)); cnn.AddLayer(new FlattenLayer(2, activation: Activation.Logistic(1))); //cnn.AddLayer(new _ActivationLayer(Activation.Logistic(1))); //cnn.AddLayer(new MaxPoolingLayer(1, 1)); cnn._Build(); cnn.RandomizeParameters(0); var alg = new ML.DeepMethods.Algorithms.BackpropAlgorithm(cnn); alg.EpochCount = 6000; alg.LearningRate = 0.01D; alg.BatchSize = 1; alg.LossFunction = Loss.Euclidean; int epoch = 0; alg.EpochEndedEvent += (o, e) => { if (epoch++ % 300 != 0) { return; } Console.WriteLine("----------------Epoch #: {0}", epoch); Console.WriteLine("L:\t{0}", alg.LossValue); Console.WriteLine("DL:\t{0}", alg.LossDelta); Console.WriteLine("DW:\t{0}", alg.Step2); }; return(alg); }
// https://code.google.com/archive/p/cuda-convnet/ - CIFAR archtectures+errors /// <summary> /// Creates CNN for CIFAR-10 training (from https://cs.stanford.edu/people/karpathy/convnetjs/demo/cifar10.html) /// </summary> public static BackpropAlgorithm CreateCIFAR10Demo2() { Console.WriteLine("init CreateCIFAR10Demo2"); var activation = Activation.LeakyReLU(); var net = new ConvNet(3, 32) { IsTraining = true }; net.AddLayer(new ConvLayer(outputDepth: 32, windowSize: 5, padding: 2, activation: activation)); net.AddLayer(new MaxPoolingLayer(windowSize: 2, stride: 2)); net.AddLayer(new ConvLayer(outputDepth: 40, windowSize: 5, padding: 2, activation: activation)); net.AddLayer(new MaxPoolingLayer(windowSize: 2, stride: 2)); net.AddLayer(new ConvLayer(outputDepth: 60, windowSize: 5, padding: 2, activation: activation)); net.AddLayer(new MaxPoolingLayer(windowSize: 2, stride: 2)); net.AddLayer(new FlattenLayer(outputDim: 1024, activation: activation)); net.AddLayer(new DropoutLayer(0.5)); net.AddLayer(new DenseLayer(outputDim: 1024, activation: activation)); net.AddLayer(new DropoutLayer(0.25)); net.AddLayer(new DenseLayer(outputDim: 10, activation: activation)); net._Build(); net.RandomizeParameters(seed: 0); var lrate = 0.05D; var alg = new BackpropAlgorithm(net) { LossFunction = Loss.Euclidean, EpochCount = 500, LearningRate = lrate, BatchSize = 8, UseBatchParallelization = true, MaxBatchThreadCount = 8, Optimizer = Optimizer.Adadelta, LearningRateScheduler = LearningRateScheduler.TimeBased(lrate, 0.005D) }; alg.Build(); return(alg); }
/// <summary> /// Creates CNN for CIFAR-10 training (from keras) /// </summary> public static BackpropAlgorithm CreateCIFAR10Demo1(ClassifiedSample <double[][, ]> training) { Console.WriteLine("init CreateCIFAR10Demo1"); var activation = Activation.ReLU; var net = new ConvNet(3, 32) { IsTraining = true }; net.AddLayer(new ConvLayer(outputDepth: 32, windowSize: 3, padding: 1, activation: activation)); net.AddLayer(new ConvLayer(outputDepth: 32, windowSize: 3, padding: 1, activation: activation)); net.AddLayer(new MaxPoolingLayer(windowSize: 2, stride: 2)); net.AddLayer(new DropoutLayer(0.25)); net.AddLayer(new ConvLayer(outputDepth: 64, windowSize: 3, padding: 1, activation: activation)); net.AddLayer(new ConvLayer(outputDepth: 64, windowSize: 3, padding: 1, activation: activation)); net.AddLayer(new MaxPoolingLayer(windowSize: 2, stride: 2)); net.AddLayer(new DropoutLayer(0.25)); net.AddLayer(new FlattenLayer(outputDim: 512, activation: Activation.ReLU)); net.AddLayer(new DropoutLayer(0.5)); net.AddLayer(new DenseLayer(outputDim: 10, activation: Activation.Logistic(1))); net._Build(); net.RandomizeParameters(seed: 0); var lrate = 0.01D; var alg = new BackpropAlgorithm(training, net) { LossFunction = Loss.CrossEntropySoftMax, EpochCount = 50, LearningRate = lrate, BatchSize = 8, UseBatchParallelization = true, MaxBatchThreadCount = 8, Optimizer = Optimizer.Adadelta, LearningRateScheduler = LearningRateScheduler.DropBased(lrate, 5, 0.5D) }; return(alg); }
public static ConvNet TestNetwork1() { var activation = Activation.Atan; var net = new ConvNet(1, 2, 2) { IsTraining = true }; net.AddLayer(new ConvLayer(outputDepth: 2, windowSize: 2, padding: 1, activation: activation)); net.AddLayer(new MaxPoolingLayer(windowSize: 2, stride: 2)); net.AddLayer(new ConvLayer(outputDepth: 2, windowSize: 1, activation: activation)); net.AddLayer(new FlattenLayer(outputDim: 3, activation: activation)); net.AddLayer(new DenseLayer(outputDim: 2, activation: activation)); net._Build(); net.RandomizeParameters(seed: 0); return(net); }
public bool Run() { tf.enable_eager_execution(); PrepareData(); // Build neural network model. var conv_net = new ConvNet(new ConvNetArgs { NumClasses = num_classes }); // ADAM optimizer. var optimizer = keras.optimizers.Adam(learning_rate); // Run training for the given number of steps. foreach (var(step, (batch_x, batch_y)) in enumerate(train_data, 1)) { // Run the optimization to update W and b values. run_optimization(conv_net, optimizer, batch_x, batch_y); if (step % display_step == 0) { var pred = conv_net.Apply(batch_x); var loss = cross_entropy_loss(pred, batch_y); var acc = accuracy(pred, batch_y); print($"step: {step}, loss: {(float)loss}, accuracy: {(float)acc}"); } } // Test model on validation set. { x_test = x_test["::100"]; y_test = y_test["::100"]; var pred = conv_net.Apply(x_test); accuracy_test = (float)accuracy(pred, y_test); print($"Test Accuracy: {accuracy_test}"); } return(accuracy_test > 0.90); }
/// <summary> /// Error = 0.92 /// </summary> public static BackpropAlgorithm CreateMNISTSimpleDemo_SEALED() { Console.WriteLine("init CreateMNISTSimpleDemo_SEALED"); var activation = Activation.LeakyReLU(); var net = new ConvNet(1, 28) { IsTraining = true }; net.AddLayer(new ConvLayer(outputDepth: 12, windowSize: 5, padding: 2)); net.AddLayer(new ConvLayer(outputDepth: 12, windowSize: 5, padding: 2)); net.AddLayer(new MaxPoolingLayer(windowSize: 2, stride: 2, activation: activation)); net.AddLayer(new ConvLayer(outputDepth: 24, windowSize: 5, padding: 2)); net.AddLayer(new MaxPoolingLayer(windowSize: 2, stride: 2, activation: activation)); net.AddLayer(new FlattenLayer(outputDim: 32, activation: activation)); net.AddLayer(new DropoutLayer(rate: 0.5D)); net.AddLayer(new DenseLayer(outputDim: 10, activation: activation)); net._Build(); net.RandomizeParameters(seed: 0); var lrate = 0.001D; var alg = new BackpropAlgorithm(net) { EpochCount = 500, LearningRate = lrate, BatchSize = 4, UseBatchParallelization = true, MaxBatchThreadCount = 4, LossFunction = Loss.Euclidean, Optimizer = Optimizer.RMSProp, Regularizator = Regularizator.L2(0.0001D), LearningRateScheduler = LearningRateScheduler.DropBased(lrate, 5, 0.5D) }; alg.Build(); return(alg); }
public static BackpropAlgorithm CreateMNISTHardDemo() { Console.WriteLine("init CreateMNISTHardDemo"); var activation = Activation.ReLU; var net = new ConvNet(1, 28) { IsTraining = true }; net.AddLayer(new ConvLayer(outputDepth: 32, windowSize: 3, padding: 1, activation: activation)); net.AddLayer(new ConvLayer(outputDepth: 64, windowSize: 3, padding: 1, activation: activation)); net.AddLayer(new MaxPoolingLayer(windowSize: 2, stride: 2)); net.AddLayer(new DropoutLayer(0.25)); net.AddLayer(new FlattenLayer(outputDim: 128, activation: activation)); net.AddLayer(new DropoutLayer(0.5)); net.AddLayer(new FlattenLayer(outputDim: 10, activation: Activation.Exp)); net._Build(); net.RandomizeParameters(seed: 0); var lrate = 0.005D; var alg = new BackpropAlgorithm(net) { EpochCount = 50, LearningRate = lrate, BatchSize = 8, UseBatchParallelization = true, MaxBatchThreadCount = 8, LossFunction = Loss.CrossEntropySoftMax, Optimizer = Optimizer.RMSProp, LearningRateScheduler = LearningRateScheduler.DropBased(lrate, 5, 0.5D) }; alg.Build(); return(alg); }
public void Gradient_SimpleDropout_1Iter_Euclidean() { // arrange var net = new ConvNet(3, 1) { IsTraining = true }; net.AddLayer(new DenseLayer(outputDim: 10, activation: Activation.Atan)); net.AddLayer(new DropoutLayer(rate: 0.5D)); net.AddLayer(new DenseLayer(outputDim: 2, activation: Activation.Atan)); net._Build(); net.RandomizeParameters(seed: 0); var point1 = RandomPoint(3, 1, 1); var point2 = RandomPoint(3, 1, 1); // just for 2 dim output var sample = new ClassifiedSample <double[][, ]>(); sample[point1] = CLASSES[0]; sample[point2] = CLASSES[1]; var alg = new BackpropAlgorithm(net) { LearningRate = 0.1D, LossFunction = Loss.Euclidean }; alg.Build(); // act alg.RunIteration(point1, EXPECTED[0]); ((DropoutLayer)alg.Net[1]).ApplyCustomMask = true; // assert AssertNetGradient(alg, point1, EXPECTED[0]); }