public void ComputeTwiceGradientShouldYieldTheSameResult() { const int inputWidth = 10; const int inputHeight = 10; const int inputDepth = 2; const int filterWidth = 3; const int filterHeight = 3; const int filterCount = 2; // Create layer var layer = new ConvLayer <double>(filterWidth, filterHeight, filterCount) { Stride = 2, BiasPref = 0.1 }; layer.Init(inputWidth, inputHeight, inputDepth); // Forward pass var input = BuilderInstance <double> .Volume.Random(new Shape(inputWidth, inputHeight, inputDepth)); var output = layer.DoForward(input, true); // Set output gradients to 1 var outputGradient = BuilderInstance <double> .Volume.From(new double[output.Shape.TotalLength].Populate(1.0), output.Shape); // Backward pass to retrieve gradients layer.Backward(outputGradient); var step1 = ((Volume <double>)layer.InputActivationGradients.Clone()).ToArray(); layer.Backward(outputGradient); var step2 = ((Volume <double>)layer.InputActivationGradients.Clone()).ToArray(); Assert.IsTrue(step1.SequenceEqual(step2)); }
public static ConvLayer<T> Conv<T>(this LayerBase<T> layer, int width, int height, int filterCount) where T : struct, IEquatable<T>, IFormattable { var conv = new ConvLayer<T>(width, height, filterCount); conv.AcceptParent(layer); return conv; }
public static ConvLayer <T> Conv <T>(this LayerBase <T> layer, int width, int height, int filterCount) where T : struct, IEquatable <T>, IFormattable { var conv = new ConvLayer <T>(width, height, filterCount); layer.ConnectTo(conv); return(conv); }
public static ConvLayer Conv(this LayerBase layer, int width, int height, int filterCount) { var conv = new ConvLayer(width, height, filterCount); layer.ConnectTo(conv); return(conv); }
public GRU(int _input_size, int _hidden_size, int output) { input_size = _input_size; hidden_size = _hidden_size; convLayerih = new ConvLayer(input_size, hidden_size * 3); convLayerhh = new ConvLayer(hidden_size, hidden_size * 3); convLayerhq = new ConvLayer(hidden_size, output); }
private static Network CreateNewNetwork() { var num_inputs = 27; // 9 eyes, each sees 3 numbers (wall, green, red thing proximity) var num_actions = 5; // 5 possible angles agent can turn var temporal_window = 1; // amount of temporal memory. 0 = agent lives in-the-moment :) var network_size = num_inputs * temporal_window + num_actions * temporal_window + num_inputs; Network net = new Network(); InputLayer il = new InputLayer(); il.OutputWidth = 1; il.OutputHeight = 1; il.OutputDepth = network_size; net.Layers.Add(il); ConvLayer conv = new ConvLayer(16, 5, 5, il.OutputDepth, il.OutputWidth, il.OutputHeight, 1, 2, 0, 1, 0.1); net.Layers.Add(conv); ReluLayer rlv = new ReluLayer(conv.OutputDepth, conv.OutputWidth, conv.OutputHeight); net.Layers.Add(rlv); MaxPoolLayer pl = new MaxPoolLayer(2, 2, rlv.OutputDepth, rlv.OutputWidth, rlv.OutputHeight, 2, 0, 0); net.Layers.Add(pl); FullyConnLayer fc = new FullyConnLayer(50, pl.OutputDepth, pl.OutputWidth, pl.OutputHeight, 0, 1, 0); net.Layers.Add(fc); ReluLayer rl = new ReluLayer(fc.OutputDepth, fc.OutputWidth, fc.OutputHeight); net.Layers.Add(rl); FullyConnLayer fc2 = new FullyConnLayer(50, rl.OutputDepth, rl.OutputWidth, rl.OutputHeight, 0, 1, 0); net.Layers.Add(fc2); ReluLayer rl2 = new ReluLayer(fc2.OutputDepth, fc2.OutputWidth, fc2.OutputHeight); net.Layers.Add(rl2); FullyConnLayer fc8 = new FullyConnLayer(5, rl2.OutputDepth, rl2.OutputWidth, rl2.OutputHeight, 0, 1, 0); net.Layers.Add(fc8); RegressionLayer sl = new RegressionLayer(fc8.OutputDepth, fc8.OutputWidth, fc8.OutputHeight); net.LossLayer = sl; return(net); }
private static Network CreateNewNetwork() { Network net = new Network(); InputLayer il = new InputLayer(); il.OutputWidth = 32; il.OutputHeight = 32; il.OutputDepth = 3; net.Layers.Add(il); ConvLayer conv = new ConvLayer(16, 5, 5, 3, 32, 32, 1, 2, 0, 1, 0.1); net.Layers.Add(conv); ReluLayer rl = new ReluLayer(conv.OutputDepth, conv.OutputWidth, conv.OutputHeight); net.Layers.Add(rl); MaxPoolLayer pl = new MaxPoolLayer(2, 2, rl.OutputDepth, rl.OutputWidth, rl.OutputHeight, 2, 0, 0); net.Layers.Add(pl); ConvLayer conv2 = new ConvLayer(20, 5, 5, pl.OutputDepth, pl.OutputWidth, pl.OutputHeight, 1, 2, 0, 1, 0.1); net.Layers.Add(conv2); ReluLayer rl2 = new ReluLayer(conv2.OutputDepth, conv2.OutputWidth, conv2.OutputHeight); net.Layers.Add(rl2); MaxPoolLayer pl2 = new MaxPoolLayer(2, 2, rl2.OutputDepth, rl2.OutputWidth, rl2.OutputHeight, 2, 0, 0); net.Layers.Add(pl2); ConvLayer conv3 = new ConvLayer(20, 5, 5, pl2.OutputDepth, pl2.OutputWidth, pl2.OutputHeight, 1, 2, 0, 1, 0.1); net.Layers.Add(conv3); ReluLayer rl3 = new ReluLayer(conv3.OutputDepth, conv3.OutputWidth, conv3.OutputHeight); net.Layers.Add(rl3); MaxPoolLayer pl3 = new MaxPoolLayer(2, 2, rl3.OutputDepth, rl3.OutputWidth, rl3.OutputHeight, 2, 0, 0); net.Layers.Add(pl3); FullyConnLayer fc = new FullyConnLayer(10, pl3.OutputDepth, pl3.OutputWidth, pl3.OutputHeight, 0, 1, 0); net.Layers.Add(fc); SoftmaxLayer sl = new SoftmaxLayer(fc.OutputDepth, fc.OutputWidth, fc.OutputHeight); net.LossLayer = sl; return(net); }
public static LeakyReluLayer<T> LeakyRelu<T>(this ConvLayer<T> layer, T alpha) where T : struct, IEquatable<T>, IFormattable { var relu = new LeakyReluLayer<T>(alpha); relu.AcceptParent(layer); layer.BiasPref = (T)Convert.ChangeType(0.1, typeof(T)); // can we do better? return relu; }
public void BinaryNetSerializerTest() { var net = new Net(); net.AddLayer(new InputLayer(5, 5, 3)); var conv = new ConvLayer(2, 2, 16); net.AddLayer(conv); var fullycon = new FullyConnLayer(3); net.AddLayer(fullycon); net.AddLayer(new SoftmaxLayer(3)); // Serialize (binary) using (var ms = new MemoryStream()) { net.SaveBinary(ms); ms.Position = 0; // Deserialize (binary) Net deserialized = SerializationExtensions.LoadBinary(ms) as Net; // Make sure deserialized is identical to serialized Assert.IsNotNull(deserialized.Layers); Assert.AreEqual(net.Layers.Count, deserialized.Layers.Count); Assert.IsTrue(net.Layers[0] is InputLayer); var deserializedConv = net.Layers[1] as ConvLayer; Assert.NotNull(deserializedConv); Assert.NotNull(deserializedConv.Filters); Assert.AreEqual(16, deserializedConv.Filters.Count); for (int i = 0; i < deserializedConv.Filters.Count; i++) { for (int k = 0; k < deserializedConv.Filters[i].Length; k++) { Assert.AreEqual(conv.Filters[i].Get(k), deserializedConv.Filters[i].Get(k)); Assert.AreEqual(conv.Filters[i].GetGradient(k), deserializedConv.Filters[i].GetGradient(k)); } } var deserializedFullyCon = net.Layers[2] as FullyConnLayer; Assert.NotNull(deserializedFullyCon); Assert.NotNull(deserializedFullyCon.Filters); Assert.AreEqual(3, deserializedFullyCon.Filters.Count); for (int i = 0; i < deserializedFullyCon.Filters.Count; i++) { for (int k = 0; k < deserializedFullyCon.Filters[i].Length; k++) { Assert.AreEqual(fullycon.Filters[i].Get(k), deserializedFullyCon.Filters[i].Get(k)); Assert.AreEqual(fullycon.Filters[i].GetGradient(k), deserializedFullyCon.Filters[i].GetGradient(k)); } } Assert.IsTrue(deserialized.Layers[3] is SoftmaxLayer); Assert.AreEqual(3, ((SoftmaxLayer)deserialized.Layers[3]).ClassCount); } }
public ConvSuperResolution() { superres_enc_front = InputLayer.Create(StartSide, 3); superres_enc_back = ActivationLayer.Create <ReLU>(); var pooling_0 = PoolingLayer.Create(2, 2); var pooling_1 = PoolingLayer.Create(2, 2); var pooling_2 = PoolingLayer.Create(2, 2); var pooling_3 = PoolingLayer.Create(2, 2); superres_enc_front.Append( ConvLayer.Create(5, 128, 2).Append( //o = 96 ActivationLayer.Create <ReLU>().Append( pooling_0.Append( //o = 48 ConvLayer.Create(3, 128, 1).Append( //o = 48 ActivationLayer.Create <ReLU>().Append( pooling_1.Append( //o = 24 ConvLayer.Create(3, 64, 1).Append( //o = 24 ActivationLayer.Create <ReLU>().Append( pooling_2.Append( //o = 12 ConvLayer.Create(3, 32, 1).Append( //o = 12 ActivationLayer.Create <ReLU>().Append( pooling_3.Append( //o = 6 ConvLayer.Create(3, 32, 1).Append( //o = 6 superres_enc_back )))))))))))))); superres_dec_front = InputLayer.Create(6, 32); superres_dec_back = ActivationLayer.Create <Tanh>(); superres_dec_front.Append( ConvLayer.Create(3, 32, 1).Append( //o = 6 ActivationLayer.Create <ReLU>().Append( UnpoolingLayer.Create(pooling_3).Append( //o = 12 ConvLayer.Create(3, 64, 1).Append( //o = 12 ActivationLayer.Create <ReLU>().Append( UnpoolingLayer.Create(pooling_2).Append( //o = 24 ConvLayer.Create(3, 128, 1).Append( //o = 24 ActivationLayer.Create <ReLU>().Append( UnpoolingLayer.Create(pooling_1).Append( //o = 48 ConvLayer.Create(3, 128, 1).Append( //o = 48 ActivationLayer.Create <ReLU>().Append( UnpoolingLayer.Create(pooling_0).Append( //o = 96 ConvLayer.Create(5, 3, 2).Append( //o = 96 superres_dec_back )))))))))))))); superres_enc_back.Append(superres_dec_front); //TODO: come up with an approach that saves the convolution/multiplication indexes and rearranges the weights etc so they fit into cache better //TODO: unpooling layer tied to pooling layers //Initialize Weights superres_enc_front.SetupInternalState(); superres_enc_front.InitializeWeights(new UniformWeightInitializer(0, 0.001f)); }
public void ForwardBackwardTest() { var input = new TensorOld(new double[] { 4, 6, 1, 4, 8, 4, 5, 1, 5, 3, 5, 7, 1, 7, 2, 8, }, 1, 1, 4, 4); var conv = new ConvLayer(4, 2, 1, 1); conv.SetFilters(new TensorOld(new double[] { 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, }, 4, 1, 2, 2)); var expected = new TensorOld(new double[] { 0, 0, 0, 0, 0, 4, 10, 7, 5, 4, 8, 12, 9, 6, 1, 5, 8, 8, 12, 7, 1, 8, 9, 10, 8, 4, 10, 7, 5, 4, 8, 12, 9, 6, 1, 5, 8, 8, 12, 7, 1, 8, 9, 10, 8, 0, 0, 0, 0, 0, 0, 4, 6, 1, 4, 0, 12, 10, 6, 5, 0, 13, 7, 10, 8, 0, 6, 10, 7, 15, 0, 1, 7, 2, 8, 4, 6, 1, 4, 0, 12, 10, 6, 5, 0, 13, 7, 10, 8, 0, 6, 10, 7, 15, 0, 1, 7, 2, 8, 0, }, 1, 4, 5, 5); conv.PrepareTrain(input); var acutal = conv.Forward(input); Assert.Equal(acutal, expected); var error = expected / 10; var back = conv.Backward(error); }
public void JsonNetSerializerTest() { var net = new Net(); net.AddLayer(new InputLayer(5, 5, 3)); var conv = new ConvLayer(2, 2, 16); net.AddLayer(conv); var fullycon = new FullyConnLayer(3); net.AddLayer(fullycon); net.AddLayer(new SoftmaxLayer(3)); // Serialize to json var json = net.ToJSON(); // Deserialize from json Net deserialized = SerializationExtensions.FromJSON(json); // Make sure deserialized is identical to serialized Assert.IsNotNull(deserialized.Layers); Assert.AreEqual(net.Layers.Count, deserialized.Layers.Count); Assert.IsTrue(net.Layers[0] is InputLayer); var deserializedConv = net.Layers[1] as ConvLayer; Assert.NotNull(deserializedConv); Assert.NotNull(deserializedConv.Filters); Assert.AreEqual(16, deserializedConv.Filters.Count); for (int i = 0; i < deserializedConv.Filters.Count; i++) { for (int k = 0; k < deserializedConv.Filters[i].Length; k++) { Assert.AreEqual(conv.Filters[i].Get(k), deserializedConv.Filters[i].Get(k)); Assert.AreEqual(conv.Filters[i].GetGradient(k), deserializedConv.Filters[i].GetGradient(k)); } } var deserializedFullyCon = net.Layers[2] as FullyConnLayer; Assert.NotNull(deserializedFullyCon); Assert.NotNull(deserializedFullyCon.Filters); Assert.AreEqual(3, deserializedFullyCon.Filters.Count); for (int i = 0; i < deserializedFullyCon.Filters.Count; i++) { for (int k = 0; k < deserializedFullyCon.Filters[i].Length; k++) { Assert.AreEqual(fullycon.Filters[i].Get(k), deserializedFullyCon.Filters[i].Get(k)); Assert.AreEqual(fullycon.Filters[i].GetGradient(k), deserializedFullyCon.Filters[i].GetGradient(k)); } } Assert.IsTrue(net.Layers[3] is SoftmaxLayer); Assert.AreEqual(3, ((SoftmaxLayer)net.Layers[3]).ClassCount); }
public static ReluLayer Relu(this ConvLayer layer) { var relu = new ReluLayer(); layer.ConnectTo(relu); layer.BiasPref = 0.1; layer.UpdateOutputSize(); return(relu); }
public static ReluLayer <T> Relu <T>(this ConvLayer <T> layer) where T : struct, IEquatable <T>, IFormattable { var relu = new ReluLayer <T>(); layer.ConnectTo(relu); layer.BiasPref = (T)Convert.ChangeType(0.1, typeof(T)); // can we do better? layer.UpdateOutputSize(); return(relu); }
public LSTMCELL(int _input_size, int _hidden_size) { input_size = _input_size; hidden_size = _hidden_size; convLayerih = new ConvLayer(input_size, hidden_size * 4); //convLayerih.weights = JsonConvert.DeserializeObject<float[][]>(util.getstr("D:\\lstmihw.json")); //convLayerih.basicData = JsonConvert.DeserializeObject<float[]>(util.getstr("D:\\lstmihb.json")); convLayerhh = new ConvLayer(hidden_size, hidden_size * 4); //convLayerhh.weights = JsonConvert.DeserializeObject<float[][]>(util.getstr("D:\\lstmhhw.json")); //convLayerhh.basicData = JsonConvert.DeserializeObject<float[]>(util.getstr("D:\\lstmhhb.json")); }
public void GradientWrtParametersCheck() { const int inputWidth = 10; const int inputHeight = 10; const int inputDepth = 2; // Create layer const int filterWidth = 3; const int filterHeight = 3; const int filterCount = 2; var layer = new ConvLayer(filterWidth, filterHeight, filterCount) { Stride = 2 }; GradientCheckTools.GradienWrtParameterstCheck(inputWidth, inputHeight, inputDepth, layer); }
public void GradientWrtInputCheck() { const int inputWidth = 30; const int inputHeight = 30; const int inputDepth = 2; // Create layer const int filterWidth = 3; const int filterHeight = 3; const int filterCount = 5; var layer = new ConvLayer(filterWidth, filterHeight, filterCount) { Stride = 2}; GradientCheckTools.GradientCheck(layer, inputWidth, inputHeight, inputDepth); }
public EvoAIndividual(int ID, ConvLayer[] clayers, int[] nlayers, int epochs, int batchSize, double errorTolerance, int generation) { this.ID = ID; //Obtener las capas convolutivas (objetos) CLayers = new ConvLayer[clayers.Length]; Array.Copy(clayers, CLayers, CLayers.Length); //Obtener las capas neuronales (números, cada número es el total de neuronas y su posición la capa) NLayers = new int[nlayers.Length]; Array.Copy(nlayers, NLayers, NLayers.Length); //Obtener los demás parámetros Epochs = epochs; BatchSize = batchSize; ErrorTolerance = errorTolerance; Generation = generation; }
public void BiasPrefUpdateWhenAddingReluLayer() { var net = new Net(); net.AddLayer(new InputLayer(10, 10, 3)); var dotProduct1 = new FullyConnLayer(5); net.AddLayer(dotProduct1); net.AddLayer(new ReluLayer()); var dotProduct2 = new ConvLayer(5, 5, 3); net.AddLayer(dotProduct2); net.AddLayer(new ReluLayer()); Assert.AreEqual(0.1, dotProduct1.BiasPref); Assert.AreEqual(0.1, dotProduct2.BiasPref); }
public void GradientWrtInputCheck() { const int inputWidth = 30; const int inputHeight = 30; const int inputDepth = 2; // Create layer const int filterWidth = 3; const int filterHeight = 3; const int filterCount = 5; var layer = new ConvLayer(filterWidth, filterHeight, filterCount) { Stride = 2 }; GradientCheckTools.GradientCheck(layer, inputWidth, inputHeight, inputDepth); }
private ConvLayer simpleRectConvLayer() { var layer = new ConvLayer(outputDepth: 2, windowHeight: 3, windowWidth: 2, strideHeight: 1, strideWidth: 2, paddingHeight: 1, paddingWidth: 0, activation: Activation.Identity); layer.InputHeight = 3; layer.InputWidth = 4; layer.InputDepth = 2; layer.IsTraining = true; layer._Build(); var weights = new double[] { // feature map #1 1, 0, 1, -1, 0, 1, 0, 1, -1, 2, 1, -1, 1, // bias #1 // feature map #2 -1, 0, -1, 1, 0, -1, 0, -1, 1, -2, -1, 1, -1, // bias #2 }; Array.Copy(weights, layer.Weights, layer.Weights.Length); return(layer); }
public void GradientWrtParametersCheck() { const int inputWidth = 10; const int inputHeight = 10; const int inputDepth = 2; const int filterWidth = 3; const int filterHeight = 3; const int filterCount = 2; const int batchSize = 1; // Create layer var layer = new ConvLayer <double>(filterWidth, filterHeight, filterCount) { Stride = 2 }; //BiasPref = 0.1 GradientCheckTools.GradienWrtParameterstCheck(inputWidth, inputHeight, inputDepth, batchSize, layer); }
public void GradientWrtInputCheck() { const int inputWidth = 15; const int inputHeight = 15; const int inputDepth = 2; const int filterWidth = 3; const int filterHeight = 3; const int filterCount = 5; const int batchSize = 3; // Create layer var layer = new ConvLayer <double>(filterWidth, filterHeight, filterCount) { Stride = 2, BiasPref = 0.1 }; GradientCheckTools.GradientCheck(layer, inputWidth, inputHeight, inputDepth, batchSize); }
public void ConvLayerStorageTest() { var doc = new XmlDocument(); var layer = new ConvLayer(3, 2, 1, 1); layer.SetFilters(new TensorOld( new double[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }, 3, 1, 2, 2)); layer.SetBias(new TensorOld(new double[] { 1, 2, 3 })); var el = XmlStorage.SaveToEl(doc, layer); var test = XmlStorage.LoadFromNode <ConvLayer>(el); Assert.True(test is ConvLayer); Assert.Equal(layer.PaddingValue, test.PaddingValue); Assert.Equal(layer.FilterRows, test.FilterRows); Assert.Equal(layer.FilterColumns, test.FilterColumns); Assert.Equal(layer.RowStride, test.RowStride); Assert.Equal(layer.ColumnStride, test.ColumnStride); Assert.Equal(layer.Filters, test.Filters); Assert.Equal(layer.Bias, test.Bias); }
public int CreateCNN(int inpx = 32, int inpy = 32, int inpd = 1, int classesCount = 11) { if (GetLayersCount() == 0) { net.AddLayer(new InputLayer(inpx, inpy, 1)); convLayer = new ConvLayer(9, 9, 16) { }; net.AddLayer(this.convLayer); net.AddLayer(new LeakyReluLayer(0.3)); net.AddLayer(new PoolLayer(2, 2) { }); //субдискр net.AddLayer(new ConvLayer(5, 5, 32) { }); net.AddLayer(new LeakyReluLayer(0.3)); net.AddLayer(new PoolLayer(2, 2) { }); net.AddLayer(new FullyConnLayer(120)); net.AddLayer(new DropoutLayer(0.5)); net.AddLayer(new FullyConnLayer(50)); net.AddLayer(new DropoutLayer(0.5)); net.AddLayer(new FullyConnLayer(classesCount)); net.AddLayer(new SoftmaxLayer(classesCount)); classes = classesCount; isNetLearned = false; } return(net.Layers.Count); }
public void ConvLayerSerialization() { var layer = new ConvLayer(5, 5, 8) { Stride = 1, Pad = 2, BiasPref = 0.5 }; layer.Init(28, 24, 1); var data = layer.GetData(); Assert.AreEqual(28, data["InputWidth"]); Assert.AreEqual(24, data["InputHeight"]); Assert.AreEqual(1, data["InputDepth"]); var deserialized = LayerBase <double> .FromData(data) as ConvLayer; Assert.IsNotNull(deserialized); Assert.AreEqual(28, deserialized.InputWidth); Assert.AreEqual(24, deserialized.InputHeight); Assert.AreEqual(1, deserialized.InputDepth); Assert.AreEqual(layer.Width, deserialized.Width); Assert.AreEqual(layer.Height, deserialized.Height); Assert.AreEqual(layer.Pad, deserialized.Pad); Assert.AreEqual(layer.Stride, deserialized.Stride); Assert.AreEqual(layer.FilterCount, deserialized.FilterCount); Assert.AreEqual(layer.Filters.Shape, deserialized.Filters.Shape); Assert.IsTrue(layer.Filters.ToArray().SequenceEqual(deserialized.Filters.ToArray())); Assert.AreEqual(layer.Bias.Shape, deserialized.Bias.Shape); Assert.IsTrue(layer.Bias.ToArray().SequenceEqual(deserialized.Bias.ToArray())); Assert.AreEqual(layer.BiasPref, deserialized.BiasPref); }
public static ConvLayer <T> Stride <T>(this ConvLayer <T> layer, int stride) where T : struct, IEquatable <T>, IFormattable { layer.Stride = stride; return(layer); }
public static ConvLayer <T> Pad <T>(this ConvLayer <T> layer, int pad) where T : struct, IEquatable <T>, IFormattable { layer.Pad = pad; return(layer); }
public ReversibleAutoencoder() { superres_enc_front = InputLayer.Create(StartSide, 3); superres_enc_back = ActivationLayer.Create <LeakyReLU>(); var pooling_0 = PoolingLayer.Create(2, 2); var pooling_1 = PoolingLayer.Create(2, 2); var pooling_2 = PoolingLayer.Create(2, 2); var pooling_3 = PoolingLayer.Create(2, 2); superres_enc_front.Append( ConvLayer.Create(3, 12).Append( //o = 94, 16 ActivationLayer.Create <LeakyReLU>().Append( ConvLayer.Create(3, 12).Append( //o = 92, 16 ActivationLayer.Create <LeakyReLU>().Append( ConvLayer.Create(3, 12).Append( //o = 90, 16 ActivationLayer.Create <LeakyReLU>().Append( ConvLayer.Create(3, 12).Append( //o = 88, 16 ActivationLayer.Create <LeakyReLU>().Append( pooling_0.Append( //o = 44, 16 ConvLayer.Create(3, 12).Append( //o = 42, 16 ActivationLayer.Create <LeakyReLU>().Append( ConvLayer.Create(3, 12).Append( //o = 40, 16 ActivationLayer.Create <LeakyReLU>().Append( ConvLayer.Create(3, 12).Append( //o = 38, 16 ActivationLayer.Create <LeakyReLU>().Append( ConvLayer.Create(3, 12).Append( //o = 36, 16 ActivationLayer.Create <LeakyReLU>().Append( ConvLayer.Create(3, 12).Append( //o = 34, 16 ActivationLayer.Create <LeakyReLU>().Append( ConvLayer.Create(3, 12).Append( //o = 32, 16 ActivationLayer.Create <LeakyReLU>().Append( ConvLayer.Create(3, 12).Append( //o = 30, 16 ActivationLayer.Create <LeakyReLU>().Append( ConvLayer.Create(3, 12).Append( //o = 28, 16 ActivationLayer.Create <LeakyReLU>().Append( pooling_2.Append( //o = 14, 16 ConvLayer.Create(3, 12).Append( //o = 12, 16 ActivationLayer.Create <LeakyReLU>().Append( ConvLayer.Create(3, 12).Append( //o = 10, 16 ActivationLayer.Create <LeakyReLU>().Append( ConvLayer.Create(3, 12).Append( //o = 8, 16 ActivationLayer.Create <LeakyReLU>().Append( pooling_3.Append( //o = 4, 16 ConvLayer.Create(3, 8, 1).Append( //o = 4, 8 superres_enc_back ))))))))))))))))))))))))))))))))))); superres_dec_front = InputLayer.Create(4, 8); superres_dec_back = ActivationLayer.Create <Tanh>(); superres_dec_front.Append( //o = 4, 8 ConvLayer.Create(3, 8, 1).Append( //o = 4, 16 ActivationLayer.Create <LeakyReLU>().Append( UnpoolingLayer.Create(pooling_3).Append( //o = 8, 16 ActivationLayer.Create <LeakyReLU>().Append( ConvLayer.Create(3, 12, 2).Append( //o = 10, 16 ActivationLayer.Create <LeakyReLU>().Append( ConvLayer.Create(3, 12, 2).Append( //o = 12, 16 ActivationLayer.Create <LeakyReLU>().Append( ConvLayer.Create(3, 12, 2).Append( //o = 14, 16 UnpoolingLayer.Create(pooling_2).Append( //o = 28, 16 ActivationLayer.Create <LeakyReLU>().Append( ConvLayer.Create(3, 12, 2).Append( //o = 30, 16 ActivationLayer.Create <LeakyReLU>().Append( ConvLayer.Create(3, 12, 2).Append( //o = 32, 16 ActivationLayer.Create <LeakyReLU>().Append( ConvLayer.Create(3, 12, 2).Append( //o = 34, 16 ActivationLayer.Create <LeakyReLU>().Append( ConvLayer.Create(3, 12, 2).Append( //o = 36, 16 ActivationLayer.Create <LeakyReLU>().Append( ConvLayer.Create(3, 12, 2).Append( //o = 38, 16 ActivationLayer.Create <LeakyReLU>().Append( ConvLayer.Create(3, 12, 2).Append( //o = 40, 16 ActivationLayer.Create <LeakyReLU>().Append( ConvLayer.Create(3, 12, 2).Append( //o = 42, 16 ActivationLayer.Create <LeakyReLU>().Append( ConvLayer.Create(3, 12, 2).Append( //o = 44, 16 UnpoolingLayer.Create(pooling_0).Append( //o = 88, 16 ConvLayer.Create(3, 12, 2).Append( //o = 120, 16 ActivationLayer.Create <LeakyReLU>().Append( ConvLayer.Create(3, 12, 2).Append( //o = 122, 16 ActivationLayer.Create <LeakyReLU>().Append( ConvLayer.Create(3, 12, 2).Append( //o = 124, 16 ActivationLayer.Create <LeakyReLU>().Append( ConvLayer.Create(3, 3, 2).Append( //o = 126, 3 superres_dec_back ))))))))))))))))))))))))))))))))))); superres_enc_back.Append(superres_dec_front); //TODO: come up with an approach that saves the convolution/multiplication indexes and rearranges the weights etc so they fit into cache better //TODO: unpooling layer tied to pooling layers //Initialize Weights superres_enc_front.SetupInternalState(); superres_enc_front.InitializeWeights(new UniformWeightInitializer(0, 0.001f)); }
public void CompareCoreVsFlow() { var inputWidth = 28; var inputHeigth = 28; var inputDepth = 3; var batchSize = 20; #region Flow network var netFlow = new Net <T>(); netFlow.AddLayer(new InputLayer <T>()); var convLayerFlow1 = new ConvLayer <T>(5, 5, 8) { BiasPref = (T)Convert.ChangeType(0.1, typeof(T)), Stride = 1, Pad = 2 }; netFlow.AddLayer(convLayerFlow1); netFlow.AddLayer(new ReluLayer <T>()); netFlow.AddLayer(new PoolLayer <T>(2, 2) { Stride = 2 }); var fullyConnLayerFlow = new FullyConnLayer <T>(10); netFlow.AddLayer(fullyConnLayerFlow); netFlow.AddLayer(new SoftmaxLayer <T>()); var trainerFlow = new SgdTrainer <T>(netFlow, (T)Convert.ChangeType(0.01f, typeof(T))) { BatchSize = batchSize }; #endregion #region Core network var netCore = new Core.Net <T>(); netCore.AddLayer(new Core.Layers.InputLayer <T>(inputWidth, inputHeigth, inputDepth)); var convLayerCore1 = new Core.Layers.ConvLayer <T>(5, 5, 8) { BiasPref = (T)Convert.ChangeType(0.1, typeof(T)), Stride = 1, Pad = 2 }; netCore.AddLayer(convLayerCore1); netCore.AddLayer(new Core.Layers.ReluLayer <T>()); netCore.AddLayer(new Core.Layers.PoolLayer <T>(2, 2) { Stride = 2 }); var fullyConnLayerCore = new Core.Layers.FullyConnLayer <T>(10); netCore.AddLayer(fullyConnLayerCore); netCore.AddLayer(new Core.Layers.SoftmaxLayer <T>(10)); var trainerCore = new Core.Training.SgdTrainer <T>(netCore) { LearningRate = (T)Convert.ChangeType(0.01f, typeof(T)), BatchSize = batchSize }; #endregion // Same weights var convfilterCore1 = netFlow.Session.GetVariableByName(netFlow.Op, (convLayerFlow1.Filter as IPersistable <T>).Name); convfilterCore1.Result = BuilderInstance <T> .Volume.SameAs(convLayerCore1.Filters.ToArray(), convLayerCore1.Filters.Shape); var fullyfilterCore = netFlow.Session.GetVariableByName(netFlow.Op, (fullyConnLayerFlow.Filter as IPersistable <T>).Name); fullyfilterCore.Result = BuilderInstance <T> .Volume.SameAs(fullyConnLayerCore.Filters.ToArray(), fullyConnLayerCore.Filters.Shape); // Create input var xStorage = new double[inputWidth * inputHeigth * inputDepth * batchSize].Populate(1.0); var x = NewVolume(xStorage, Volume.Shape.From(inputWidth, inputHeigth, inputDepth, batchSize)); // Create output var yStorage = new double[10 * batchSize]; var y = NewVolume(yStorage, Volume.Shape.From(1, 1, 10, batchSize)); for (var i = 0; i < batchSize; i++) { y.Set(0, 0, i % 10, i, Ops <T> .One); } for (var k = 0; k < 10; k++) { xStorage = new double[inputWidth * inputHeigth * inputDepth * batchSize].Populate(1.0 + k); x = NewVolume(xStorage, Volume.Shape.From(inputWidth, inputHeigth, inputDepth, batchSize)); var flowResult = netFlow.Forward(x); var coreResult = netCore.Forward(x); var sum1 = BuilderInstance <T> .Volume.SameAs(new Shape(1)); flowResult.DoSum(sum1); var sum2 = BuilderInstance <T> .Volume.SameAs(new Shape(1)); coreResult.DoSum(sum2); var diff = Ops <T> .Subtract(sum1.Get(0), sum2.Get(0)); Console.WriteLine(diff); AssertNumber.AreSequenceEqual(flowResult.ToArray(), coreResult.ToArray(), 1e-6); trainerCore.Train(x, y); trainerFlow.Train(x, y); } }
/// <summary> /// BP网络测试 /// </summary> static void BP() { float[][] x = JsonConvert.DeserializeObject <float[][]>(util.getstr("D:\\bpx.json")); //训练数据 float[][] y = JsonConvert.DeserializeObject <float[][]>(util.getstr("D:\\bpy.json")); //训练标签 float[][] w1 = JsonConvert.DeserializeObject <float[][]>(util.getstr("D:\\bpw.json")); ConvLayer cl1 = new ConvLayer(13, 5, true); cl1.weights = w1; SigmodLayer sl = new SigmodLayer(); float lr = 0.5f; ConvLayer cl2 = new ConvLayer(5, 1, true); //SigmodLayer s2 = new SigmodLayer(); int i = 0, a = 0; while (a < 5000) { //i = 0; //while (i < 100) //{ // float[][] xx2 = new float[1][]; // xx2[0] = new float[x[0].GetLength(0)]; // for (var f = 0; f < x[0].GetLength(0); f++) // { // xx2[0][f] = x[i][f]; // } dynamic ff = cl1.Forward(x); ff = sl.Forward(ff); ff = cl2.Forward(ff); // dynamic ff22 = s2.forward(ff); //计算误差 MSELoss mloss = new MSELoss(); //float[][] yy2= new float[1][]; //yy2[0] = y[i]; var loss = mloss.Forward(ff, y); Console.WriteLine("误差:" + loss); dynamic grid = mloss.Backward(); //反传播w2 // dynamic grid2 =s2.backward(grid); dynamic w22 = cl2.backweight(grid); //反传播W1 dynamic grid1 = cl2.backward(grid); grid1 = sl.Backward(grid1); dynamic w11 = cl1.backweight(grid1); cl2.weights = Matrix.MatrixSub(cl2.weights, Matrix.multiply(w22.grid, lr)); cl2.basicData = Matrix.MatrixSub(cl2.basicData, Matrix.multiply(w22.basic, lr)); cl1.weights = Matrix.MatrixSub(cl1.weights, Matrix.multiply(w11.grid, lr)); cl1.basicData = Matrix.MatrixSub(cl1.basicData, Matrix.multiply(w11.basic, lr)); i++; // } a++; } //测试网络 float[][] xx = new float[1][]; xx[0] = new float[x[0].GetLength(0)]; var aa = 3; for (var f = 0; f < x[0].GetLength(0); f++) { xx[0][f] = x[aa][f]; } dynamic ff2 = cl1.Forward(xx); ff2 = sl.Forward(ff2); ff2 = cl2.Forward(ff2); util.prirt(ff2); util.prirt(y[aa]); }