public void ComputeTwiceGradientShouldYieldTheSameResult()
{
const int inputWidth = 10;
const int inputHeight = 10;
const int inputDepth = 2;
const int filterWidth = 3;
const int filterHeight = 3;
const int filterCount = 2;
// Create layer
var layer = new ConvLayer <double>(filterWidth, filterHeight, filterCount)
{
Stride = 2, BiasPref = 0.1
};
layer.Init(inputWidth, inputHeight, inputDepth);
// Forward pass
var input = BuilderInstance <double> .Volume.Random(new Shape(inputWidth, inputHeight, inputDepth));
var output = layer.DoForward(input, true);
// Set output gradients to 1
var outputGradient = BuilderInstance <double> .Volume.From(new double[output.Shape.TotalLength].Populate(1.0), output.Shape);
// Backward pass to retrieve gradients
layer.Backward(outputGradient);
var step1 = ((Volume <double>)layer.InputActivationGradients.Clone()).ToArray();
layer.Backward(outputGradient);
var step2 = ((Volume <double>)layer.InputActivationGradients.Clone()).ToArray();
Assert.IsTrue(step1.SequenceEqual(step2));
}
public static ConvLayer<T> Conv<T>(this LayerBase<T> layer, int width, int height, int filterCount) where T : struct, IEquatable<T>, IFormattable
{
var conv = new ConvLayer<T>(width, height, filterCount);
conv.AcceptParent(layer);
return conv;
}
public static ConvLayer <T> Conv <T>(this LayerBase <T> layer, int width, int height, int filterCount) where T : struct, IEquatable <T>, IFormattable
{
var conv = new ConvLayer <T>(width, height, filterCount);
layer.ConnectTo(conv);
return(conv);
}
public static ConvLayer Conv(this LayerBase layer, int width, int height, int filterCount)
{
var conv = new ConvLayer(width, height, filterCount);
layer.ConnectTo(conv);
return(conv);
}
public GRU(int _input_size, int _hidden_size, int output)
{
input_size = _input_size;
hidden_size = _hidden_size;
convLayerih = new ConvLayer(input_size, hidden_size * 3);
convLayerhh = new ConvLayer(hidden_size, hidden_size * 3);
convLayerhq = new ConvLayer(hidden_size, output);
}
private static Network CreateNewNetwork()
{
var num_inputs = 27; // 9 eyes, each sees 3 numbers (wall, green, red thing proximity)
var num_actions = 5; // 5 possible angles agent can turn
var temporal_window = 1; // amount of temporal memory. 0 = agent lives in-the-moment :)
var network_size = num_inputs * temporal_window + num_actions * temporal_window + num_inputs;
Network net = new Network();
InputLayer il = new InputLayer();
il.OutputWidth = 1;
il.OutputHeight = 1;
il.OutputDepth = network_size;
net.Layers.Add(il);
ConvLayer conv = new ConvLayer(16, 5, 5, il.OutputDepth, il.OutputWidth, il.OutputHeight, 1, 2, 0, 1, 0.1);
net.Layers.Add(conv);
ReluLayer rlv = new ReluLayer(conv.OutputDepth, conv.OutputWidth, conv.OutputHeight);
net.Layers.Add(rlv);
MaxPoolLayer pl = new MaxPoolLayer(2, 2, rlv.OutputDepth, rlv.OutputWidth, rlv.OutputHeight, 2, 0, 0);
net.Layers.Add(pl);
FullyConnLayer fc = new FullyConnLayer(50, pl.OutputDepth, pl.OutputWidth, pl.OutputHeight, 0, 1, 0);
net.Layers.Add(fc);
ReluLayer rl = new ReluLayer(fc.OutputDepth, fc.OutputWidth, fc.OutputHeight);
net.Layers.Add(rl);
FullyConnLayer fc2 = new FullyConnLayer(50, rl.OutputDepth, rl.OutputWidth, rl.OutputHeight, 0, 1, 0);
net.Layers.Add(fc2);
ReluLayer rl2 = new ReluLayer(fc2.OutputDepth, fc2.OutputWidth, fc2.OutputHeight);
net.Layers.Add(rl2);
FullyConnLayer fc8 = new FullyConnLayer(5, rl2.OutputDepth, rl2.OutputWidth, rl2.OutputHeight, 0, 1, 0);
net.Layers.Add(fc8);
RegressionLayer sl = new RegressionLayer(fc8.OutputDepth, fc8.OutputWidth, fc8.OutputHeight);
net.LossLayer = sl;
return(net);
}
private static Network CreateNewNetwork()
{
Network net = new Network();
InputLayer il = new InputLayer();
il.OutputWidth = 32;
il.OutputHeight = 32;
il.OutputDepth = 3;
net.Layers.Add(il);
ConvLayer conv = new ConvLayer(16, 5, 5, 3, 32, 32, 1, 2, 0, 1, 0.1);
net.Layers.Add(conv);
ReluLayer rl = new ReluLayer(conv.OutputDepth, conv.OutputWidth, conv.OutputHeight);
net.Layers.Add(rl);
MaxPoolLayer pl = new MaxPoolLayer(2, 2, rl.OutputDepth, rl.OutputWidth, rl.OutputHeight, 2, 0, 0);
net.Layers.Add(pl);
ConvLayer conv2 = new ConvLayer(20, 5, 5, pl.OutputDepth, pl.OutputWidth, pl.OutputHeight, 1, 2, 0, 1, 0.1);
net.Layers.Add(conv2);
ReluLayer rl2 = new ReluLayer(conv2.OutputDepth, conv2.OutputWidth, conv2.OutputHeight);
net.Layers.Add(rl2);
MaxPoolLayer pl2 = new MaxPoolLayer(2, 2, rl2.OutputDepth, rl2.OutputWidth, rl2.OutputHeight, 2, 0, 0);
net.Layers.Add(pl2);
ConvLayer conv3 = new ConvLayer(20, 5, 5, pl2.OutputDepth, pl2.OutputWidth, pl2.OutputHeight, 1, 2, 0, 1, 0.1);
net.Layers.Add(conv3);
ReluLayer rl3 = new ReluLayer(conv3.OutputDepth, conv3.OutputWidth, conv3.OutputHeight);
net.Layers.Add(rl3);
MaxPoolLayer pl3 = new MaxPoolLayer(2, 2, rl3.OutputDepth, rl3.OutputWidth, rl3.OutputHeight, 2, 0, 0);
net.Layers.Add(pl3);
FullyConnLayer fc = new FullyConnLayer(10, pl3.OutputDepth, pl3.OutputWidth, pl3.OutputHeight, 0, 1, 0);
net.Layers.Add(fc);
SoftmaxLayer sl = new SoftmaxLayer(fc.OutputDepth, fc.OutputWidth, fc.OutputHeight);
net.LossLayer = sl;
return(net);
}
public static LeakyReluLayer<T> LeakyRelu<T>(this ConvLayer<T> layer, T alpha) where T : struct, IEquatable<T>, IFormattable
{
var relu = new LeakyReluLayer<T>(alpha);
relu.AcceptParent(layer);
layer.BiasPref = (T)Convert.ChangeType(0.1, typeof(T)); // can we do better?
return relu;
}
public void BinaryNetSerializerTest()
{
var net = new Net();
net.AddLayer(new InputLayer(5, 5, 3));
var conv = new ConvLayer(2, 2, 16);
net.AddLayer(conv);
var fullycon = new FullyConnLayer(3);
net.AddLayer(fullycon);
net.AddLayer(new SoftmaxLayer(3));
// Serialize (binary)
using (var ms = new MemoryStream())
{
net.SaveBinary(ms);
ms.Position = 0;
// Deserialize (binary)
Net deserialized = SerializationExtensions.LoadBinary(ms) as Net;
// Make sure deserialized is identical to serialized
Assert.IsNotNull(deserialized.Layers);
Assert.AreEqual(net.Layers.Count, deserialized.Layers.Count);
Assert.IsTrue(net.Layers[0] is InputLayer);
var deserializedConv = net.Layers[1] as ConvLayer;
Assert.NotNull(deserializedConv);
Assert.NotNull(deserializedConv.Filters);
Assert.AreEqual(16, deserializedConv.Filters.Count);
for (int i = 0; i < deserializedConv.Filters.Count; i++)
{
for (int k = 0; k < deserializedConv.Filters[i].Length; k++)
{
Assert.AreEqual(conv.Filters[i].Get(k), deserializedConv.Filters[i].Get(k));
Assert.AreEqual(conv.Filters[i].GetGradient(k), deserializedConv.Filters[i].GetGradient(k));
}
}
var deserializedFullyCon = net.Layers[2] as FullyConnLayer;
Assert.NotNull(deserializedFullyCon);
Assert.NotNull(deserializedFullyCon.Filters);
Assert.AreEqual(3, deserializedFullyCon.Filters.Count);
for (int i = 0; i < deserializedFullyCon.Filters.Count; i++)
{
for (int k = 0; k < deserializedFullyCon.Filters[i].Length; k++)
{
Assert.AreEqual(fullycon.Filters[i].Get(k), deserializedFullyCon.Filters[i].Get(k));
Assert.AreEqual(fullycon.Filters[i].GetGradient(k), deserializedFullyCon.Filters[i].GetGradient(k));
}
}
Assert.IsTrue(deserialized.Layers[3] is SoftmaxLayer);
Assert.AreEqual(3, ((SoftmaxLayer)deserialized.Layers[3]).ClassCount);
}
}
public ConvSuperResolution()
{
superres_enc_front = InputLayer.Create(StartSide, 3);
superres_enc_back = ActivationLayer.Create <ReLU>();
var pooling_0 = PoolingLayer.Create(2, 2);
var pooling_1 = PoolingLayer.Create(2, 2);
var pooling_2 = PoolingLayer.Create(2, 2);
var pooling_3 = PoolingLayer.Create(2, 2);
superres_enc_front.Append(
ConvLayer.Create(5, 128, 2).Append( //o = 96
ActivationLayer.Create <ReLU>().Append(
pooling_0.Append( //o = 48
ConvLayer.Create(3, 128, 1).Append( //o = 48
ActivationLayer.Create <ReLU>().Append(
pooling_1.Append( //o = 24
ConvLayer.Create(3, 64, 1).Append( //o = 24
ActivationLayer.Create <ReLU>().Append(
pooling_2.Append( //o = 12
ConvLayer.Create(3, 32, 1).Append( //o = 12
ActivationLayer.Create <ReLU>().Append(
pooling_3.Append( //o = 6
ConvLayer.Create(3, 32, 1).Append( //o = 6
superres_enc_back
))))))))))))));
superres_dec_front = InputLayer.Create(6, 32);
superres_dec_back = ActivationLayer.Create <Tanh>();
superres_dec_front.Append(
ConvLayer.Create(3, 32, 1).Append( //o = 6
ActivationLayer.Create <ReLU>().Append(
UnpoolingLayer.Create(pooling_3).Append( //o = 12
ConvLayer.Create(3, 64, 1).Append( //o = 12
ActivationLayer.Create <ReLU>().Append(
UnpoolingLayer.Create(pooling_2).Append( //o = 24
ConvLayer.Create(3, 128, 1).Append( //o = 24
ActivationLayer.Create <ReLU>().Append(
UnpoolingLayer.Create(pooling_1).Append( //o = 48
ConvLayer.Create(3, 128, 1).Append( //o = 48
ActivationLayer.Create <ReLU>().Append(
UnpoolingLayer.Create(pooling_0).Append( //o = 96
ConvLayer.Create(5, 3, 2).Append( //o = 96
superres_dec_back
))))))))))))));
superres_enc_back.Append(superres_dec_front);
//TODO: come up with an approach that saves the convolution/multiplication indexes and rearranges the weights etc so they fit into cache better
//TODO: unpooling layer tied to pooling layers
//Initialize Weights
superres_enc_front.SetupInternalState();
superres_enc_front.InitializeWeights(new UniformWeightInitializer(0, 0.001f));
}
public void ForwardBackwardTest()
{
var input = new TensorOld(new double[]
{
4, 6, 1, 4,
8, 4, 5, 1,
5, 3, 5, 7,
1, 7, 2, 8,
}, 1, 1, 4, 4);
var conv = new ConvLayer(4, 2, 1, 1);
conv.SetFilters(new TensorOld(new double[]
{
1, 1, 0, 0,
0, 0, 1, 1,
1, 0, 1, 0,
0, 1, 0, 1,
}, 4, 1, 2, 2));
var expected = new TensorOld(new double[]
{
0, 0, 0, 0, 0,
4, 10, 7, 5, 4,
8, 12, 9, 6, 1,
5, 8, 8, 12, 7,
1, 8, 9, 10, 8,
4, 10, 7, 5, 4,
8, 12, 9, 6, 1,
5, 8, 8, 12, 7,
1, 8, 9, 10, 8,
0, 0, 0, 0, 0,
0, 4, 6, 1, 4,
0, 12, 10, 6, 5,
0, 13, 7, 10, 8,
0, 6, 10, 7, 15,
0, 1, 7, 2, 8,
4, 6, 1, 4, 0,
12, 10, 6, 5, 0,
13, 7, 10, 8, 0,
6, 10, 7, 15, 0,
1, 7, 2, 8, 0,
}, 1, 4, 5, 5);
conv.PrepareTrain(input);
var acutal = conv.Forward(input);
Assert.Equal(acutal, expected);
var error = expected / 10;
var back = conv.Backward(error);
}
public void JsonNetSerializerTest()
{
var net = new Net();
net.AddLayer(new InputLayer(5, 5, 3));
var conv = new ConvLayer(2, 2, 16);
net.AddLayer(conv);
var fullycon = new FullyConnLayer(3);
net.AddLayer(fullycon);
net.AddLayer(new SoftmaxLayer(3));
// Serialize to json
var json = net.ToJSON();
// Deserialize from json
Net deserialized = SerializationExtensions.FromJSON(json);
// Make sure deserialized is identical to serialized
Assert.IsNotNull(deserialized.Layers);
Assert.AreEqual(net.Layers.Count, deserialized.Layers.Count);
Assert.IsTrue(net.Layers[0] is InputLayer);
var deserializedConv = net.Layers[1] as ConvLayer;
Assert.NotNull(deserializedConv);
Assert.NotNull(deserializedConv.Filters);
Assert.AreEqual(16, deserializedConv.Filters.Count);
for (int i = 0; i < deserializedConv.Filters.Count; i++)
{
for (int k = 0; k < deserializedConv.Filters[i].Length; k++)
{
Assert.AreEqual(conv.Filters[i].Get(k), deserializedConv.Filters[i].Get(k));
Assert.AreEqual(conv.Filters[i].GetGradient(k), deserializedConv.Filters[i].GetGradient(k));
}
}
var deserializedFullyCon = net.Layers[2] as FullyConnLayer;
Assert.NotNull(deserializedFullyCon);
Assert.NotNull(deserializedFullyCon.Filters);
Assert.AreEqual(3, deserializedFullyCon.Filters.Count);
for (int i = 0; i < deserializedFullyCon.Filters.Count; i++)
{
for (int k = 0; k < deserializedFullyCon.Filters[i].Length; k++)
{
Assert.AreEqual(fullycon.Filters[i].Get(k), deserializedFullyCon.Filters[i].Get(k));
Assert.AreEqual(fullycon.Filters[i].GetGradient(k), deserializedFullyCon.Filters[i].GetGradient(k));
}
}
Assert.IsTrue(net.Layers[3] is SoftmaxLayer);
Assert.AreEqual(3, ((SoftmaxLayer)net.Layers[3]).ClassCount);
}
public static ReluLayer Relu(this ConvLayer layer)
{
var relu = new ReluLayer();
layer.ConnectTo(relu);
layer.BiasPref = 0.1;
layer.UpdateOutputSize();
return(relu);
}
public static ReluLayer <T> Relu <T>(this ConvLayer <T> layer) where T : struct, IEquatable <T>, IFormattable
{
var relu = new ReluLayer <T>();
layer.ConnectTo(relu);
layer.BiasPref = (T)Convert.ChangeType(0.1, typeof(T)); // can we do better?
layer.UpdateOutputSize();
return(relu);
}
public LSTMCELL(int _input_size, int _hidden_size)
{
input_size = _input_size;
hidden_size = _hidden_size;
convLayerih = new ConvLayer(input_size, hidden_size * 4);
//convLayerih.weights = JsonConvert.DeserializeObject<float[][]>(util.getstr("D:\\lstmihw.json"));
//convLayerih.basicData = JsonConvert.DeserializeObject<float[]>(util.getstr("D:\\lstmihb.json"));
convLayerhh = new ConvLayer(hidden_size, hidden_size * 4);
//convLayerhh.weights = JsonConvert.DeserializeObject<float[][]>(util.getstr("D:\\lstmhhw.json"));
//convLayerhh.basicData = JsonConvert.DeserializeObject<float[]>(util.getstr("D:\\lstmhhb.json"));
}
public void GradientWrtParametersCheck()
{
const int inputWidth = 10;
const int inputHeight = 10;
const int inputDepth = 2;
// Create layer
const int filterWidth = 3;
const int filterHeight = 3;
const int filterCount = 2;
var layer = new ConvLayer(filterWidth, filterHeight, filterCount) { Stride = 2 };
GradientCheckTools.GradienWrtParameterstCheck(inputWidth, inputHeight, inputDepth, layer);
}
public void GradientWrtInputCheck()
{
const int inputWidth = 30;
const int inputHeight = 30;
const int inputDepth = 2;
// Create layer
const int filterWidth = 3;
const int filterHeight = 3;
const int filterCount = 5;
var layer = new ConvLayer(filterWidth, filterHeight, filterCount) { Stride = 2};
GradientCheckTools.GradientCheck(layer, inputWidth, inputHeight, inputDepth);
}
public EvoAIndividual(int ID, ConvLayer[] clayers, int[] nlayers, int epochs, int batchSize, double errorTolerance, int generation)
{
this.ID = ID;
//Obtener las capas convolutivas (objetos)
CLayers = new ConvLayer[clayers.Length];
Array.Copy(clayers, CLayers, CLayers.Length);
//Obtener las capas neuronales (números, cada número es el total de neuronas y su posición la capa)
NLayers = new int[nlayers.Length];
Array.Copy(nlayers, NLayers, NLayers.Length);
//Obtener los demás parámetros
Epochs = epochs;
BatchSize = batchSize;
ErrorTolerance = errorTolerance;
Generation = generation;
}
public void BiasPrefUpdateWhenAddingReluLayer()
{
var net = new Net();
net.AddLayer(new InputLayer(10, 10, 3));
var dotProduct1 = new FullyConnLayer(5);
net.AddLayer(dotProduct1);
net.AddLayer(new ReluLayer());
var dotProduct2 = new ConvLayer(5, 5, 3);
net.AddLayer(dotProduct2);
net.AddLayer(new ReluLayer());
Assert.AreEqual(0.1, dotProduct1.BiasPref);
Assert.AreEqual(0.1, dotProduct2.BiasPref);
}
public void GradientWrtInputCheck()
{
const int inputWidth = 30;
const int inputHeight = 30;
const int inputDepth = 2;
// Create layer
const int filterWidth = 3;
const int filterHeight = 3;
const int filterCount = 5;
var layer = new ConvLayer(filterWidth, filterHeight, filterCount)
{
Stride = 2
};
GradientCheckTools.GradientCheck(layer, inputWidth, inputHeight, inputDepth);
}
public void GradientWrtParametersCheck()
{
const int inputWidth = 10;
const int inputHeight = 10;
const int inputDepth = 2;
// Create layer
const int filterWidth = 3;
const int filterHeight = 3;
const int filterCount = 2;
var layer = new ConvLayer(filterWidth, filterHeight, filterCount)
{
Stride = 2
};
GradientCheckTools.GradienWrtParameterstCheck(inputWidth, inputHeight, inputDepth, layer);
}
private ConvLayer simpleRectConvLayer()
{
var layer = new ConvLayer(outputDepth: 2,
windowHeight: 3, windowWidth: 2,
strideHeight: 1, strideWidth: 2,
paddingHeight: 1, paddingWidth: 0,
activation: Activation.Identity);
layer.InputHeight = 3;
layer.InputWidth = 4;
layer.InputDepth = 2;
layer.IsTraining = true;
layer._Build();
var weights = new double[]
{
// feature map #1
1, 0,
1, -1,
0, 1,
0, 1,
-1, 2,
1, -1,
1, // bias #1
// feature map #2
-1, 0,
-1, 1,
0, -1,
0, -1,
1, -2,
-1, 1,
-1, // bias #2
};
Array.Copy(weights, layer.Weights, layer.Weights.Length);
return(layer);
}
public void GradientWrtParametersCheck()
{
const int inputWidth = 10;
const int inputHeight = 10;
const int inputDepth = 2;
const int filterWidth = 3;
const int filterHeight = 3;
const int filterCount = 2;
const int batchSize = 1;
// Create layer
var layer = new ConvLayer <double>(filterWidth, filterHeight, filterCount)
{
Stride = 2
}; //BiasPref = 0.1
GradientCheckTools.GradienWrtParameterstCheck(inputWidth, inputHeight, inputDepth, batchSize, layer);
}
public void GradientWrtInputCheck()
{
const int inputWidth = 15;
const int inputHeight = 15;
const int inputDepth = 2;
const int filterWidth = 3;
const int filterHeight = 3;
const int filterCount = 5;
const int batchSize = 3;
// Create layer
var layer = new ConvLayer <double>(filterWidth, filterHeight, filterCount)
{
Stride = 2, BiasPref = 0.1
};
GradientCheckTools.GradientCheck(layer, inputWidth, inputHeight, inputDepth, batchSize);
}
public void ConvLayerStorageTest()
{
var doc = new XmlDocument();
var layer = new ConvLayer(3, 2, 1, 1);
layer.SetFilters(new TensorOld(
new double[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }, 3, 1, 2, 2));
layer.SetBias(new TensorOld(new double[] { 1, 2, 3 }));
var el = XmlStorage.SaveToEl(doc, layer);
var test = XmlStorage.LoadFromNode <ConvLayer>(el);
Assert.True(test is ConvLayer);
Assert.Equal(layer.PaddingValue, test.PaddingValue);
Assert.Equal(layer.FilterRows, test.FilterRows);
Assert.Equal(layer.FilterColumns, test.FilterColumns);
Assert.Equal(layer.RowStride, test.RowStride);
Assert.Equal(layer.ColumnStride, test.ColumnStride);
Assert.Equal(layer.Filters, test.Filters);
Assert.Equal(layer.Bias, test.Bias);
}
public int CreateCNN(int inpx = 32, int inpy = 32, int inpd = 1, int classesCount = 11)
{
if (GetLayersCount() == 0)
{
net.AddLayer(new InputLayer(inpx, inpy, 1));
convLayer = new ConvLayer(9, 9, 16)
{
};
net.AddLayer(this.convLayer);
net.AddLayer(new LeakyReluLayer(0.3));
net.AddLayer(new PoolLayer(2, 2)
{
}); //субдискр
net.AddLayer(new ConvLayer(5, 5, 32)
{
});
net.AddLayer(new LeakyReluLayer(0.3));
net.AddLayer(new PoolLayer(2, 2)
{
});
net.AddLayer(new FullyConnLayer(120));
net.AddLayer(new DropoutLayer(0.5));
net.AddLayer(new FullyConnLayer(50));
net.AddLayer(new DropoutLayer(0.5));
net.AddLayer(new FullyConnLayer(classesCount));
net.AddLayer(new SoftmaxLayer(classesCount));
classes = classesCount;
isNetLearned = false;
}
return(net.Layers.Count);
}
public void ConvLayerSerialization()
{
var layer = new ConvLayer(5, 5, 8)
{
Stride = 1, Pad = 2, BiasPref = 0.5
};
layer.Init(28, 24, 1);
var data = layer.GetData();
Assert.AreEqual(28, data["InputWidth"]);
Assert.AreEqual(24, data["InputHeight"]);
Assert.AreEqual(1, data["InputDepth"]);
var deserialized = LayerBase <double> .FromData(data) as ConvLayer;
Assert.IsNotNull(deserialized);
Assert.AreEqual(28, deserialized.InputWidth);
Assert.AreEqual(24, deserialized.InputHeight);
Assert.AreEqual(1, deserialized.InputDepth);
Assert.AreEqual(layer.Width, deserialized.Width);
Assert.AreEqual(layer.Height, deserialized.Height);
Assert.AreEqual(layer.Pad, deserialized.Pad);
Assert.AreEqual(layer.Stride, deserialized.Stride);
Assert.AreEqual(layer.FilterCount, deserialized.FilterCount);
Assert.AreEqual(layer.Filters.Shape, deserialized.Filters.Shape);
Assert.IsTrue(layer.Filters.ToArray().SequenceEqual(deserialized.Filters.ToArray()));
Assert.AreEqual(layer.Bias.Shape, deserialized.Bias.Shape);
Assert.IsTrue(layer.Bias.ToArray().SequenceEqual(deserialized.Bias.ToArray()));
Assert.AreEqual(layer.BiasPref, deserialized.BiasPref);
}
public static ConvLayer <T> Stride <T>(this ConvLayer <T> layer, int stride) where T : struct, IEquatable <T>, IFormattable
{
layer.Stride = stride;
return(layer);
}
public static ConvLayer <T> Pad <T>(this ConvLayer <T> layer, int pad) where T : struct, IEquatable <T>, IFormattable
{
layer.Pad = pad;
return(layer);
}
public ReversibleAutoencoder()
{
superres_enc_front = InputLayer.Create(StartSide, 3);
superres_enc_back = ActivationLayer.Create <LeakyReLU>();
var pooling_0 = PoolingLayer.Create(2, 2);
var pooling_1 = PoolingLayer.Create(2, 2);
var pooling_2 = PoolingLayer.Create(2, 2);
var pooling_3 = PoolingLayer.Create(2, 2);
superres_enc_front.Append(
ConvLayer.Create(3, 12).Append( //o = 94, 16
ActivationLayer.Create <LeakyReLU>().Append(
ConvLayer.Create(3, 12).Append( //o = 92, 16
ActivationLayer.Create <LeakyReLU>().Append(
ConvLayer.Create(3, 12).Append( //o = 90, 16
ActivationLayer.Create <LeakyReLU>().Append(
ConvLayer.Create(3, 12).Append( //o = 88, 16
ActivationLayer.Create <LeakyReLU>().Append(
pooling_0.Append( //o = 44, 16
ConvLayer.Create(3, 12).Append( //o = 42, 16
ActivationLayer.Create <LeakyReLU>().Append(
ConvLayer.Create(3, 12).Append( //o = 40, 16
ActivationLayer.Create <LeakyReLU>().Append(
ConvLayer.Create(3, 12).Append( //o = 38, 16
ActivationLayer.Create <LeakyReLU>().Append(
ConvLayer.Create(3, 12).Append( //o = 36, 16
ActivationLayer.Create <LeakyReLU>().Append(
ConvLayer.Create(3, 12).Append( //o = 34, 16
ActivationLayer.Create <LeakyReLU>().Append(
ConvLayer.Create(3, 12).Append( //o = 32, 16
ActivationLayer.Create <LeakyReLU>().Append(
ConvLayer.Create(3, 12).Append( //o = 30, 16
ActivationLayer.Create <LeakyReLU>().Append(
ConvLayer.Create(3, 12).Append( //o = 28, 16
ActivationLayer.Create <LeakyReLU>().Append(
pooling_2.Append( //o = 14, 16
ConvLayer.Create(3, 12).Append( //o = 12, 16
ActivationLayer.Create <LeakyReLU>().Append(
ConvLayer.Create(3, 12).Append( //o = 10, 16
ActivationLayer.Create <LeakyReLU>().Append(
ConvLayer.Create(3, 12).Append( //o = 8, 16
ActivationLayer.Create <LeakyReLU>().Append(
pooling_3.Append( //o = 4, 16
ConvLayer.Create(3, 8, 1).Append( //o = 4, 8
superres_enc_back
)))))))))))))))))))))))))))))))))));
superres_dec_front = InputLayer.Create(4, 8);
superres_dec_back = ActivationLayer.Create <Tanh>();
superres_dec_front.Append( //o = 4, 8
ConvLayer.Create(3, 8, 1).Append( //o = 4, 16
ActivationLayer.Create <LeakyReLU>().Append(
UnpoolingLayer.Create(pooling_3).Append( //o = 8, 16
ActivationLayer.Create <LeakyReLU>().Append(
ConvLayer.Create(3, 12, 2).Append( //o = 10, 16
ActivationLayer.Create <LeakyReLU>().Append(
ConvLayer.Create(3, 12, 2).Append( //o = 12, 16
ActivationLayer.Create <LeakyReLU>().Append(
ConvLayer.Create(3, 12, 2).Append( //o = 14, 16
UnpoolingLayer.Create(pooling_2).Append( //o = 28, 16
ActivationLayer.Create <LeakyReLU>().Append(
ConvLayer.Create(3, 12, 2).Append( //o = 30, 16
ActivationLayer.Create <LeakyReLU>().Append(
ConvLayer.Create(3, 12, 2).Append( //o = 32, 16
ActivationLayer.Create <LeakyReLU>().Append(
ConvLayer.Create(3, 12, 2).Append( //o = 34, 16
ActivationLayer.Create <LeakyReLU>().Append(
ConvLayer.Create(3, 12, 2).Append( //o = 36, 16
ActivationLayer.Create <LeakyReLU>().Append(
ConvLayer.Create(3, 12, 2).Append( //o = 38, 16
ActivationLayer.Create <LeakyReLU>().Append(
ConvLayer.Create(3, 12, 2).Append( //o = 40, 16
ActivationLayer.Create <LeakyReLU>().Append(
ConvLayer.Create(3, 12, 2).Append( //o = 42, 16
ActivationLayer.Create <LeakyReLU>().Append(
ConvLayer.Create(3, 12, 2).Append( //o = 44, 16
UnpoolingLayer.Create(pooling_0).Append( //o = 88, 16
ConvLayer.Create(3, 12, 2).Append( //o = 120, 16
ActivationLayer.Create <LeakyReLU>().Append(
ConvLayer.Create(3, 12, 2).Append( //o = 122, 16
ActivationLayer.Create <LeakyReLU>().Append(
ConvLayer.Create(3, 12, 2).Append( //o = 124, 16
ActivationLayer.Create <LeakyReLU>().Append(
ConvLayer.Create(3, 3, 2).Append( //o = 126, 3
superres_dec_back
)))))))))))))))))))))))))))))))))));
superres_enc_back.Append(superres_dec_front);
//TODO: come up with an approach that saves the convolution/multiplication indexes and rearranges the weights etc so they fit into cache better
//TODO: unpooling layer tied to pooling layers
//Initialize Weights
superres_enc_front.SetupInternalState();
superres_enc_front.InitializeWeights(new UniformWeightInitializer(0, 0.001f));
}
public void CompareCoreVsFlow()
{
var inputWidth = 28;
var inputHeigth = 28;
var inputDepth = 3;
var batchSize = 20;
#region Flow network
var netFlow = new Net <T>();
netFlow.AddLayer(new InputLayer <T>());
var convLayerFlow1 = new ConvLayer <T>(5, 5, 8)
{
BiasPref = (T)Convert.ChangeType(0.1, typeof(T)), Stride = 1, Pad = 2
};
netFlow.AddLayer(convLayerFlow1);
netFlow.AddLayer(new ReluLayer <T>());
netFlow.AddLayer(new PoolLayer <T>(2, 2)
{
Stride = 2
});
var fullyConnLayerFlow = new FullyConnLayer <T>(10);
netFlow.AddLayer(fullyConnLayerFlow);
netFlow.AddLayer(new SoftmaxLayer <T>());
var trainerFlow = new SgdTrainer <T>(netFlow, (T)Convert.ChangeType(0.01f, typeof(T)))
{
BatchSize = batchSize
};
#endregion
#region Core network
var netCore = new Core.Net <T>();
netCore.AddLayer(new Core.Layers.InputLayer <T>(inputWidth, inputHeigth, inputDepth));
var convLayerCore1 = new Core.Layers.ConvLayer <T>(5, 5, 8)
{
BiasPref = (T)Convert.ChangeType(0.1, typeof(T)), Stride = 1, Pad = 2
};
netCore.AddLayer(convLayerCore1);
netCore.AddLayer(new Core.Layers.ReluLayer <T>());
netCore.AddLayer(new Core.Layers.PoolLayer <T>(2, 2)
{
Stride = 2
});
var fullyConnLayerCore = new Core.Layers.FullyConnLayer <T>(10);
netCore.AddLayer(fullyConnLayerCore);
netCore.AddLayer(new Core.Layers.SoftmaxLayer <T>(10));
var trainerCore = new Core.Training.SgdTrainer <T>(netCore)
{
LearningRate = (T)Convert.ChangeType(0.01f, typeof(T)),
BatchSize = batchSize
};
#endregion
// Same weights
var convfilterCore1 = netFlow.Session.GetVariableByName(netFlow.Op, (convLayerFlow1.Filter as IPersistable <T>).Name);
convfilterCore1.Result = BuilderInstance <T> .Volume.SameAs(convLayerCore1.Filters.ToArray(), convLayerCore1.Filters.Shape);
var fullyfilterCore = netFlow.Session.GetVariableByName(netFlow.Op, (fullyConnLayerFlow.Filter as IPersistable <T>).Name);
fullyfilterCore.Result = BuilderInstance <T> .Volume.SameAs(fullyConnLayerCore.Filters.ToArray(), fullyConnLayerCore.Filters.Shape);
// Create input
var xStorage = new double[inputWidth * inputHeigth * inputDepth * batchSize].Populate(1.0);
var x = NewVolume(xStorage, Volume.Shape.From(inputWidth, inputHeigth, inputDepth, batchSize));
// Create output
var yStorage = new double[10 * batchSize];
var y = NewVolume(yStorage, Volume.Shape.From(1, 1, 10, batchSize));
for (var i = 0; i < batchSize; i++)
{
y.Set(0, 0, i % 10, i, Ops <T> .One);
}
for (var k = 0; k < 10; k++)
{
xStorage = new double[inputWidth * inputHeigth * inputDepth * batchSize].Populate(1.0 + k);
x = NewVolume(xStorage, Volume.Shape.From(inputWidth, inputHeigth, inputDepth, batchSize));
var flowResult = netFlow.Forward(x);
var coreResult = netCore.Forward(x);
var sum1 = BuilderInstance <T> .Volume.SameAs(new Shape(1));
flowResult.DoSum(sum1);
var sum2 = BuilderInstance <T> .Volume.SameAs(new Shape(1));
coreResult.DoSum(sum2);
var diff = Ops <T> .Subtract(sum1.Get(0), sum2.Get(0));
Console.WriteLine(diff);
AssertNumber.AreSequenceEqual(flowResult.ToArray(), coreResult.ToArray(), 1e-6);
trainerCore.Train(x, y);
trainerFlow.Train(x, y);
}
}
/// <summary>
/// BP网络测试
/// </summary>
static void BP()
{
float[][] x = JsonConvert.DeserializeObject <float[][]>(util.getstr("D:\\bpx.json")); //训练数据
float[][] y = JsonConvert.DeserializeObject <float[][]>(util.getstr("D:\\bpy.json")); //训练标签
float[][] w1 = JsonConvert.DeserializeObject <float[][]>(util.getstr("D:\\bpw.json"));
ConvLayer cl1 = new ConvLayer(13, 5, true);
cl1.weights = w1;
SigmodLayer sl = new SigmodLayer();
float lr = 0.5f;
ConvLayer cl2 = new ConvLayer(5, 1, true);
//SigmodLayer s2 = new SigmodLayer();
int i = 0, a = 0;
while (a < 5000)
{
//i = 0;
//while (i < 100)
//{
// float[][] xx2 = new float[1][];
// xx2[0] = new float[x[0].GetLength(0)];
// for (var f = 0; f < x[0].GetLength(0); f++)
// {
// xx2[0][f] = x[i][f];
// }
dynamic ff = cl1.Forward(x);
ff = sl.Forward(ff);
ff = cl2.Forward(ff);
// dynamic ff22 = s2.forward(ff);
//计算误差
MSELoss mloss = new MSELoss();
//float[][] yy2= new float[1][];
//yy2[0] = y[i];
var loss = mloss.Forward(ff, y);
Console.WriteLine("误差:" + loss);
dynamic grid = mloss.Backward();
//反传播w2
// dynamic grid2 =s2.backward(grid);
dynamic w22 = cl2.backweight(grid);
//反传播W1
dynamic grid1 = cl2.backward(grid);
grid1 = sl.Backward(grid1);
dynamic w11 = cl1.backweight(grid1);
cl2.weights = Matrix.MatrixSub(cl2.weights, Matrix.multiply(w22.grid, lr));
cl2.basicData = Matrix.MatrixSub(cl2.basicData, Matrix.multiply(w22.basic, lr));
cl1.weights = Matrix.MatrixSub(cl1.weights, Matrix.multiply(w11.grid, lr));
cl1.basicData = Matrix.MatrixSub(cl1.basicData, Matrix.multiply(w11.basic, lr));
i++;
// }
a++;
}
//测试网络
float[][] xx = new float[1][];
xx[0] = new float[x[0].GetLength(0)];
var aa = 3;
for (var f = 0; f < x[0].GetLength(0); f++)
{
xx[0][f] = x[aa][f];
}
dynamic ff2 = cl1.Forward(xx);
ff2 = sl.Forward(ff2);
ff2 = cl2.Forward(ff2);
util.prirt(ff2);
util.prirt(y[aa]);
}
