public void ConvolutionBackward()
{
ConvolutionalLayer
cpu = new ConvolutionalLayer(new TensorInfo(58, 58, 3), ConvolutionInfo.Default, (5, 5), 20, ActivationType.LeCunTanh, BiasInitializationMode.Gaussian),
gpu = new CuDnnConvolutionalLayer(cpu.InputInfo, ConvolutionInfo.Default, cpu.KernelInfo, cpu.OutputInfo, cpu.Weights, cpu.Biases, ActivationType.LeCunTanh);
TestBackward(cpu, gpu, 127);
}
public void ConvolutionForward()
{
float[,] x = WeightsProvider.NewFullyConnectedWeights(TensorInfo.Linear(127), 58 * 58 * 3, WeightsInitializationMode.GlorotNormal).AsSpan().AsMatrix(127, 58 * 58 * 3);
ConvolutionalLayer
cpu = new ConvolutionalLayer(new TensorInfo(58, 58, 3), ConvolutionInfo.Default, (5, 5), 20, ActivationFunctionType.LeakyReLU, BiasInitializationMode.Gaussian),
gpu = new CuDnnConvolutionalLayer(cpu.InputInfo, ConvolutionInfo.Default, cpu.KernelInfo, cpu.OutputInfo, cpu.Weights, cpu.Biases, cpu.ActivationFunctionType);
TestForward(cpu, gpu, x);
}
/// <summary>
/// Инициализация слоёв нейронной сети.
/// </summary>
/// <param name="listOfPicturesmatrix">Список матриц изображений.</param>
/// <param name="layers">Список слоёв.</param>
/// <param name="filterCore">Ядро фильтра.</param>
/// <param name="inputLayerNeurons">Нейроны выходного слоя.</param>
/// <param name="convolutionalLayerNeurons">Нейроны свёрточного слоя.</param>
/// <param name="hiddenLayerNeurons">Нейроны скрытого слоя.</param>
/// <param name="outputNeuron">Нейроны выходного слоя.</param>
private static void LayersInitialize(List <double[, ]> listOfPicturesmatrix, List <Layer> layers,
double[,] filterCore, out Dictionary <string, double> inputLayerNeurons,
out List <NeuronModel> convolutionalLayerNeurons, out List <NeuronModel> hiddenLayerNeurons,
out NeuronModel outputNeuron)
{
var firstDataSet = listOfPicturesmatrix.First();
var inputLayer = new InputLayer(firstDataSet);
inputLayer.Initialize();
layers.Add(inputLayer);
inputLayerNeurons = inputLayer.GetLayerNeurons();
var convolutionalLayer = new ConvolutionalLayer(inputLayerNeurons);
convolutionalLayer.Initialize(filterCore);
layers.Add(convolutionalLayer);
convolutionalLayerNeurons = convolutionalLayer.GetLayerNeurons();
var hiddenLayer = new HiddenLayer(convolutionalLayerNeurons);
hiddenLayer.Initialize();
layers.Add(hiddenLayer);
hiddenLayerNeurons = hiddenLayer.GetLayerNeurons();
var outputLayer = new OutputLayer(hiddenLayerNeurons);
outputLayer.Initilize();
layers.Add(outputLayer);
outputNeuron = outputLayer.GetOutputNeuron();
}
private static NeuralNetwork InitializeNeuralNetwork(int seed)
{
Random random = new Random(seed == 0 ? new Random().Next() : seed);
float RandomWeight() => (float)(random.NextDouble() * 2 - 1);
Layer prevLayer;
InputLayer li = new InputLayer(3, 5);
prevLayer = li;
ConvolutionalLayer l0 = new ConvolutionalLayer(8, 2, 1, 0, prevLayer, ActivationFunctions.ReLU(true));
prevLayer = l0;
prevLayer.InitializeWeights(RandomWeight);
ConvolutionalLayer l2 = new ConvolutionalLayer(16, 2, 1, 0, prevLayer, ActivationFunctions.ReLU(true));
prevLayer = l2;
prevLayer.InitializeWeights(RandomWeight);
FullyConnectedLayer l7 = new FullyConnectedLayer(16, prevLayer, ActivationFunctions.Sigmoid(1));
prevLayer = l7;
prevLayer.InitializeWeights(RandomWeight);
FullyConnectedLayer l8 = new FullyConnectedLayer(10, prevLayer, ActivationFunctions.SoftMax(1));
prevLayer = l8;
prevLayer.InitializeWeights(RandomWeight);
return(new NeuralNetwork(li, l0, l2, l7, l8));
}
public NormalizedImage(ConvolutionalLayer srcLayer)
{
Debug.AssertNotNull(srcLayer);
Debug.AssertNotNull(srcLayer.FeatureMaps);
Debug.Assert(srcLayer.FeatureMaps.Length > 0);
const int maxRowsInColumn = 10;
var featureMaps = srcLayer.FeatureMaps;
// подсчет количества строк и столбцов в изображении
var rowCount = Math.Min(maxRowsInColumn, featureMaps.Length);
var columnsCount = (int)Math.Ceiling((double)featureMaps.Length / maxRowsInColumn);
// инициализируем новое поле, размеры которого вмещают в себя матрицу всех изображений с отступами
InitializeCanvas((featureMaps[0].Width + 1) * columnsCount - 1,
(featureMaps[0].Height + 1) * rowCount - 1);
for (int y = 0, fm = 0; y < rowCount; y++)
{
for (var x = 0; x < columnsCount && fm < featureMaps.Length; x++, fm++)
{
var image = new NormalizedImage(featureMaps[fm]);
ArrayHelper <double> .CopyData(image.RawData, RawData, y *(image.Height + 1), x *(image.Width + 1));
}
}
}
public void SetUp()
{
var mock = new Mock <IWeightInitializer>();
var queue = new Queue <double>();
List <double[, , ]> kernels = new List <double[, , ]>
{
new double[2, 3, 3]
{
{
{ 0, 1, 1 },
{ -1, 0, 0 },
{ -1, 1, -1 }
},
{
{ 0, 0, -1 },
{ 1, 1, 1 },
{ 0, -1, -1 }
}
},
new double[2, 3, 3]
{
{
{ 0, -1, 1 },
{ -1, -1, -1 },
{ -1, 1, 0 }
},
{
{ 1, 1, 1 },
{ -1, -1, 0 },
{ -1, 1, 1 }
}
},
new double[2, 3, 3]
{
{
{ -1, 0, 0 },
{ 1, -1, -1 },
{ 1, -1, -1 }
},
{
{ 0, 0, -1 },
{ 1, 0, -1 },
{ -1, -1, -1 }
}
}
};
foreach (var kernel in kernels)
{
kernel.ForEach((q) => queue.Enqueue(q));
}
mock
.Setup(q => q.GenerateRandom(It.IsAny <double>()))
.Returns(queue.Dequeue);
_layer = new ConvolutionalLayer(3, 3, 1, new FilterMeta(5, 2), mock.Object, LearningRateAnnealerType.Adagrad);
}
/// <summary>
/// Constructor of fully connected layer type. Specify number of units as argument.
/// </summary>
/// <param name="nUnits"></param>
public ResidualModule(int FilterSize, int NumberOfFilters, int StrideLength, int ZeroPadding, string NonlinearityType)
{
this.type = "ResidualModule";
filterSize = FilterSize;
nFilters = NumberOfFilters;
strideLength = StrideLength;
zeroPadding = ZeroPadding;
if (NonlinearityType != "ReLU" && NonlinearityType != "ELU")
{
throw new ArgumentException("Only ReLU or ELU nonlinearities are currently supported in ResidualModules");
}
nonlinearityType = NonlinearityType;
convolutionalLayer1 = new ConvolutionalLayer(filterSize, nFilters, strideLength, zeroPadding);
if (NonlinearityType == "ReLU")
{
nonlinearityReLU = new ReLU();
}
else if (NonlinearityType == "ELU")
{
nonlinearityELU = new ELU(1.0f);
}
convolutionalLayer2 = new ConvolutionalLayer(filterSize, nFilters, strideLength, zeroPadding);
}
private static NeuralNetwork InitializeNeuralNetwork(int seed)
{
Random random = new Random(seed == 0 ? new Random().Next() : seed);
float RandomWeight() => (float)(random.NextDouble() * 2 - 1);
Layer prevLayer;
InputLayer li = new InputLayer(28, 28);
prevLayer = li;
ConvolutionalLayer l0 = new ConvolutionalLayer(15, 5, 1, 0, prevLayer, ActivationFunctions.ReLU(true));
prevLayer = l0;
prevLayer.InitializeWeights(RandomWeight);
MaxPoolingLayer l1 = new MaxPoolingLayer(2, 2, prevLayer);
prevLayer = l1;
ConvolutionalLayer l2 = new ConvolutionalLayer(30, 4, 1, 0, prevLayer, ActivationFunctions.ReLU(true));
prevLayer = l2;
prevLayer.InitializeWeights(RandomWeight);
MaxPoolingLayer l3 = new MaxPoolingLayer(3, 2, prevLayer);
prevLayer = l3;
ConvolutionalLayer l4 = new ConvolutionalLayer(45, 2, 2, 0, prevLayer, ActivationFunctions.ReLU(true));
prevLayer = l4;
prevLayer.InitializeWeights(RandomWeight);
MaxPoolingLayer l5 = new MaxPoolingLayer(2, 1, prevLayer);
prevLayer = l5;
FullyConnectedLayer l6 = new FullyConnectedLayer(64, prevLayer, ActivationFunctions.Sigmoid(1));
prevLayer = l6;
prevLayer.InitializeWeights(RandomWeight);
FullyConnectedLayer l7 = new FullyConnectedLayer(32, prevLayer, ActivationFunctions.Sigmoid(1));
prevLayer = l7;
prevLayer.InitializeWeights(RandomWeight);
FullyConnectedLayer l8 = new FullyConnectedLayer(10, prevLayer, ActivationFunctions.SoftMax(1));
prevLayer = l8;
prevLayer.InitializeWeights(RandomWeight);
return(new NeuralNetwork(li, l0, l1, l2, l3, l4, l5, l6, l7, l8));
}
public unsafe void ConvolutionGradient()
{
float[,]
x = WeightsProvider.NewFullyConnectedWeights(TensorInfo.Linear(127), 58 * 58 * 3, WeightsInitializationMode.GlorotNormal).AsSpan().AsMatrix(127, 58 * 58 * 3),
delta = WeightsProvider.NewFullyConnectedWeights(TensorInfo.Linear(127), 54 * 54 * 5, WeightsInitializationMode.GlorotNormal).AsSpan().AsMatrix(127, 54 * 54 * 5);
ConvolutionalLayer
cpu = new ConvolutionalLayer(new TensorInfo(58, 58, 3), ConvolutionInfo.Default, (5, 5), 5, ActivationFunctionType.LeCunTanh, BiasInitializationMode.Gaussian),
gpu = new CuDnnConvolutionalLayer(cpu.InputInfo, ConvolutionInfo.Default, cpu.KernelInfo, cpu.OutputInfo, cpu.Weights, cpu.Biases, ActivationFunctionType.LeCunTanh);
fixed(float *px = x)
{
Tensor.Reshape(px, x.GetLength(0), x.GetLength(1), out Tensor xTensor);
gpu.Forward(xTensor, out Tensor z_gpu, out Tensor a_gpu);
z_gpu.Free();
a_gpu.Free();
}
TestGradient(cpu, gpu, x, delta);
}
//JAVA TO C# CONVERTER WARNING: 'final' parameters are not allowed in .NET:
//ORIGINAL LINE: public Builder withConvolutionLayer(final org.neuroph.nnet.comp.Dimension2D kernelDimension, int numberOfMaps, Class transferFunction)
public virtual Builder withConvolutionLayer(comp.Dimension2D kernelDimension, int numberOfMaps, Type transferFunction)
{
FeatureMapsLayer prevLayer = LastFeatureMapLayer;
ConvolutionalLayer convolutionLayer = new ConvolutionalLayer(prevLayer, kernelDimension, numberOfMaps, transferFunction);
network.addLayer(convolutionLayer);
ConvolutionalUtils.fullConnectMapLayers(prevLayer, convolutionLayer);
return(this);
}
public virtual Builder withConvolutionLayer(int kernelWidth, int kernelHeight, int numberOfMaps)
{
FeatureMapsLayer prevLayer = LastFeatureMapLayer;
ConvolutionalLayer convolutionLayer = new ConvolutionalLayer(prevLayer, new comp.Dimension2D(kernelWidth, kernelHeight), numberOfMaps);
network.addLayer(convolutionLayer);
ConvolutionalUtils.fullConnectMapLayers(prevLayer, convolutionLayer);
return(this);
}
/// <summary>
/// Метод обратного распространения ошибки.
/// </summary>
/// <param name="inputLayer">Входной слой.</param>
/// <param name="convolutionalLayer">Свёрточный слой.</param>
/// <param name="hiddenLayer">Скрытый слой.</param>
/// <param name="outputLayer">Выходной слой.</param>
private void Backpropagation(InputLayer inputLayer, ConvolutionalLayer convolutionalLayer,
HiddenLayer hiddenLayer, OutputLayer outputLayer)
{
HiddenToOutputDeltasWork(outputLayer, hiddenLayer);
HiddentToOutputWeightsWork(outputLayer, hiddenLayer);
ConvolutionalToHiddenDeltasWork(hiddenLayer, convolutionalLayer);
ConvolutionalToHiddenWeightsWork(hiddenLayer, convolutionalLayer);
FilterCoreWork(convolutionalLayer, inputLayer);
}
public void Test_ValidateImage_Correct()
{
var input = new double[, , ]
{
{
{ 1, 2, 3 },
{ 4, 5, 6 },
{ 7, 8, 9 }
},
{
{ 1, 2, 3 },
{ 4, 5, 6 },
{ 7, 8, 9 }
},
{
{ 1, 2, 3 },
{ 4, 5, 6 },
{ 7, 8, 9 }
}
};
var filter = new double[, , ]
{
{
{ 1.1, 1.2, 1.3 },
{ 1.4, 1.5, 1.6 }
},
{
{ 1.1, 1.2, 1.3 },
{ 1.4, 1.5, 1.6 }
}
};
var filter2 = new double[, , ]
{
{
{ 2.1, 2.2, 2.3 },
{ 2.4, 2.5, 2.6 }
},
{
{ 2.1, 2.2, 2.3 },
{ 2.4, 2.5, 2.6 }
}
};
var layer = new ConvolutionalLayer();
layer.Neurals.Add(new Filter {
Weights = new Array3D(filter)
});
layer.Neurals.Add(new Filter {
Weights = new Array3D(filter2)
});
layer.FormOutput(new Array3D(input));
}
public static IConvLayer ToConvLayer(this CnnLayer layer)
{
switch (layer.LayerType)
{
case (byte)LayerType.CovolutionalLayer:
var convLayer = new ConvolutionalLayer
{
Kernels = new List <double[][][]>(),
KernelPadding = 0,
KernelStride = 1,
KernelSize = layer.KernelHeight
};
var weights = layer.Weights.Weights.Split(';');
for (int i = 0; i < layer.KernelsCount; ++i)
{
var kernels = new double[layer.FeatureMapsCountIn][][];
for (int j = 0; j < layer.FeatureMapsCountIn; ++j)
{
kernels[j] = new double[layer.KernelHeight][];
for (int a = 0; a < layer.KernelHeight; ++a)
{
kernels[j][a] = new double[layer.KernelWidth];
for (int b = 0; b < layer.KernelWidth; ++b)
{
kernels[j][a][b] =
double.Parse(weights[j * layer.KernelHeight * layer.KernelWidth + a * layer.KernelWidth + b]);
}
}
}
convLayer.Kernels.Add(kernels);
}
return(convLayer);
case (byte)LayerType.PoolingLayer:
var poolingLayer = new PollingLayer(layer.KernelHeight, 0, 1);
return(poolingLayer);
case (byte)LayerType.ReluLayer:
var reluLayer = new ReLuLayer();
return(reluLayer);
default:
throw new Exception();
}
}
/// <summary>
/// Получение и обновление дельты свёрточного слоя.
/// </summary>
/// <param name="hiddenLayer">Скрытый слой.</param>
/// <param name="convolutionalLayer">Свёрточный слой.</param>
private void ConvolutionalToHiddenDeltasWork(HiddenLayer hiddenLayer, ConvolutionalLayer convolutionalLayer)
{
var convolutionalLayerNeurons = convolutionalLayer.GetLayerNeurons();
var convolutionalLayerOutputs = new List <double>();
convolutionalLayerNeurons.ForEach(neuron =>
convolutionalLayerOutputs.Add(neuron.Output));
var convolutionalLayerDeltas = GetConvolutionalLayerDeltas(convolutionalLayerOutputs,
hiddenLayer.GetLayerNeurons());
convolutionalLayer.UpdateDeltas(convolutionalLayerDeltas);
}
/// <summary>
/// Выполнить вычисления для ядра фильтра.
/// </summary>
/// <param name="convolutionalLayer">Свёрточный слой.</param>
/// <param name="inputLayer">Входной слой.</param>
private void FilterCoreWork(ConvolutionalLayer convolutionalLayer, InputLayer inputLayer)
{
var inputLayerNeurons = inputLayer.GetLayerNeurons();
var convolutionalLayerDeltas = new List <double>();
convolutionalLayer.GetLayerNeurons().ForEach(neuron => convolutionalLayerDeltas.Add(neuron.Delta));
var filterCoreMatrixDeltas = GetFilterCoreDeltasMatrix(
inputLayerNeurons, convolutionalLayerDeltas);
var filterCoreMatrixGradients = GetFilterCoreGradientMatrix(
filterCoreMatrixDeltas, inputLayerNeurons);
UpdateCore(filterCoreMatrixGradients);
}
public void should_output_tensor_of_expected_size(int inputSize, int weightLength, int expectedOutputSize)
{
// Arrange
_sut = (ConvolutionalLayer) new Net(inputSize)
.Convolution(new [] { weightLength, 1 }, kernelCount: 1, strideArray: new [] { 1 })
.Layers[0];
var input = new float[inputSize];
// Act
var output = _sut.FeedForwards(new Tensor(input));
// Assert
Assert.That(output.Size.Dimensions.Length, Is.EqualTo(1));
Assert.That(output.Size.Dimensions[0], Is.EqualTo(expectedOutputSize));
}
/// <summary>
/// Получение градиента между нейронами свёрточного слоя и скрытого, и обновление весов.
/// </summary>
/// <param name="hiddenLayer">Скрытый слой.</param>
/// <param name="convolutionalLayer">Свёрточный слой.</param>
private void ConvolutionalToHiddenWeightsWork(HiddenLayer hiddenLayer, ConvolutionalLayer convolutionalLayer)
{
var convolutionalLayerNeurons = convolutionalLayer.GetLayerNeurons();
var convolutionalLayerOutputs = new List <double>();
var hiddenLayerNeurons = hiddenLayer.GetLayerNeurons();
var hiddenLayerDeltas = new List <double>();
convolutionalLayerNeurons.ForEach(neuron => convolutionalLayerOutputs.Add(neuron.Output));
hiddenLayerNeurons.ForEach(neuron => hiddenLayerDeltas.Add(neuron.Delta));
var convolutionalNeuronToGradientsDictionary = GetConvolutionalToHiddenGradients(
convolutionalLayerOutputs, hiddenLayerDeltas);
UpdateConvolutionalToHiddenWeights(convolutionalNeuronToGradientsDictionary, hiddenLayer);
}
internal static INetworkLayer CpuLayerDeserialize([NotNull] Stream stream, LayerType type)
{
switch (type)
{
case LayerType.FullyConnected: return(FullyConnectedLayer.Deserialize(stream));
case LayerType.Convolutional: return(ConvolutionalLayer.Deserialize(stream));
case LayerType.Pooling: return(PoolingLayer.Deserialize(stream));
case LayerType.Output: return(OutputLayer.Deserialize(stream));
case LayerType.Softmax: return(SoftmaxLayer.Deserialize(stream));
default: throw new ArgumentOutOfRangeException(nameof(type), $"The {type} layer type is not supported by the default deserializer");
}
}
public void should_output_tensor_of_correct_size(int inputSize, int weightLength, int expectedOutputSize)
{
// Arrange
_sut = (ConvolutionalLayer) new Net(inputSize, inputSize)
.ConvolutionTranspose(new[] { weightLength, weightLength })
.Layers[0];
var input = new float[inputSize, inputSize].ToMatrix();
// Act
var output = _sut.FeedForwards(new Tensor(input));
// Assert
Assert.That(output.Size.Dimensions.Length, Is.EqualTo(2));
Assert.That(output.Size.Dimensions[0], Is.EqualTo(expectedOutputSize));
Assert.That(output.Size.Dimensions[1], Is.EqualTo(expectedOutputSize));
}
public void WriteWeightsToDirectory(string weightsDirectory)
{
Task[] tasks = new Task[model.NetworkLayers.Count];
for (int i = 0; i < model.NetworkLayers.Count; i++)
{
int taski = 0 + i;
tasks[taski] = Task.Run(() =>
{
string layerPath = weightsDirectory + "\\" + model.NetworkLayers[taski].Type + taski;
switch (model.NetworkLayers[taski].Type)
{
case "Convolutional":
Directory.CreateDirectory(layerPath);
ConvolutionalLayer auxLayer = (ConvolutionalLayer)model.NetworkLayers[taski];
for (int filter = 0; filter < auxLayer.FilterNumber; filter++)
{
int taskf = 0 + filter;
File.WriteAllText(layerPath + "\\Filter" + filter + ".json", JsonConvert.SerializeObject(auxLayer.Filters[taskf]));
}
break;
case "Dense":
Directory.CreateDirectory(layerPath);
DenseLayer auxDense = (DenseLayer)model.NetworkLayers[taski];
for (int unit = 0; unit < auxDense.NumberOfUnits; unit++)
{
File.WriteAllText(layerPath + "\\Unit" + unit + ".json", JsonConvert.SerializeObject(auxDense.Units[unit]));
}
break;
default:
break;
}
});
}
Task.WaitAll(tasks);
Console.WriteLine("Weights written to file");
}
private void CompareLayers(ConvolutionalLayer first, ConvolutionalLayer second)
{
for (int j = 0; j < first.KernelsCount; ++j)
{
for (int k = 0; k < first.KernelDepth; ++k)
{
for (int a = 0; a < first.Kernels[j][k].Length; ++a)
{
for (int b = 0; b < first.Kernels[j][k][a].Length; ++b)
{
if (Math.Round(first.Kernels[j][k][a][b], 10) != Math.Round(second.Kernels[j][k][a][b], 10))
{
throw new Exception();
}
}
}
}
}
}
public static IConvLayer Create(int neuronsCount = 0, int inputMapsCount = 0, int kernelsCount = 0, byte layerType = 0, double lr = 0, int prevNeuronsCount = 0)
{
switch (layerType)
{
case (byte)LayerType.CovolutionalLayer:
var convLayer = new ConvolutionalLayer
{
Kernels = new List <double[][][]>(),
LearningRate = lr,
KernelPadding = 0,
KernelStride = 1,
KernelSize = 3
};
var kernels = new double[inputMapsCount][][];
for (int i = 0; i < kernelsCount; ++i)
{
for (int j = 0; j < inputMapsCount; ++j)
{
kernels[j] = CreateKernel(3, neuronsCount);
}
convLayer.Kernels.Add(kernels);
}
return(convLayer);
case (byte)LayerType.PoolingLayer:
var poolingLayer = new PollingLayer(2, 0, 1);
return(poolingLayer);
case (byte)LayerType.ReluLayer:
var reluLayer = new ReLuLayer();
return(reluLayer);
default:
throw new Exception();
}
}
/// <summary>
/// Инициализировать нейронную сеть.
/// </summary>
public void InitializeNetwork()
{
var inputLayer = new InputLayer(_imageData);
inputLayer.Initialize();
var convolutionalLayer = new ConvolutionalLayer(inputLayer.GetLayerNeurons());
convolutionalLayer.RecognizeMode(FilterCoreModel.GetCore);
var hiddenLayer = new HiddenLayer(convolutionalLayer.GetLayerNeurons());
hiddenLayer.RecognizeMode(_weightsInHiddenLayer);
var outputLayer = new OutputLayer(hiddenLayer.GetLayerNeurons());
outputLayer.RecognizeMode(_weightsInOutputLayer);
ToRecognizeImage(outputLayer.GetOutputNeuron().Output);
}
public void should_backpropagate_tensor_of_correct_size(int inputSize, int weightLength, int outputSize)
{
// Arrange
_sut = (ConvolutionalLayer) new Net(inputSize, inputSize)
.ConvolutionTranspose(new[] { weightLength, weightLength })
.Layers[0];
var trainingRun = new TrainingRun(1)
{
Input = new float[inputSize, inputSize].ToMatrix(),
Output = _sut.FeedForwards(new float[inputSize, inputSize].ToMatrix()),
OutputError = new float[outputSize, outputSize].ToMatrix()
};
// Act
_sut.BackPropagate(trainingRun);
// Assert
Assert.That(trainingRun.InputError.Size, Is.EqualTo(trainingRun.Input.Size));
}
/// <summary>
/// Обновить слои.
/// </summary>
/// <param name="outputLayer">Выходной слой.</param>
/// <param name="hiddenLayer">Скрытый слой.</param>
/// <param name="convolutionalLayer">Свёрточный слой.</param>
/// <param name="inputLayer">Входной слой.</param>
/// <param name="currentIteration">Текущая итерация.</param>
private void LayersUpdate(OutputLayer outputLayer, HiddenLayer hiddenLayer,
ConvolutionalLayer convolutionalLayer, InputLayer inputLayer, int currentIteration)
{
inputLayer.UpdateInputData(_dataSets[currentIteration]);
convolutionalLayer.UpdateData(FilterCoreModel.GetCore, inputLayer.GetLayerNeurons());
var inputsToHiddenLayer = new List <double>();
convolutionalLayer.GetLayerNeurons().ForEach(neuron =>
inputsToHiddenLayer.Add(neuron.Output));
hiddenLayer.UpdateNeuronsInputs(inputsToHiddenLayer);
var inputsToOutputLayer = new List <double>();
hiddenLayer.GetLayerNeurons().ForEach(neuron =>
inputsToOutputLayer.Add(neuron.Output));
outputLayer.UpdateNeuronInputs(inputsToOutputLayer);
}
public void ReadWeightsFromDirectory(string weightsDirectory)
{
Task[] tasks = new Task[model.NetworkLayers.Count];
for (int i = 0; i < model.NetworkLayers.Count; i++)
{
int taski = 0 + i;
tasks[taski] = Task.Run(() =>
{
string layerPath = weightsDirectory + "\\" + model.NetworkLayers[taski].Type + taski;
switch (model.NetworkLayers[taski].Type)
{
case "Convolutional":
ConvolutionalLayer auxLayer = (ConvolutionalLayer)model.NetworkLayers[taski];
for (int filter = 0; filter < auxLayer.FilterNumber; filter++)
{
string filterPath = layerPath + "\\Filter" + filter + ".json";
string json = File.ReadAllText(filterPath);
auxLayer.Filters[filter] = JsonConvert.DeserializeObject <Filter>(json);
}
break;
case "Dense":
DenseLayer auxDense = (DenseLayer)model.NetworkLayers[taski];
for (int unit = 0; unit < auxDense.NumberOfUnits; unit++)
{
string json = File.ReadAllText(layerPath + "\\Unit" + unit + ".json");
auxDense.Units[unit] = JsonConvert.DeserializeObject <Unit>(json);
}
break;
default:
break;
}
});
}
Task.WaitAll(tasks);
}
public void should_backpropagate_tensor_of_correct_kernel_size(int kernelSize)
{
// Arrange
_sut = (ConvolutionalLayer) new Net(1, 1)
.ConvolutionTranspose(new[] { 5, 5 }, kernelSize)
.Layers[0];
var input = new float[, ] {
{ 0 }
}.ToMatrix();
var trainingRun = new TrainingRun(1)
{
Input = input,
Output = _sut.FeedForwards(input),
OutputError = new Tensor(new Size(5, 5, kernelSize), new float[5 * 5 * kernelSize])
};
// Act
_sut.BackPropagate(trainingRun);
// Assert
Assert.That(trainingRun.InputError.Size, Is.EqualTo(trainingRun.Input.Size));
}
public static ConvolutionalNeuralNetwork Parse(string filename)
{
var jobject = JObject.Parse(File.ReadAllText(filename));
var jsonLayers = jobject.GetValue("layers");
var network = new ConvolutionalNeuralNetwork();
var netLayers = new List <CNNLayer>();
foreach (var jsonLayer in jsonLayers.ToArray())
{
var type = jsonLayer["layer_type"].ToObject <string>();
CNNLayer layer;
switch (type)
{
case "input":
layer = new InputLayer();
break;
case "conv":
layer = new ConvolutionalLayer
{
FilterSize = jsonLayer["sx"].ToObject <int>(),
Stride = jsonLayer["stride"].ToObject <int>(),
L1Decay = jsonLayer["l1_decay_mul"].ToObject <double>(),
L2Decay = jsonLayer["l2_decay_mul"].ToObject <double>(),
Pad = jsonLayer["pad"].ToObject <int>(),
Biases = Volume.ParseJSON(jsonLayer["biases"]),
Kernels = Volume.ArrayParseJSON(jsonLayer["filters"])
};
break;
case "relu":
layer = new ReLuLayer();
break;
case "pool":
layer = new SubsamplingLayer
{
FilterSize = jsonLayer["sx"].ToObject <int>(),
Stride = jsonLayer["stride"].ToObject <int>(),
Pad = jsonLayer["pad"].ToObject <int>(),
};
break;
case "fc":
layer = new FullyConnectedLayer
{
L1Decay = jsonLayer["l1_decay_mul"].ToObject <double>(),
L2Decay = jsonLayer["l2_decay_mul"].ToObject <double>(),
NeuronsCount = jsonLayer["out_depth"].ToObject <int>(),
Biases = Volume.ParseJSON(jsonLayer["biases"]),
Weights = Volume.ArrayParseJSON(jsonLayer["filters"])
};
break;
case "softmax":
layer = new SoftmaxLayer();
break;
default:
throw new ArgumentException("Unknown layer type");
}
if (netLayers.Count != 0)
{
layer.InSize = netLayers.Last().OutSize;
layer.InDepth = netLayers.Last().OutDepth;
}
layer.OutDepth = jsonLayer["out_depth"].ToObject <int>();
layer.OutSize = new Size(
jsonLayer["out_sx"].ToObject <int>(),
jsonLayer["out_sy"].ToObject <int>());
if (type == "fc")
{
(layer as FullyConnectedLayer)._inputsCount =
layer.InSize.Width * layer.InSize.Height * layer.InDepth;
}
else if (type == "pool")
{
(layer as SubsamplingLayer)._oldX = new int[layer.OutSize.Width * layer.OutSize.Height * layer.OutDepth];
(layer as SubsamplingLayer)._oldY = new int[layer.OutSize.Width * layer.OutSize.Height * layer.OutDepth];
}
else if (type == "softmax")
{
(layer as SoftmaxLayer)._es = new double[layer.OutDepth];
}
layer.OutVolume = new Volume(layer.OutSize.Width, layer.OutSize.Height, layer.OutDepth, 0);
netLayers.Add(layer);
}
network.Layers = netLayers;
network.InputLayer = (InputLayer)netLayers.First();
return(network);
}
static void Main(string[] args)
{
//Read CL arguments
for (int i = 0; i < args.Length; i++)
{
if (args[i] == "-d")
{
deviceID = int.Parse(args[++i]);
}
if (args[i] == "-lr")
{
learning_rate = double.Parse(args[++i], System.Globalization.NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture);
}
if (args[i] == "-iso")
{
ISO = args[++i];
}
if (args[i] == "-t")
{
crosscheck = true;
}
if (args[i] == "-w")
{
warmStart = int.Parse(args[++i]);
Console.WriteLine("Start with epoch " + warmStart);
}
if (args[i] == "-s")
{
saveImages = true;
}
}
Console.WriteLine("Using device ID: " + deviceID);
Console.WriteLine("Learning rate: " + learning_rate);
//Init Cuda stuff
ctx = new PrimaryContext(deviceID);
ctx.SetCurrent();
Console.WriteLine("Context created");
CUmodule modPatch = ctx.LoadModulePTX("PatchProcessing.ptx");
Console.WriteLine("modPatch loaded");
CUmodule modBorder = ctx.LoadModulePTX("BorderTreatment.ptx");
Console.WriteLine("modBorder loaded");
CUmodule modError = ctx.LoadModulePTX("ErrorComputation.ptx");
Console.WriteLine("modError loaded");
CUmodule modPRelu = ctx.LoadModulePTX("PRelu.ptx");
Console.WriteLine("modPRelu loaded");
CUmodule modDeBayer = ctx.LoadModulePTX("DeBayer.ptx");
Console.WriteLine("all modules loaded");
deBayerGreenKernel = new DeBayerGreenKernel(modDeBayer, ctx);
deBayerRedBlueKernel = new DeBayerRedBlueKernel(modDeBayer, ctx);
//Both deBayer kernels are load from the same module: setting the constant variable for bayer pattern one is enough...
deBayerGreenKernel.BayerPattern = new BayerColor[] { BayerColor.Red, BayerColor.Green, BayerColor.Green, BayerColor.Blue };
prepareDataKernel = new PrepareDataKernel(modPatch, ctx);
restoreImageKernel = new RestoreImageKernel(modPatch, ctx);
Console.WriteLine("kernels loaded");
int countOwn = 468083;
int count5k = 33408;
string fileBase = @"/ssd/data/TrainingsDataNN/";
List <float3> WhiteBalanceFactors = new List <float3>();
FileStream fs1 = new FileStream(fileBase + "FromOwnDataset/WhiteBalancesOwn.txt", FileMode.Open, FileAccess.Read);
FileStream fs2 = new FileStream(fileBase + "From5kDataset/WhiteBalances5k.txt", FileMode.Open, FileAccess.Read);
StreamReader sr1 = new StreamReader(fs1);
StreamReader sr2 = new StreamReader(fs2);
for (int i = 0; i < countOwn; i++)
{
fileRawList.Add(fileBase + "FromOwnDataset/ISO" + ISO + "/img_" + i.ToString("0000000") + ".bin");
fileTrouthList.Add(fileBase + "FromOwnDataset/GroundTruth/img_" + i.ToString("0000000") + ".bin");
string line = sr1.ReadLine();
string[] values = line.Split('\t');
float3 wb = new float3(float.Parse(values[1], System.Globalization.NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture),
float.Parse(values[2], System.Globalization.NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture),
float.Parse(values[3], System.Globalization.NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture));
WhiteBalanceFactors.Add(wb);
}
for (int i = 0; i < count5k; i++)
{
fileRawList.Add(fileBase + "From5kDataset/ISO" + ISO + "/img_" + i.ToString("0000000") + ".bin");
fileTrouthList.Add(fileBase + "From5kDataset/GroundTruth/img_" + i.ToString("0000000") + ".bin");
string line = sr2.ReadLine();
string[] values = line.Split('\t');
float3 wb = new float3(float.Parse(values[1], System.Globalization.NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture),
float.Parse(values[2], System.Globalization.NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture),
float.Parse(values[3], System.Globalization.NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture));
WhiteBalanceFactors.Add(wb);
}
sr2.Close();
sr1.Close();
baOriginal = new float3[countOwn + count5k][];
baRAW = new float[countOwn + count5k][];
Random rand = new Random(0);
//random order for the image patches
for (int i = 0; i < countOwn + count5k - 1; i++)
{
int r = i + (rand.Next() % (countOwn + count5k - i));
string temp = fileRawList[i];
fileRawList[i] = fileRawList[r];
fileRawList[r] = temp;
temp = fileTrouthList[i];
fileTrouthList[i] = fileTrouthList[r];
fileTrouthList[r] = temp;
float3 tempf = WhiteBalanceFactors[i];
WhiteBalanceFactors[i] = WhiteBalanceFactors[r];
WhiteBalanceFactors[r] = tempf;
}
Console.WriteLine("Initialization done!");
int trainingSize = (int)((countOwn + count5k) * 0.9f); //4 patches per file
int testSize = fileRawList.Count - trainingSize;
CudaBlas blas = new CudaBlas(PointerMode.Host);
CudaDNNContext cudnn = new CudaDNNContext();
int patchSize = 31;
int patchSize4 = 66; //Size of an 2x2 patch read from file
int batch = 64;
float normalization = 0.5f;
//define neural network:
StartLayer start = new StartLayer(patchSize, patchSize, 3, batch);
FinalLayer final = new FinalLayer(patchSize, patchSize, 3, batch, FinalLayer.Norm.Mix, ctx, modError);
ConvolutionalLayer conv1 = new ConvolutionalLayer(patchSize, patchSize, 3, patchSize, patchSize, 64, batch, 9, 9, ConvolutionalLayer.Activation.PRelu, blas, cudnn, ctx, modBorder, modPRelu);
ConvolutionalLayer conv2 = new ConvolutionalLayer(patchSize, patchSize, 64, patchSize, patchSize, 64, batch, 5, 5, ConvolutionalLayer.Activation.PRelu, blas, cudnn, ctx, modBorder, modPRelu);
ConvolutionalLayer conv3 = new ConvolutionalLayer(patchSize, patchSize, 64, patchSize, patchSize, 3, batch, 5, 5, ConvolutionalLayer.Activation.None, blas, cudnn, ctx, modBorder, modPRelu);
start.ConnectFollowingLayer(conv1);
conv1.ConnectFollowingLayer(conv2);
conv2.ConnectFollowingLayer(conv3);
conv3.ConnectFollowingLayer(final);
CudaDeviceVariable <float3> imgA = new CudaDeviceVariable <float3>(patchSize4 * patchSize4);
CudaDeviceVariable <float3> imgB = new CudaDeviceVariable <float3>(patchSize4 * patchSize4);
CudaDeviceVariable <float> rawd = new CudaDeviceVariable <float>(patchSize4 * patchSize4);
CudaDeviceVariable <float> inputImgs = new CudaDeviceVariable <float>(patchSize * patchSize * 3 * batch);
CudaDeviceVariable <float> groundTrouth = new CudaDeviceVariable <float>(patchSize * patchSize * 3 * batch);
NPPImage_8uC3 imgU3a = new NPPImage_8uC3(patchSize, patchSize);
NPPImage_8uC3 imgU3b = new NPPImage_8uC3(patchSize, patchSize);
NPPImage_8uC3 imgU3c = new NPPImage_8uC3(patchSize, patchSize);
Bitmap a = new Bitmap(patchSize, patchSize, PixelFormat.Format24bppRgb);
Bitmap b = new Bitmap(patchSize, patchSize, PixelFormat.Format24bppRgb);
Bitmap c = new Bitmap(patchSize, patchSize, PixelFormat.Format24bppRgb);
Random randImageOutput = new Random(0);
Random randForInit = new Random(0);
start.InitRandomWeight(randForInit);
conv1.SetActivation(0.1f);
conv2.SetActivation(0.1f);
int startEpoch = warmStart;
FileStream fs;
//restore network in case of warm start:
if (warmStart > 0)
{
fs = new FileStream("epoch_" + learning_rate.ToString(CultureInfo.InvariantCulture) + "_" + ISO + "_" + (warmStart - 1) + ".cnn", FileMode.Open, FileAccess.Read);
start.RestoreValues(fs);
fs.Close();
fs.Dispose();
}
//validate results on validation data set
if (crosscheck)
{
FileStream csvResult = new FileStream("results_" + learning_rate.ToString(CultureInfo.InvariantCulture) + "_" + ISO + ".csv", FileMode.Append, FileAccess.Write);
StreamWriter sw = new StreamWriter(csvResult);
sw.WriteLine("L1;L2;Mix;Filename");
for (int i = 0; i < 2000; i += 1)
{
string filename = "epoch_" + learning_rate.ToString(CultureInfo.InvariantCulture) + "_" + ISO + "_" + i + ".cnn";
try
{
FileStream cnn = new FileStream(filename, FileMode.Open, FileAccess.Read);
start.RestoreValues(cnn);
cnn.Close();
cnn.Dispose();
}
catch (Exception)
{
Console.WriteLine("Skipping: " + i);
continue;
}
double errorL1 = 0;
double errorL2 = 0;
double errorMix = 0;
for (int iter = 0; iter < testSize / batch * 4; iter++)
{
//Prepare batch for training:
for (int ba = 0; ba < batch / 4; ba++)
{
int idx = iter * (batch / 4) + ba + trainingSize;
float3[] original;
float[] raw;
if (baRAW[idx - trainingSize] == null)
{
original = ReadRAWFloat3(fileTrouthList[idx]);
raw = ReadRAWFloat(fileRawList[idx]);
baOriginal[idx - trainingSize] = original;
baRAW[idx - trainingSize] = raw;
}
else
{
original = baOriginal[idx - trainingSize];
raw = baRAW[idx - trainingSize];
}
rawd.CopyToDevice(raw);
imgA.CopyToDevice(original);
deBayerGreenKernel.RunSafe(rawd, imgB, patchSize4, new float3(0, 0, 0), WhiteBalanceFactors[idx]);
deBayerRedBlueKernel.RunSafe(rawd, imgB, patchSize4, new float3(0, 0, 0), WhiteBalanceFactors[idx]);
prepareDataKernel.RunSafe(imgA, imgB, groundTrouth, inputImgs, ba, normalization, WhiteBalanceFactors[idx]);
}
start.SetData(inputImgs);
final.SetGroundTrouth(groundTrouth);
float err = start.InferenceTraining(inputImgs);
errorMix += err;
errorL1 += final.GetError(FinalLayer.Norm.L1);
errorL2 += final.GetError(FinalLayer.Norm.L2);
}
Console.WriteLine("Results for: " + filename);
Console.WriteLine("Mean Error L1: " + errorL1 / testSize * batch / 4);
Console.WriteLine("Mean Error L2: " + errorL2 / testSize * batch / 4);
Console.WriteLine("Mean Error Mix: " + errorMix / testSize * batch / 4);
sw.Write((errorL1 / testSize * batch / 4).ToString().Replace(".", ","));
sw.Write(";");
sw.Write((errorL2 / testSize * batch / 4).ToString().Replace(".", ","));
sw.Write(";");
sw.Write((errorMix / testSize * batch / 4).ToString().Replace(".", ","));
sw.Write(";");
sw.WriteLine(filename);
sw.Flush();
}
sw.Close();
csvResult.Close();
csvResult.Dispose();
}
//or train existing network:
else
{
double error = 0;
double errorEpoch = 0;
for (int epoch = startEpoch; epoch < 2000; epoch++)
{
errorEpoch = 0;
error = 0;
for (int iter = 0; iter < trainingSize / batch * 4; iter++)
{
//Prepare batch for training:
for (int ba = 0; ba < batch / 4; ba++)
{
int idx = iter * (batch / 4) + ba;
float3[] original;
float[] raw;
if (baRAW[idx] == null)
{
original = ReadRAWFloat3(fileTrouthList[idx]);
raw = ReadRAWFloat(fileRawList[idx]);
baOriginal[idx] = original;
baRAW[idx] = raw;
}
else
{
original = baOriginal[idx];
raw = baRAW[idx];
}
rawd.CopyToDevice(raw);
imgA.CopyToDevice(original);
deBayerGreenKernel.RunSafe(rawd, imgB, patchSize4, new float3(0, 0, 0), WhiteBalanceFactors[idx]);
deBayerRedBlueKernel.RunSafe(rawd, imgB, patchSize4, new float3(0, 0, 0), WhiteBalanceFactors[idx]);
prepareDataKernel.RunSafe(imgA, imgB, groundTrouth, inputImgs, ba, normalization, WhiteBalanceFactors[idx]);
}
start.SetData(inputImgs);
final.SetGroundTrouth(groundTrouth);
float err = start.InferenceTraining(inputImgs);
final.BackPropagation(groundTrouth);
start.UpdateWeights(GetLearningRate(epoch * (trainingSize) / batch * 4 + iter));//*0+951342
error += err;
errorEpoch += err;
if ((epoch * trainingSize / batch * 4 + iter) % 1000 == 0 && iter != 0)
{
FileStream status = new FileStream("status_" + learning_rate.ToString(CultureInfo.InvariantCulture) + "_" + ISO + ".csv", FileMode.Append, FileAccess.Write);
StreamWriter sw = new StreamWriter(status);
sw.WriteLine((error / 1000.0).ToString().Replace(".", ",") + ";" + GetLearningRate(epoch * trainingSize / batch * 4 + iter).ToString().Replace(".", ","));
sw.Close();
status.Close();
status.Dispose();
error = 0;
}
//if ((epoch * trainingSize / batch * 4 + iter) % 10000 == 0)
//{
// fs = new FileStream("iter_" + learning_rate.ToString(CultureInfo.InvariantCulture) + "_" + ISO + "_" + (epoch * trainingSize / batch * 4 + iter) + ".cnn", FileMode.Create, FileAccess.Write);
// start.SaveValues(fs);
// fs.Close();
// fs.Dispose();
// Console.WriteLine("Network saved for iteration " + (epoch * trainingSize / batch * 4 + iter) + "!");
//}
Console.WriteLine("Epoch: " + epoch + " Iteration: " + (epoch * trainingSize / batch * 4 + iter) + ", Error: " + err);
if (saveImages && iter == 0)//(epoch * trainingSize / batch * 4 + iter) % 10000 == 0 &&
{
for (int i = 0; i < 1; i++)
{
int imgidx = randImageOutput.Next(batch);
float3 wb = WhiteBalanceFactors[iter * (batch / 4) + imgidx / 4];
restoreImageKernel.RunSafe(groundTrouth, imgU3a, imgidx, wb.x, wb.y, wb.z, normalization);
restoreImageKernel.RunSafe(inputImgs, imgU3b, imgidx, wb.x, wb.y, wb.z, normalization);
CudaDeviceVariable <float> res = final.GetResult();
restoreImageKernel.RunSafe(res, imgU3c, imgidx, wb.x, wb.y, wb.z, normalization);
imgU3a.CopyToHost(a);
imgU3b.CopyToHost(b);
imgU3c.CopyToHost(c);
a.Save("GroundTrouth_" + learning_rate.ToString(CultureInfo.InvariantCulture) + "_" + ISO + "_" + epoch + "_" + imgidx + ".png");// * trainingSize / batch * 4 + iter
b.Save("Input_" + learning_rate.ToString(CultureInfo.InvariantCulture) + "_" + ISO + "_" + epoch + "_" + imgidx + ".png");
c.Save("Result_" + learning_rate.ToString(CultureInfo.InvariantCulture) + "_" + ISO + "_" + epoch + "_" + imgidx + ".png");
}
}
}
errorEpoch /= trainingSize / batch * 4;
fs = new FileStream("errorEpoch_" + learning_rate.ToString(CultureInfo.InvariantCulture) + "_" + ISO + ".csv", FileMode.Append, FileAccess.Write);
StreamWriter sw2 = new StreamWriter(fs);
sw2.WriteLine(errorEpoch.ToString().Replace(".", ","));
sw2.Close();
fs.Close();
fs.Dispose();
fs = new FileStream("epoch_" + learning_rate.ToString(CultureInfo.InvariantCulture) + "_" + ISO + "_" + epoch + ".cnn", FileMode.Create, FileAccess.Write);
start.SaveValues(fs);
fs.Close();
fs.Dispose();
}
}
}
