public void PropagateBackward(NNBackpropagationData backpropagationData) { NNTrainingData trainingData = backpropagationData.TrainingData; // Validate training data dimensions if (trainingData.InputDataWidth != InputLayer.Width || trainingData.InputDataHeight != InputLayer.Height || trainingData.InputDataDepth != InputLayer.Depth) { throw new ArgumentException(); } if (trainingData.TargetDataWidth != OutputLayer.Width || trainingData.TargetDataHeight != OutputLayer.Height || trainingData.TargetDataDepth != OutputLayer.Depth) { throw new ArgumentException(); } for (int trainingDataIndex = 0; trainingDataIndex < trainingData.DataSize; trainingDataIndex++) { backpropagationData.BatchTrainingStartingCallback?.Invoke(trainingDataIndex, trainingData.DataSize); // Initialize backpropagation run NNDetailedBackpropagationData detailedBackpropagationData = new NNDetailedBackpropagationData(backpropagationData, trainingDataIndex); // Feed forward through the network and gather neccessary data NNFeedData feedForwardInputData = trainingData.GetInputData(trainingDataIndex); for (int i = 0; i < this.layers.Length; i++) { Layer layer = this.layers[i]; NNDetailedFeedData feedData = layer.FeedForward(feedForwardInputData); detailedBackpropagationData.FeedForwardData[layer] = feedData; feedForwardInputData = feedData.OutputData; } // Propagate error backwards through the network for (int i = this.layers.Length - 1; i >= 0; i--) { Layer layer = this.layers[i]; layer.PropagateBackward(detailedBackpropagationData); } // Update weights for each layer foreach (KeyValuePair <Layer, float[]> updatedWeight in detailedBackpropagationData.UpdatedWeights) { Layer layer = updatedWeight.Key; float[] weights = updatedWeight.Value; for (int i = 0; i < weights.Length; i++) { layer.SetWeight(i, weights[i]); } } backpropagationData.BatchTrainingFinishedCallback?.Invoke(trainingDataIndex, trainingData.DataSize); } }
private static void Train(NeuralNetwork nn, float learningRate, int runs) { Console.WriteLine("Preparing Training..."); NNFeedData[] trainingInputData = new NNFeedData[training.Count]; NNFeedData[] trainingTargetData = new NNFeedData[training.Count]; for (int i = 0; i < training.Count; i++) { MnistEntry entry = training[i]; trainingInputData[i] = new NNFeedData(28, 28, 1, ConvertArray(entry.Image)); trainingTargetData[i] = new NNFeedData(10, 1, 1, LabelToArray(entry.Label)); } NNTrainingData trainingData = new NNPreloadedTrainingData(trainingInputData, trainingTargetData); NNBackpropagationData backpropagationData = new NNBackpropagationData(trainingData, learningRate, (o, t) => o - t); double totalTime = 0; Console.WriteLine("Starting Training..."); for (int trainingRuns = 0; trainingRuns < runs; trainingRuns++) { DateTime start = DateTime.UtcNow; backpropagationData.BatchTrainingStartingCallback = (trainingDataIndex, trainingSets) => { start = DateTime.UtcNow; }; backpropagationData.BatchTrainingFinishedCallback = (trainingDataIndex, trainingSets) => { totalTime += (DateTime.UtcNow - start).TotalMilliseconds; double avgTime = totalTime / (trainingDataIndex + 1); double remainingTime = avgTime * (trainingSets - trainingDataIndex); TimeSpan avgTSpan = TimeSpan.FromMilliseconds(avgTime); TimeSpan remTSpan = TimeSpan.FromMilliseconds(remainingTime); TimeSpan totTSpan = TimeSpan.FromMilliseconds(totalTime); string avgTS = $"{avgTSpan:ss\\.ffff}"; string remTS = $"{remTSpan:hh\\:mm\\:ss}"; string totTS = $"{totTSpan:hh\\:mm\\:ss}"; Save(nn, $"primary_{trainingDataIndex}"); Console.WriteLine($"Finished Training {trainingDataIndex}/{trainingSets} Passed:{totTS} Remaining:{remTS} Avg:{avgTS}"); }; nn.PropagateBackward(backpropagationData); } }
private static void Train(NeuralNetwork nn, float learningRate, int runs) { NNFeedData[] trainingInputData = new NNFeedData[360]; NNFeedData[] trainingTargetData = new NNFeedData[360]; for (int i = 0; i < 360; i++) { float x = i * (float)Math.PI / 180f; trainingInputData[i] = new NNFeedData(1, height: 1, depth: 1, data: new float[] { x }); trainingTargetData[i] = new NNFeedData(1, height: 1, depth: 1, data: new float[] { (float)Math.Sin(x) / 2f + 0.5f }); } NNTrainingData trainingData = new NNPreloadedTrainingData(trainingInputData, trainingTargetData); NNBackpropagationData backpropagationData = new NNBackpropagationData(trainingData, learningRate, (o, t) => o - t); for (int trainingRuns = 0; trainingRuns < runs; trainingRuns++) { nn.PropagateBackward(backpropagationData); } }
private static void TrainXOR(NeuralNetwork nn, float learningRate, int runs) { NNFeedData[] trainingInputData = new NNFeedData[(int)Math.Pow(2, 2)]; NNFeedData[] trainingTargetData = new NNFeedData[(int)Math.Pow(2, 2)]; for (int i = 0; i < Math.Pow(2, 2); i++) { int a = i % 2; int b = i / 2; int r = a ^ b; trainingInputData[i] = new NNFeedData(2, 1, 1, a, b); trainingTargetData[i] = new NNFeedData(1, 1, 1, r); } NNTrainingData trainingData = new NNPreloadedTrainingData(trainingInputData, trainingTargetData); NNBackpropagationData backpropagationData = new NNBackpropagationData(trainingData, learningRate, (o, t) => o - t); for (int trainingRuns = 0; trainingRuns < runs; trainingRuns++) { nn.PropagateBackward(backpropagationData); } }
internal static void ConvolutionTest() { InputLayer iL = new InputLayer(3, 3, 1); ConvolutionalLayer cL = new ConvolutionalLayer(1, 2, 1, 0, iL, ActivationFunctions.None()); //Debug.WriteLine("Filter ---------------"); int i = 0; for (int y = 0; y < 2; y++) { StringBuilder sb = new StringBuilder(); for (int x = 0; x < 2; x++, i++) { if (x != 0) { sb.Append(", "); } float w = 0.1f * (i + 1); cL.SetWeight(i, w); sb.Append(w); } //Debug.WriteLine(sb.ToString()); } FullyConnectedLayer fcL = new FullyConnectedLayer(1, cL, ActivationFunctions.None()); //Debug.WriteLine("Weights ---------------"); StringBuilder SB = new StringBuilder(); for (int j = 0; j < fcL.Weights; j++, i++) { if (j != 0) { SB.Append(", "); } float w = 1f; fcL.SetWeight(j, w); SB.Append(w); } //Debug.WriteLine(SB.ToString()); NeuralNetwork nn = new NeuralNetwork(iL, cL, fcL); float[,,] inputValues = new float[3, 3, 1]; i = 0; for (int z = 0; z < 1; z++) { //Debug.WriteLine($"{z} --------------------"); for (int y = 0; y < 3; y++) { StringBuilder sb = new StringBuilder(); for (int x = 0; x < 3; x++, i++) { //if (x != 0) // sb.Append(",\t"); inputValues[x, y, z] = i + 1; //sb.Append(inputValues[x, y, z]); } //Debug.WriteLine(sb.ToString()); } } NNFeedData inputData = new NNFeedData(inputValues); Debug.WriteLine(nn.FeedForward(inputData)[0]); NNFeedData targetData = new NNFeedData(1, 1, 1, -3.2f); NNTrainingData trainingData = new NNPreloadedTrainingData(new[] { inputData }, new [] { targetData }); NNBackpropagationData backpropagationData = new NNBackpropagationData(trainingData, 0.2f, (o, t) => o - t); //for (int j = 0; j < 1000; j++) { nn.PropagateBackward(backpropagationData); //} Debug.WriteLine(nn.FeedForward(inputData)[0]); //NNDetailedFeedData outputData = iL.FeedForward(inputData); //outputData = cL.FeedForward(outputData.OutputData); //Debug.WriteLine("Convolution Out"); //for (int z = 0; z < cL.Depth; z++) { // Debug.WriteLine($"{z} --------------------"); // for (int y = 0; y < cL.Height; y++) { // StringBuilder sb = new StringBuilder(); // for (int x = 0; x < cL.Width; x++) { // if (x != 0) // sb.Append(", "); // sb.Append($"{outputData.OutputData[outputData.OutputData.ToNeuronIndex(x, y, z)]}"); // } // Debug.WriteLine(sb.ToString()); // } //} //outputData = fcL.FeedForward(outputData.OutputData); //Debug.WriteLine("Fully Connected Out"); //Debug.WriteLine(outputData.OutputData[0]); }