/// <summary> /// Performs training iteration. /// </summary> public bool Iteration() { if (AttemptEpoch == MaxAttemptEpoch) { //Max epoch reached, try new attempt if (!NextAttempt()) { //Next attempt is not available return(false); } } //Next epoch ++AttemptEpoch; //Store previous iteration error _prevMSE = MSE; //Store previously accumulated weight gradients _weigthsGradsAcc.CopyTo(_weigthsPrevGradsAcc, 0); //Reset accumulated weight gradients _weigthsGradsAcc.Populate(0); //Get copy of the network weights double[] networkFlatWeights = _net.GetWeightsCopy(); //Network output layer shortcut FeedForwardNetwork.Layer outputLayer = _net.LayerCollection[_net.LayerCollection.Count - 1]; //Process gradient workers threads Parallel.ForEach(_gradientWorkerDataCollection, worker => { //---------------------------------------------------------------------------------------------------- //Gradient worker local variables List <double[]> layerInputCollection = new List <double[]>(_net.LayerCollection.Count); double[] gradients = new double[_net.NumOfNeurons]; double[] derivatives = new double[_net.NumOfNeurons]; //Reset gradient worker data worker.Reset(); //Loop over the planned range of samples for (int row = worker._fromRow; row <= worker._toRow; row++) { //---------------------------------------------------------------------------------------------------- //Reset of row dependents layerInputCollection.Clear(); gradients.Populate(0); derivatives.Populate(0); //---------------------------------------------------------------------------------------------------- //Network computation (collect layers inputs and activation derivatives) double[] computedOutputs = _net.Compute(_inputVectorCollection[row], layerInputCollection, derivatives); //---------------------------------------------------------------------------------------------------- //Compute output layer gradients and update error int outputLayerNeuronsFlatIdx = outputLayer.NeuronsStartFlatIdx; for (int neuronIdx = 0; neuronIdx < outputLayer.NumOfLayerNeurons; neuronIdx++) { double error = _outputVectorCollection[row][neuronIdx] - computedOutputs[neuronIdx]; gradients[outputLayerNeuronsFlatIdx] = derivatives[outputLayerNeuronsFlatIdx] * error; //Accumulate error worker._sumSquaredErr += error * error; ++outputLayerNeuronsFlatIdx; }//neuronIdx //---------------------------------------------------------------------------------------------------- //Hidden layers gradients for (int layerIdx = _net.LayerCollection.Count - 2; layerIdx >= 0; layerIdx--) { FeedForwardNetwork.Layer currLayer = _net.LayerCollection[layerIdx]; FeedForwardNetwork.Layer nextLayer = _net.LayerCollection[layerIdx + 1]; int currLayerNeuronFlatIdx = currLayer.NeuronsStartFlatIdx; for (int currLayerNeuronIdx = 0; currLayerNeuronIdx < currLayer.NumOfLayerNeurons; currLayerNeuronIdx++) { double sum = 0; for (int nextLayerNeuronIdx = 0; nextLayerNeuronIdx < nextLayer.NumOfLayerNeurons; nextLayerNeuronIdx++) { int weightFlatIdx = nextLayer.WeightsStartFlatIdx + nextLayerNeuronIdx * nextLayer.NumOfInputNodes + currLayerNeuronIdx; sum += gradients[nextLayer.NeuronsStartFlatIdx + nextLayerNeuronIdx] * networkFlatWeights[weightFlatIdx]; }//nextLayerNeuronIdx gradients[currLayerNeuronFlatIdx] = derivatives[currLayerNeuronFlatIdx] * sum; ++currLayerNeuronFlatIdx; } //currLayerNeuronIdx } //layerIdx //---------------------------------------------------------------------------------------------------- //Compute increments for gradients accumulator for (int layerIdx = 0; layerIdx < _net.LayerCollection.Count; layerIdx++) { FeedForwardNetwork.Layer layer = _net.LayerCollection[layerIdx]; double[] layerInputs = layerInputCollection[layerIdx]; int neuronFlatIdx = layer.NeuronsStartFlatIdx; int weightFlatIdx = layer.WeightsStartFlatIdx; int biasFlatIdx = layer.BiasesStartFlatIdx; for (int neuronIdx = 0; neuronIdx < layer.NumOfLayerNeurons; neuronIdx++) { //Weights gradients accumulation //Layer's inputs for (int inputIdx = 0; inputIdx < layer.NumOfInputNodes; inputIdx++) { worker._weigthsGradsAcc[weightFlatIdx] += layerInputs[inputIdx] * gradients[neuronFlatIdx]; ++weightFlatIdx; } //Layer's input bias worker._weigthsGradsAcc[biasFlatIdx] += FeedForwardNetwork.BiasValue * gradients[neuronFlatIdx]; ++neuronFlatIdx; ++biasFlatIdx; } //neuronIdx } //layerIdx } //Worker main loop });//Worker finish //Update of gradient accumulator and MSE by workers ProcessGradientWorkersData(); //Update all weights and biases Parallel.For(0, networkFlatWeights.Length, weightFlatIdx => { AdjustWeight(networkFlatWeights, weightFlatIdx); }); //Set adjusted weights back into the network under training _net.SetWeights(networkFlatWeights); return(true); }
/// <summary> /// Performs training iteration. /// </summary> public void Iteration() { //Next epoch ++_epoch; //Store previous iteration error _prevMSE = _lastMSE; //Reset iteration error _lastMSE = 0; //Accumulated weights gradients over all training data _weigthsGradsAcc.Populate(0); //Get copy of the network weights double[] networkFlatWeights = _net.GetWeights(); //Process gradient workers threads Parallel.ForEach(_workerRangeCollection, range => { //Gradient worker variables double sumOfErrSquares = 0; double[] weigthsGradsAccInput = new double[_net.NumOfWeights]; weigthsGradsAccInput.Populate(0); double[] neuronsGradients = new double[_net.NumOfNeurons]; double[] derivatives = new double[_net.NumOfNeurons]; List <double[]> layersInputs = new List <double[]>(_net.LayerCollection.Count); //Go parallely over all samples for (int row = range.FromRow; row <= range.ToRow; row++) { //Network computation (collect layers inputs and activation derivatives) layersInputs.Clear(); double[] computedOutputs = _net.Compute(_inputVectorCollection[row], layersInputs, derivatives); //Compute network neurons gradients //Compute output layer gradients and update last error FeedForwardNetwork.Layer outputLayer = _net.LayerCollection[_net.LayerCollection.Count - 1]; for (int neuronIdx = 0, outputLayerNeuronsFlatIdx = outputLayer.NeuronsStartFlatIdx; neuronIdx < outputLayer.NumOfLayerNeurons; neuronIdx++, outputLayerNeuronsFlatIdx++) { double error = _outputVectorCollection[row][neuronIdx] - computedOutputs[neuronIdx]; neuronsGradients[outputLayerNeuronsFlatIdx] = derivatives[outputLayerNeuronsFlatIdx] * error; //Accumulate error sumOfErrSquares += error * error; } //Hidden layers gradients for (int layerIdx = _net.LayerCollection.Count - 2; layerIdx >= 0; layerIdx--) { FeedForwardNetwork.Layer currLayer = _net.LayerCollection[layerIdx]; FeedForwardNetwork.Layer nextLayer = _net.LayerCollection[layerIdx + 1]; int currLayerNeuronFlatIdx = currLayer.NeuronsStartFlatIdx; for (int currLayerNeuronIdx = 0; currLayerNeuronIdx < currLayer.NumOfLayerNeurons; currLayerNeuronIdx++, currLayerNeuronFlatIdx++) { double sum = 0; for (int nextLayerNeuronIdx = 0; nextLayerNeuronIdx < nextLayer.NumOfLayerNeurons; nextLayerNeuronIdx++) { int nextLayerWeightFlatIdx = nextLayer.WeightsStartFlatIdx + nextLayerNeuronIdx * nextLayer.NumOfInputNodes + currLayerNeuronIdx; sum += neuronsGradients[nextLayer.NeuronsStartFlatIdx + nextLayerNeuronIdx] * networkFlatWeights[nextLayerWeightFlatIdx]; } neuronsGradients[currLayerNeuronFlatIdx] = derivatives[currLayerNeuronFlatIdx] * sum; } } //Compute increments for gradients accumulator for (int layerIdx = 0; layerIdx < _net.LayerCollection.Count; layerIdx++) { FeedForwardNetwork.Layer layer = _net.LayerCollection[layerIdx]; double[] layerInputs = layersInputs[layerIdx]; int neuronFlatIdx = layer.NeuronsStartFlatIdx; int biasFlatIdx = layer.BiasesStartFlatIdx; int weightFlatIdx = layer.WeightsStartFlatIdx; for (int neuronIdx = 0; neuronIdx < layer.NumOfLayerNeurons; neuronIdx++, neuronFlatIdx++, biasFlatIdx++) { //Weights gradients accumulation for (int inputIdx = 0; inputIdx < layer.NumOfInputNodes; inputIdx++, weightFlatIdx++) { weigthsGradsAccInput[weightFlatIdx] += layerInputs[inputIdx] * neuronsGradients[neuronFlatIdx]; } //Bias gradients accumulation weigthsGradsAccInput[biasFlatIdx] += neuronsGradients[neuronFlatIdx] * FeedForwardNetwork.BiasValue; } } }//Worker loop //Worker finish ProcessGradientWorkerOutputs(weigthsGradsAccInput, sumOfErrSquares); }); //Update all network weights and biases Parallel.For(0, networkFlatWeights.Length, weightFlatIdx => { PerformIRPropPlusWeightChange(networkFlatWeights, weightFlatIdx); }); _net.SetWeights(networkFlatWeights); //Store accumulated gradients for next iteration _weigthsGradsAcc.CopyTo(_weigthsPrevGradsAcc, 0); //Finish the MSE computation _lastMSE /= (double)(_inputVectorCollection.Count * _net.NumOfOutputValues); return; }