/// <summary>
/// Performs training iteration.
/// </summary>
public bool Iteration()
{
if (AttemptEpoch == MaxAttemptEpoch)
{
//Max epoch reached, try new attempt
if (!NextAttempt())
{
//Next attempt is not available
return(false);
}
}
//Next epoch
++AttemptEpoch;
//Store previous iteration error
_prevMSE = MSE;
//Store previously accumulated weight gradients
_weigthsGradsAcc.CopyTo(_weigthsPrevGradsAcc, 0);
//Reset accumulated weight gradients
_weigthsGradsAcc.Populate(0);
//Get copy of the network weights
double[] networkFlatWeights = _net.GetWeightsCopy();
//Network output layer shortcut
FeedForwardNetwork.Layer outputLayer = _net.LayerCollection[_net.LayerCollection.Count - 1];
//Process gradient workers threads
Parallel.ForEach(_gradientWorkerDataCollection, worker =>
{
//----------------------------------------------------------------------------------------------------
//Gradient worker local variables
List <double[]> layerInputCollection = new List <double[]>(_net.LayerCollection.Count);
double[] gradients = new double[_net.NumOfNeurons];
double[] derivatives = new double[_net.NumOfNeurons];
//Reset gradient worker data
worker.Reset();
//Loop over the planned range of samples
for (int row = worker._fromRow; row <= worker._toRow; row++)
{
//----------------------------------------------------------------------------------------------------
//Reset of row dependents
layerInputCollection.Clear();
gradients.Populate(0);
derivatives.Populate(0);
//----------------------------------------------------------------------------------------------------
//Network computation (collect layers inputs and activation derivatives)
double[] computedOutputs = _net.Compute(_inputVectorCollection[row], layerInputCollection, derivatives);
//----------------------------------------------------------------------------------------------------
//Compute output layer gradients and update error
int outputLayerNeuronsFlatIdx = outputLayer.NeuronsStartFlatIdx;
for (int neuronIdx = 0; neuronIdx < outputLayer.NumOfLayerNeurons; neuronIdx++)
{
double error = _outputVectorCollection[row][neuronIdx] - computedOutputs[neuronIdx];
gradients[outputLayerNeuronsFlatIdx] = derivatives[outputLayerNeuronsFlatIdx] * error;
//Accumulate error
worker._sumSquaredErr += error * error;
++outputLayerNeuronsFlatIdx;
}//neuronIdx
//----------------------------------------------------------------------------------------------------
//Hidden layers gradients
for (int layerIdx = _net.LayerCollection.Count - 2; layerIdx >= 0; layerIdx--)
{
FeedForwardNetwork.Layer currLayer = _net.LayerCollection[layerIdx];
FeedForwardNetwork.Layer nextLayer = _net.LayerCollection[layerIdx + 1];
int currLayerNeuronFlatIdx = currLayer.NeuronsStartFlatIdx;
for (int currLayerNeuronIdx = 0; currLayerNeuronIdx < currLayer.NumOfLayerNeurons; currLayerNeuronIdx++)
{
double sum = 0;
for (int nextLayerNeuronIdx = 0; nextLayerNeuronIdx < nextLayer.NumOfLayerNeurons; nextLayerNeuronIdx++)
{
int weightFlatIdx = nextLayer.WeightsStartFlatIdx + nextLayerNeuronIdx * nextLayer.NumOfInputNodes + currLayerNeuronIdx;
sum += gradients[nextLayer.NeuronsStartFlatIdx + nextLayerNeuronIdx] * networkFlatWeights[weightFlatIdx];
}//nextLayerNeuronIdx
gradients[currLayerNeuronFlatIdx] = derivatives[currLayerNeuronFlatIdx] * sum;
++currLayerNeuronFlatIdx;
} //currLayerNeuronIdx
} //layerIdx
//----------------------------------------------------------------------------------------------------
//Compute increments for gradients accumulator
for (int layerIdx = 0; layerIdx < _net.LayerCollection.Count; layerIdx++)
{
FeedForwardNetwork.Layer layer = _net.LayerCollection[layerIdx];
double[] layerInputs = layerInputCollection[layerIdx];
int neuronFlatIdx = layer.NeuronsStartFlatIdx;
int weightFlatIdx = layer.WeightsStartFlatIdx;
int biasFlatIdx = layer.BiasesStartFlatIdx;
for (int neuronIdx = 0; neuronIdx < layer.NumOfLayerNeurons; neuronIdx++)
{
//Weights gradients accumulation
//Layer's inputs
for (int inputIdx = 0; inputIdx < layer.NumOfInputNodes; inputIdx++)
{
worker._weigthsGradsAcc[weightFlatIdx] += layerInputs[inputIdx] * gradients[neuronFlatIdx];
++weightFlatIdx;
}
//Layer's input bias
worker._weigthsGradsAcc[biasFlatIdx] += FeedForwardNetwork.BiasValue * gradients[neuronFlatIdx];
++neuronFlatIdx;
++biasFlatIdx;
} //neuronIdx
} //layerIdx
} //Worker main loop
});//Worker finish
//Update of gradient accumulator and MSE by workers
ProcessGradientWorkersData();
//Update all weights and biases
Parallel.For(0, networkFlatWeights.Length, weightFlatIdx =>
{
AdjustWeight(networkFlatWeights, weightFlatIdx);
});
//Set adjusted weights back into the network under training
_net.SetWeights(networkFlatWeights);
return(true);
}
/// <summary>
/// Performs training iteration.
/// </summary>
public void Iteration()
{
//Next epoch
++_epoch;
//Store previous iteration error
_prevMSE = _lastMSE;
//Reset iteration error
_lastMSE = 0;
//Accumulated weights gradients over all training data
_weigthsGradsAcc.Populate(0);
//Get copy of the network weights
double[] networkFlatWeights = _net.GetWeights();
//Process gradient workers threads
Parallel.ForEach(_workerRangeCollection, range =>
{
//Gradient worker variables
double sumOfErrSquares = 0;
double[] weigthsGradsAccInput = new double[_net.NumOfWeights];
weigthsGradsAccInput.Populate(0);
double[] neuronsGradients = new double[_net.NumOfNeurons];
double[] derivatives = new double[_net.NumOfNeurons];
List <double[]> layersInputs = new List <double[]>(_net.LayerCollection.Count);
//Go parallely over all samples
for (int row = range.FromRow; row <= range.ToRow; row++)
{
//Network computation (collect layers inputs and activation derivatives)
layersInputs.Clear();
double[] computedOutputs = _net.Compute(_inputVectorCollection[row], layersInputs, derivatives);
//Compute network neurons gradients
//Compute output layer gradients and update last error
FeedForwardNetwork.Layer outputLayer = _net.LayerCollection[_net.LayerCollection.Count - 1];
for (int neuronIdx = 0, outputLayerNeuronsFlatIdx = outputLayer.NeuronsStartFlatIdx; neuronIdx < outputLayer.NumOfLayerNeurons; neuronIdx++, outputLayerNeuronsFlatIdx++)
{
double error = _outputVectorCollection[row][neuronIdx] - computedOutputs[neuronIdx];
neuronsGradients[outputLayerNeuronsFlatIdx] = derivatives[outputLayerNeuronsFlatIdx] * error;
//Accumulate error
sumOfErrSquares += error * error;
}
//Hidden layers gradients
for (int layerIdx = _net.LayerCollection.Count - 2; layerIdx >= 0; layerIdx--)
{
FeedForwardNetwork.Layer currLayer = _net.LayerCollection[layerIdx];
FeedForwardNetwork.Layer nextLayer = _net.LayerCollection[layerIdx + 1];
int currLayerNeuronFlatIdx = currLayer.NeuronsStartFlatIdx;
for (int currLayerNeuronIdx = 0; currLayerNeuronIdx < currLayer.NumOfLayerNeurons; currLayerNeuronIdx++, currLayerNeuronFlatIdx++)
{
double sum = 0;
for (int nextLayerNeuronIdx = 0; nextLayerNeuronIdx < nextLayer.NumOfLayerNeurons; nextLayerNeuronIdx++)
{
int nextLayerWeightFlatIdx = nextLayer.WeightsStartFlatIdx + nextLayerNeuronIdx * nextLayer.NumOfInputNodes + currLayerNeuronIdx;
sum += neuronsGradients[nextLayer.NeuronsStartFlatIdx + nextLayerNeuronIdx] * networkFlatWeights[nextLayerWeightFlatIdx];
}
neuronsGradients[currLayerNeuronFlatIdx] = derivatives[currLayerNeuronFlatIdx] * sum;
}
}
//Compute increments for gradients accumulator
for (int layerIdx = 0; layerIdx < _net.LayerCollection.Count; layerIdx++)
{
FeedForwardNetwork.Layer layer = _net.LayerCollection[layerIdx];
double[] layerInputs = layersInputs[layerIdx];
int neuronFlatIdx = layer.NeuronsStartFlatIdx;
int biasFlatIdx = layer.BiasesStartFlatIdx;
int weightFlatIdx = layer.WeightsStartFlatIdx;
for (int neuronIdx = 0; neuronIdx < layer.NumOfLayerNeurons; neuronIdx++, neuronFlatIdx++, biasFlatIdx++)
{
//Weights gradients accumulation
for (int inputIdx = 0; inputIdx < layer.NumOfInputNodes; inputIdx++, weightFlatIdx++)
{
weigthsGradsAccInput[weightFlatIdx] += layerInputs[inputIdx] * neuronsGradients[neuronFlatIdx];
}
//Bias gradients accumulation
weigthsGradsAccInput[biasFlatIdx] += neuronsGradients[neuronFlatIdx] * FeedForwardNetwork.BiasValue;
}
}
}//Worker loop
//Worker finish
ProcessGradientWorkerOutputs(weigthsGradsAccInput, sumOfErrSquares);
});
//Update all network weights and biases
Parallel.For(0, networkFlatWeights.Length, weightFlatIdx =>
{
PerformIRPropPlusWeightChange(networkFlatWeights, weightFlatIdx);
});
_net.SetWeights(networkFlatWeights);
//Store accumulated gradients for next iteration
_weigthsGradsAcc.CopyTo(_weigthsPrevGradsAcc, 0);
//Finish the MSE computation
_lastMSE /= (double)(_inputVectorCollection.Count * _net.NumOfOutputValues);
return;
}
