/// <summary>
/// Construct the backpropagation trainer.
/// </summary>
/// <param name="theNetwork">The network to train.</param>
/// <param name="theTraining">The training data to use.</param>
/// <param name="theLearningRate">The learning rate. Can be changed as training runs.</param>
/// <param name="theMomentum">The momentum. Can be changed as training runs.</param>
public BackPropagation(BasicNetwork theNetwork, IList<BasicData> theTraining, double theLearningRate,
double theMomentum)
{
BatchSize = 500;
Stochastic = new MersenneTwisterGenerateRandom();
NesterovUpdate = true;
_network = theNetwork;
_training = theTraining;
LearningRate = theLearningRate;
Momentum = theMomentum;
_gradients = new GradientCalc(_network, new CrossEntropyErrorFunction(), this);
_lastDelta = new double[theNetwork.Weights.Length];
}
/// <summary>
/// Construct the backpropagation trainer.
/// </summary>
/// <param name="theNetwork">The network to train.</param>
/// <param name="theTraining">The training data to use.</param>
/// <param name="theLearningRate">The learning rate. Can be changed as training runs.</param>
/// <param name="theMomentum">The momentum. Can be changed as training runs.</param>
public BackPropagation(BasicNetwork theNetwork, IList <BasicData> theTraining, double theLearningRate,
double theMomentum)
{
BatchSize = 500;
Stochastic = new MersenneTwisterGenerateRandom();
NesterovUpdate = true;
_network = theNetwork;
_training = theTraining;
LearningRate = theLearningRate;
Momentum = theMomentum;
_gradients = new GradientCalc(_network, new CrossEntropyErrorFunction(), this);
_lastDelta = new double[theNetwork.Weights.Length];
}
/// <summary>
/// Construct a RPROP trainer.
/// </summary>
/// <param name="theNetwork">The network.</param>
/// <param name="theTraining">The training data.</param>
public ResilientPropagation(BasicNetwork theNetwork, IList <BasicData> theTraining)
{
_network = theNetwork;
_training = theTraining;
_gradients = new GradientCalc(_network, new CrossEntropyErrorFunction(), this);
_lastDelta = new double[theNetwork.Weights.Length];
_updateValues = new double[theNetwork.Weights.Length];
_lastGradients = new double[theNetwork.Weights.Length];
for (var i = 0; i < _updateValues.Length; i++)
{
_updateValues[i] = DEFAULT_INITIAL_UPDATE;
}
}
/// <inheritdoc />
public override void ComputeGradient(GradientCalc calc)
{
var prev = Owner.GetPreviousLayer(this);
var fromLayerSize = prev.TotalCount;
var toLayerSize = Count;
// Calculate the output for each filter (depth).
for (var dOutput = 0; dOutput < _numFilters; dOutput++)
{
for (var dInput = 0; dInput < _inDepth; dInput++)
{
ComputeGradient(calc, 0, 0, fromLayerSize, toLayerSize);
}
}
}
/// <summary>
/// Compute gradients for this layer.
/// </summary>
/// <param name="calc">The gradient calculator.</param>
/// <param name="inputOffset">The input offset.</param>
/// <param name="outputOffset">The output offset.</param>
/// <param name="fromLayerSize">The from layer size.</param>
/// <param name="toLayerSize">The to layer size.</param>
public void ComputeGradient(GradientCalc calc, int inputOffset, int outputOffset, int fromLayerSize,
int toLayerSize)
{
var prev = Owner.GetPreviousLayer(this);
var fromLayerIndex = prev.NeuronIndex;
var toLayerIndex = NeuronIndex;
var weightSize = WeightDepthUnit;
var outputSize = NeuronDepthUnit;
var index = WeightIndex + weightSize*inputOffset; // this.weightIndex[currentLevel];
var activation = Activation;
// handle weights
// array references are made method local to avoid one indirection
var layerDelta = calc.LayerDelta;
var weights = Owner.Weights;
var layerOutput = Owner.LayerOutput;
var layerSums = Owner.LayerSums;
var y = fromLayerIndex;
for (var yi = 0; yi < fromLayerSize; yi++)
{
var output = layerOutput[y];
double sum = 0;
var wi = index + yi;
for (var xi = 0; xi < toLayerSize; xi++, wi += fromLayerSize)
{
var x = xi + toLayerIndex;
if (prev.IsActive(yi) && IsActive(xi))
calc.Gradients[wi] += -(output*layerDelta[x]);
sum += weights[wi]*layerDelta[x];
}
layerDelta[y] = sum
*activation.DerivativeFunction(layerSums[y], layerOutput[y]);
y++;
}
}
/// <inheritdoc /> public abstract void ComputeGradient(GradientCalc calc);
/// <summary>
/// Construct a RPROP trainer.
/// </summary>
/// <param name="theNetwork">The network.</param>
/// <param name="theTraining">The training data.</param>
public ResilientPropagation(BasicNetwork theNetwork, IList<BasicData> theTraining)
{
LastError = 1.0;
_network = theNetwork;
_training = theTraining;
_gradients = new GradientCalc(_network, new CrossEntropyErrorFunction(), this);
_lastDelta = new double[theNetwork.Weights.Length];
_updateValues = new double[theNetwork.Weights.Length];
_lastGradients = new double[theNetwork.Weights.Length];
for (var i = 0; i < _updateValues.Length; i++)
{
_updateValues[i] = DEFAULT_INITIAL_UPDATE;
}
}
/// <inheritdoc />
public override void ComputeGradient(GradientCalc calc)
{
var prev = Owner.GetPreviousLayer(this);
var fromLayerSize = prev.TotalCount;
var toLayerSize = Count;
ComputeGradient(calc, 0, 0, fromLayerSize, toLayerSize);
}
/// <inheritdoc />
public void ComputeGradient(GradientCalc calc)
{
throw new NotImplementedException();
}
