//Constructor
/// <summary>
/// Constructs new instance of linear regression trainer
/// </summary>
/// <param name="net">FF network to be trained</param>
/// <param name="inputVectorCollection">Predictors (input)</param>
/// <param name="outputVectorCollection">Ideal outputs (the same number of rows as number of inputs)</param>
/// <param name="maxEpoch">Maximum allowed training epochs</param>
/// <param name="rand">Random object to be used for adding a white-noise to predictors</param>
/// <param name="settings">Optional startup parameters of the trainer</param>
public LinRegrTrainer(FeedForwardNetwork net,
List <double[]> inputVectorCollection,
List <double[]> outputVectorCollection,
int maxEpoch,
System.Random rand,
LinRegrTrainerSettings settings = null
)
{
//Check network readyness
if (!net.Finalized)
{
throw new Exception("Can´t create LinRegr trainer. Network structure was not finalized.");
}
//Check network conditions
if (net.LayerCollection.Count != 1 || !(net.LayerCollection[0].Activation is Identity))
{
throw new Exception("Can´t create LinRegr trainer. Network structure is not complient (single layer having Identity activation).");
}
//Check samples conditions
if (inputVectorCollection.Count < inputVectorCollection[0].Length + 1)
{
throw new Exception("Can´t create LinRegr trainer. Insufficient number of training samples. Minimum is " + (inputVectorCollection[0].Length + 1).ToString() + ".");
}
//Parameters
_settings = settings;
if (_settings == null)
{
//Default parameters
_settings = new LinRegrTrainerSettings();
}
_net = net;
_inputVectorCollection = inputVectorCollection;
_outputVectorCollection = outputVectorCollection;
_outputSingleColMatrixCollection = new List <Matrix>(_net.NumOfOutputValues);
for (int outputIdx = 0; outputIdx < _net.NumOfOutputValues; outputIdx++)
{
Matrix outputSingleColMatrix = new Matrix(_outputVectorCollection.Count, 1);
for (int row = 0; row < _outputVectorCollection.Count; row++)
{
//Output
outputSingleColMatrix.Data[row][0] = _outputVectorCollection[row][outputIdx];
}
_outputSingleColMatrixCollection.Add(outputSingleColMatrix);
}
_rand = rand;
_maxEpoch = maxEpoch;
_epoch = 0;
_alphas = new double[_maxEpoch];
//Plan the iterations alphas
double coeff = (maxEpoch > 1) ? _settings.MaxStretch / (maxEpoch - 1) : 0;
for (int i = 0; i < _maxEpoch; i++)
{
_alphas[i] = _settings.HiNoiseIntensity - _settings.HiNoiseIntensity * Math.Tanh(i * coeff);
_alphas[i] = Math.Max(0, _alphas[i]);
}
//Ensure the last alpha is zero
_alphas[_maxEpoch - 1] = 0;
return;
}
//Constructor
/// <summary>
/// Instantiates the RPropTrainer
/// </summary>
/// <param name="net">
/// The feed forward network to be trained
/// </param>
/// <param name="inputVectorCollection">
/// Collection of the training input vectors
/// </param>
/// <param name="outputVectorCollection">
/// Collection of the desired outputs
/// </param>
/// Trainer parameters
/// <param name="settings">
/// </param>
public RPropTrainer(FeedForwardNetwork net,
List <double[]> inputVectorCollection,
List <double[]> outputVectorCollection,
RPropTrainerSettings settings = null
)
{
if (!net.Finalized)
{
throw new Exception("Can´t create trainer. Network structure was not finalized.");
}
_settings = settings;
if (_settings == null)
{
//Default parameters
_settings = new RPropTrainerSettings();
}
_net = net;
_inputVectorCollection = inputVectorCollection;
_outputVectorCollection = outputVectorCollection;
_weigthsGradsAcc = new double[_net.NumOfWeights];
_weigthsGradsAcc.Populate(0);
_weigthsPrevGradsAcc = new double[_net.NumOfWeights];
_weigthsPrevGradsAcc.Populate(0);
_weigthsPrevDeltas = new double[_net.NumOfWeights];
_weigthsPrevDeltas.Populate(_settings.IniDelta);
_weigthsPrevChanges = new double[_net.NumOfWeights];
_weigthsPrevChanges.Populate(0);
_prevMSE = 0;
_lastMSE = 0;
_epoch = 0;
//Parallel gradient workers / batch ranges preparation
_workerRangeCollection = new List <WorkerRange>();
int numOfWorkers = Math.Min(Environment.ProcessorCount, _inputVectorCollection.Count);
numOfWorkers = Math.Max(1, numOfWorkers);
int workerBatchSize = _inputVectorCollection.Count / numOfWorkers;
for (int workerIdx = 0, fromRow = 0; workerIdx < numOfWorkers; workerIdx++, fromRow += workerBatchSize)
{
WorkerRange workerRange = new WorkerRange();
workerRange.FromRow = fromRow;
if (workerIdx == numOfWorkers - 1)
{
workerRange.ToRow = _inputVectorCollection.Count - 1;
}
else
{
workerRange.ToRow = (fromRow + workerBatchSize) - 1;
}
_workerRangeCollection.Add(workerRange);
}
return;
}
/// <summary>
/// Creates a deep copy of this network
/// </summary>
public INonRecurrentNetwork DeepClone()
{
FeedForwardNetwork clone = new FeedForwardNetwork(NumOfInputValues, NumOfOutputValues);
clone._numOfNeurons = _numOfNeurons;
foreach (Layer layer in LayerCollection)
{
clone.LayerCollection.Add(layer.DeepClone());
}
clone._flatWeights = (double[])_flatWeights.Clone();
return(clone);
}
/// <summary>
/// Creates a deep copy of this network
/// </summary>
public INonRecurrentNetwork DeepClone()
{
FeedForwardNetwork clone = new FeedForwardNetwork(NumOfInputValues, NumOfOutputValues);
clone.NumOfNeurons = NumOfNeurons;
foreach (Layer layer in LayerCollection)
{
clone.LayerCollection.Add(layer.DeepClone());
}
clone._flatWeights = (double[])_flatWeights.Clone();
clone._isAllowedNguyenWidrowRandomization = _isAllowedNguyenWidrowRandomization;
return(clone);
}
//Constructor
/// <summary>
/// Instantiates the RPropTrainer
/// </summary>
/// <param name="net">
/// The feed forward network to be trained
/// </param>
/// <param name="inputVectorCollection">
/// Collection of the training input vectors
/// </param>
/// <param name="outputVectorCollection">
/// Collection of the desired outputs
/// </param>
/// Trainer parameters
/// <param name="settings">
/// </param>
public RPropTrainer(FeedForwardNetwork net,
List <double[]> inputVectorCollection,
List <double[]> outputVectorCollection,
RPropTrainerSettings settings = null
)
{
if (!net.Finalized)
{
throw new Exception("Can´t create trainer. Network structure was not finalized.");
}
_settings = settings;
if (_settings == null)
{
//Default parameters
_settings = new RPropTrainerSettings();
}
_net = net;
_inputVectorCollection = inputVectorCollection;
_outputVectorCollection = outputVectorCollection;
_weigthsGradsAcc = new double[_net.NumOfWeights];
_weigthsGradsAcc.Populate(0);
_weigthsPrevGradsAcc = new double[_net.NumOfWeights];
_weigthsPrevGradsAcc.Populate(0);
_weigthsPrevDeltas = new double[_net.NumOfWeights];
_weigthsPrevDeltas.Populate(_settings.IniDelta);
_weigthsPrevChanges = new double[_net.NumOfWeights];
_weigthsPrevChanges.Populate(0);
_prevMSE = 0;
MSE = 0;
Epoch = 0;
//Parallel gradient workers (batch ranges) preparation
int numOfWorkers = Math.Max(1, Math.Min(Environment.ProcessorCount - 1, _inputVectorCollection.Count));
_gradientWorkerDataCollection = new GradientWorkerData[numOfWorkers];
int workerBatchSize = _inputVectorCollection.Count / numOfWorkers;
for (int workerIdx = 0, fromRow = 0; workerIdx < numOfWorkers; workerIdx++, fromRow += workerBatchSize)
{
GradientWorkerData gwd = new GradientWorkerData
(
fromRow: fromRow,
toRow: (workerIdx == numOfWorkers - 1 ? _inputVectorCollection.Count - 1 : (fromRow + workerBatchSize) - 1),
numOfWeights: _net.NumOfWeights
);
_gradientWorkerDataCollection[workerIdx] = gwd;
}
return;
}
//Constructor
/// <summary>
/// Constructs an initialized instance
/// </summary>
/// <param name="net">FF network to be trained</param>
/// <param name="inputVectorCollection">Predictors (input)</param>
/// <param name="outputVectorCollection">Ideal outputs (the same number of rows as number of inputs)</param>
/// <param name="rand">Random object to be used for adding a white-noise to predictors</param>
/// <param name="settings">Startup parameters of the trainer</param>
public QRDRegrTrainer(FeedForwardNetwork net,
List <double[]> inputVectorCollection,
List <double[]> outputVectorCollection,
QRDRegrTrainerSettings settings,
Random rand
)
{
//Check network readyness
if (!net.Finalized)
{
throw new Exception("Can´t create trainer. Network structure was not finalized.");
}
//Check network conditions
if (net.LayerCollection.Count != 1 || !(net.LayerCollection[0].Activation is Identity))
{
throw new Exception("Can´t create trainer. Network structure is not complient (single layer having Identity activation).");
}
//Check samples conditions
if (inputVectorCollection.Count < inputVectorCollection[0].Length + 1)
{
throw new Exception($"Can´t create trainer. Insufficient number of training samples {inputVectorCollection.Count}. Minimum is {(inputVectorCollection[0].Length + 1)}.");
}
//Parameters
_settings = settings;
MaxAttempt = _settings.NumOfAttempts;
MaxAttemptEpoch = _settings.NumOfAttemptEpochs;
_net = net;
_rand = rand;
_inputVectorCollection = inputVectorCollection;
_outputVectorCollection = outputVectorCollection;
_outputSingleColMatrixCollection = new List <Matrix>(_net.NumOfOutputValues);
for (int outputIdx = 0; outputIdx < _net.NumOfOutputValues; outputIdx++)
{
Matrix outputSingleColMatrix = new Matrix(_outputVectorCollection.Count, 1);
for (int row = 0; row < _outputVectorCollection.Count; row++)
{
//Output
outputSingleColMatrix.Data[row][0] = _outputVectorCollection[row][outputIdx];
}
_outputSingleColMatrixCollection.Add(outputSingleColMatrix);
}
//Start training attempt
Attempt = 0;
NextAttempt();
return;
}
//Constructor
/// <summary>
/// Instantiates the RPropTrainer
/// </summary>
/// <param name="net"> The feed forward network to be trained </param>
/// <param name="inputVectorCollection"> Collection of the training input vectors </param>
/// <param name="outputVectorCollection"> Collection of the desired outputs </param>
/// <param name="settings"> Trainer parameters </param>
/// <param name="rand">Random object to be used</param>
public RPropTrainer(FeedForwardNetwork net,
List <double[]> inputVectorCollection,
List <double[]> outputVectorCollection,
RPropTrainerSettings settings,
Random rand
)
{
if (!net.Finalized)
{
throw new Exception("Can´t create trainer. Network structure was not finalized.");
}
_settings = settings;
MaxAttempt = _settings.NumOfAttempts;
MaxAttemptEpoch = _settings.NumOfAttemptEpochs;
_net = net;
_rand = rand;
_inputVectorCollection = inputVectorCollection;
_outputVectorCollection = outputVectorCollection;
_weigthsGradsAcc = new double[_net.NumOfWeights];
_weigthsPrevGradsAcc = new double[_net.NumOfWeights];
_weigthsPrevDeltas = new double[_net.NumOfWeights];
_weigthsPrevChanges = new double[_net.NumOfWeights];
//Parallel gradient workers (batch ranges) preparation
int numOfWorkers = Math.Max(1, Math.Min(Environment.ProcessorCount - 1, _inputVectorCollection.Count));
_gradientWorkerDataCollection = new GradientWorkerData[numOfWorkers];
int workerBatchSize = _inputVectorCollection.Count / numOfWorkers;
for (int workerIdx = 0, fromRow = 0; workerIdx < numOfWorkers; workerIdx++, fromRow += workerBatchSize)
{
GradientWorkerData gwd = new GradientWorkerData
(
fromRow: fromRow,
toRow: (workerIdx == numOfWorkers - 1 ? _inputVectorCollection.Count - 1 : (fromRow + workerBatchSize) - 1),
numOfWeights: _net.NumOfWeights
);
_gradientWorkerDataCollection[workerIdx] = gwd;
}
InfoMessage = string.Empty;
//Start training attempt
Attempt = 0;
NextAttempt();
return;
}
//Constructor
/// <summary>
/// Constructs an initialized instance
/// </summary>
/// <param name="net">FF network to be trained</param>
/// <param name="inputVectorCollection">Predictors (input)</param>
/// <param name="outputVectorCollection">Ideal outputs (the same number of rows as number of inputs)</param>
/// <param name="settings">Startup parameters of the trainer</param>
public ElasticRegrTrainer(FeedForwardNetwork net,
List <double[]> inputVectorCollection,
List <double[]> outputVectorCollection,
ElasticRegrTrainerSettings settings
)
{
//Check network readyness
if (!net.Finalized)
{
throw new Exception("Can´t create trainer. Network structure was not finalized.");
}
//Check network conditions
if (net.LayerCollection.Count != 1 || !(net.LayerCollection[0].Activation is Identity))
{
throw new Exception("Can´t create trainer. Network structure is not complient (single layer having Identity activation).");
}
//Check samples conditions
if (inputVectorCollection.Count == 0)
{
throw new Exception("Can´t create trainer. Missing training samples.");
}
//Collections
_inputVectorCollection = new List <double[]>(inputVectorCollection);
_outputVectorCollection = new List <double[]>(outputVectorCollection);
var rangePartitioner = Partitioner.Create(0, _inputVectorCollection.Count);
_parallelRanges = new List <Tuple <int, int> >(rangePartitioner.GetDynamicPartitions());
//Parameters
_settings = settings;
MaxAttempt = _settings.NumOfAttempts;
MaxAttemptEpoch = _settings.NumOfAttemptEpochs;
Attempt = 1;
AttemptEpoch = 0;
_net = net;
_gamma = _settings.Lambda * _settings.Alpha;
return;
}
//Constructor
/// <summary>
/// Constructs an initialized instance
/// </summary>
/// <param name="net">FF network to be trained</param>
/// <param name="inputVectorCollection">Predictors (input)</param>
/// <param name="outputVectorCollection">Ideal outputs (the same number of rows as number of inputs)</param>
/// <param name="settings">Optional startup parameters of the trainer</param>
/// <param name="rand">Random object to be used</param>
public RidgeRegrTrainer(FeedForwardNetwork net,
List <double[]> inputVectorCollection,
List <double[]> outputVectorCollection,
RidgeRegrTrainerSettings settings,
Random rand
)
{
//Check network readyness
if (!net.Finalized)
{
throw new Exception("Can´t create trainer. Network structure was not finalized.");
}
//Check network conditions
if (net.LayerCollection.Count != 1 || !(net.LayerCollection[0].Activation is Identity))
{
throw new Exception("Can´t create trainer. Network structure is not complient (single layer having Identity activation).");
}
//Check samples conditions
if (inputVectorCollection.Count == 0)
{
throw new Exception("Can´t create trainer. Missing training samples.");
}
//Collections
_inputVectorCollection = new List <double[]>(inputVectorCollection);
_outputVectorCollection = new List <double[]>(outputVectorCollection);
//Parameters
_settings = settings;
MaxAttempt = _settings.NumOfAttempts;
MaxAttemptEpoch = _settings.NumOfAttemptEpochs;
Attempt = 1;
AttemptEpoch = 0;
_net = net;
_rand = rand;
_outputSingleColVectorCollection = new List <Vector>(_net.NumOfOutputValues);
for (int outputIdx = 0; outputIdx < _net.NumOfOutputValues; outputIdx++)
{
Vector outputSingleColVector = new Vector(outputVectorCollection.Count);
for (int row = 0; row < outputVectorCollection.Count; row++)
{
//Output
outputSingleColVector.Data[row] = outputVectorCollection[row][outputIdx];
}
_outputSingleColVectorCollection.Add(outputSingleColVector);
}
//Lambda seeker
_lambdaSeeker = new ParamSeeker(_settings.LambdaSeekerCfg);
_currLambda = 1e6;
//Matrix setup
Matrix predictorsMatrix = new Matrix(inputVectorCollection.Count, _net.NumOfInputValues + 1);
for (int row = 0; row < inputVectorCollection.Count; row++)
{
//Predictors
for (int col = 0; col < _net.NumOfInputValues; col++)
{
predictorsMatrix.Data[row][col] = inputVectorCollection[row][col];
}
//Add constant bias to predictors
predictorsMatrix.Data[row][_net.NumOfInputValues] = 1;
}
_transposedPredictorsMatrix = predictorsMatrix.Transpose();
_baseSquareMatrix = _transposedPredictorsMatrix * predictorsMatrix;
return;
}