// wrap an actual model in a surrograte
public GradientBoostedTreesModelSurrogate(IRegressionProblemData trainingProblemData, uint seed,
ILossFunction lossFunction, int iterations, int maxSize, double r, double m, double nu,
IGradientBoostedTreesModel model)
: this(trainingProblemData, seed, lossFunction, iterations, maxSize, r, m, nu)
{
this.actualModel = model;
}
public GbmState(IRegressionProblemData problemData, ILossFunction lossFunction, uint randSeed, int maxSize, double r, double m, double nu) {
// default settings for MaxSize, Nu and R
this.maxSize = maxSize;
this.nu = nu;
this.r = r;
this.m = m;
this.randSeed = randSeed;
random = new MersenneTwister(randSeed);
this.problemData = problemData;
this.trainingRows = problemData.TrainingIndices.ToArray();
this.testRows = problemData.TestIndices.ToArray();
this.lossFunction = lossFunction;
int nRows = trainingRows.Length;
y = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, trainingRows).ToArray();
treeBuilder = new RegressionTreeBuilder(problemData, random);
activeIdx = Enumerable.Range(0, nRows).ToArray();
var zeros = Enumerable.Repeat(0.0, nRows).ToArray();
double f0 = lossFunction.LineSearch(y, zeros, activeIdx, 0, nRows - 1); // initial constant value (mean for squared errors)
pred = Enumerable.Repeat(f0, nRows).ToArray();
predTest = Enumerable.Repeat(f0, testRows.Length).ToArray();
pseudoRes = new double[nRows];
models = new List<IRegressionModel>();
weights = new List<double>();
// add constant model
models.Add(new ConstantModel(f0, problemData.TargetVariable));
weights.Add(1.0);
}
public LinearModel(ILossFunction loss, float learningRate, float L2, float L1)
{
_loss = loss;
_learningRate = learningRate;
_L2 = L2;
_L1 = L1;
}
public DenseLayerNoBias(int nbInput, int nbOutput, IActivation activation, ILossFunction lossFunction) : base(nbInput, nbOutput, activation, lossFunction)
{
errors = new double[nbOutput];
derivatives = new double[nbOutput];
weightsError = new double[nbInput, nbOutput];
}
public Network(List <Layer> layers, ILossFunction lossFunction)
{
Seed = 123; // TODO: temp
Layers = layers;
LossFunction = lossFunction;
InitializeWeightMatricies();
}
/// <summary>
/// Connect networks with 0th entry as input
/// </summary>
/// <param name="nets"></param>
public NeuralNetwork(params NeuralNetwork[] nets)
{
int layerCount = 1;
for (int i = 0; i < nets.Length; i++)
{
layerCount += (nets[i].LayerCount - 1);
}
this.layers = new int[layerCount];
this.function = new IActivationFunction[layerCount];
this.lossFunc = nets[0].LossFunction;
this.Optimizer = nets[0].Optimizer;
this.Weights = new Matrix[layerCount];
this.Biases = new Vector[layerCount];
int idx = 1;
for (int i = 0; i < nets.Length; i++)
{
for (int j = 1; j < nets[i].LayerCount; j++)
{
Weights[idx] = nets[i].Weights[j];
Biases[idx] = nets[i].Biases[j];
layers[idx] = nets[i].Layers[j];
function[idx] = nets[i].ActivationFunctions[j];
idx++;
}
}
this.Solver = new GradientSolver(this);
}
// processes potential splits from the queue as long as splits are left and the maximum size of the tree is not reached
private void CreateRegressionTreeFromQueue(int maxNodes, ILossFunction lossFunction)
{
while (queue.Any() && curTreeNodeIdx + 1 < maxNodes) // two nodes are created in each loop
{
var f = queue[queue.Count - 1]; // last element has the largest improvement
queue.RemoveAt(queue.Count - 1);
var startIdx = f.StartIdx;
var endIdx = f.EndIndex;
Debug.Assert(endIdx - startIdx >= 0);
Debug.Assert(startIdx >= 0);
Debug.Assert(endIdx < internalIdx.Length);
// split partition into left and right
int splitIdx;
SplitPartition(f.StartIdx, f.EndIndex, f.SplittingVariable, f.SplittingThreshold, out splitIdx);
Debug.Assert(splitIdx + 1 <= endIdx);
Debug.Assert(startIdx <= splitIdx);
// create two leaf nodes (and enqueue best splits for both)
var leftTreeIdx = CreateLeafNode(startIdx, splitIdx, lossFunction);
var rightTreeIdx = CreateLeafNode(splitIdx + 1, endIdx, lossFunction);
// overwrite existing leaf node with an internal node
tree[f.ParentNodeIdx] = new RegressionTreeModel.TreeNode(f.SplittingVariable, f.SplittingThreshold, leftTreeIdx, rightTreeIdx);
}
}
// returns the index of the newly created tree node
private int CreateLeafNode(int startIdx, int endIdx, ILossFunction lossFunction)
{
// write a leaf node
var val = lossFunction.LineSearch(originalY, curPred, internalIdx, startIdx, endIdx);
tree[curTreeNodeIdx] = new RegressionTreeModel.TreeNode(RegressionTreeModel.TreeNode.NO_VARIABLE, val);
EnqueuePartitionSplit(curTreeNodeIdx, startIdx, endIdx);
curTreeNodeIdx++;
return(curTreeNodeIdx - 1);
}
// create only the surrogate model without an actual model
public GradientBoostedTreesModelSurrogate(IRegressionProblemData trainingProblemData, uint seed, ILossFunction lossFunction, int iterations, int maxSize, double r, double m, double nu)
: base("Gradient boosted tree model", string.Empty) {
this.trainingProblemData = trainingProblemData;
this.seed = seed;
this.lossFunction = lossFunction;
this.iterations = iterations;
this.maxSize = maxSize;
this.r = r;
this.m = m;
this.nu = nu;
}
public LayerBase(int nbInput, int nbOutput, IActivation activation, ILossFunction lossFunction)
{
this.InputError = new double[nbInput];
this.NbOutput = nbOutput;
this.NbInput = nbInput;
this.Output = new double[nbOutput];
this.Weights = new NNMatrix(nbInput, nbOutput);
this.NonActivatedOutput = new double[nbOutput];
this.Activation = activation;
this.LossFunction = lossFunction;
}
protected override void DoLossDerivative(Tensor correct, ILossFunction lossFunction, Tensor dy)
{
var tStorage = correct.Storage as GpuStorage ?? throw new UnsupportedStorageException(nameof(correct));
var dyStorage = dy.Storage as GpuStorage ?? throw new UnsupportedStorageException(nameof(dy));
var dO = _storage.DeviceStorage;
var dT = tStorage.DeviceStorage;
var dDy = dyStorage.DeviceStorage;
_context.Methods.LossDerivative(dO, dT, dDy, lossFunction, dy.Storage.Descriptor);
}
protected override void DoLoss(Tensor correct, ILossFunction lossFunction, Tensor loss)
{
var tStorage = correct.Storage as GpuStorage ?? throw new UnsupportedStorageException(nameof(correct));
var lossStorage = loss.Storage as GpuStorage ?? throw new UnsupportedStorageException(nameof(loss));
var dO = _storage.DeviceStorage;
var dT = tStorage.DeviceStorage;
var dLoss = lossStorage.DeviceStorage;
_context.Methods.Loss(dO, dT, dLoss, lossFunction, _storage.Descriptor);
}
// create only the surrogate model without an actual model
public GradientBoostedTreesModelSurrogate(IRegressionProblemData trainingProblemData, uint seed, ILossFunction lossFunction, int iterations, int maxSize, double r, double m, double nu)
: base("Gradient boosted tree model", string.Empty)
{
this.trainingProblemData = trainingProblemData;
this.seed = seed;
this.lossFunction = lossFunction;
this.iterations = iterations;
this.maxSize = maxSize;
this.r = r;
this.m = m;
this.nu = nu;
}
// for custom stepping & termination
public static IGbmState CreateGbmState(IRegressionProblemData problemData, ILossFunction lossFunction, uint randSeed, int maxSize = 3, double r = 0.66, double m = 0.5, double nu = 0.01)
{
// check input variables. Only double variables are allowed.
var invalidInputs =
problemData.AllowedInputVariables.Where(name => !problemData.Dataset.VariableHasType <double>(name));
if (invalidInputs.Any())
{
throw new NotSupportedException("Gradient tree boosting only supports real-valued variables. Unsupported inputs: " + string.Join(", ", invalidInputs));
}
return(new GbmState(problemData, lossFunction, randSeed, maxSize, r, m, nu));
}
private GradientBoostedTreesModelSurrogate(GradientBoostedTreesModelSurrogate original, Cloner cloner)
: base(original, cloner) {
if (original.actualModel != null) this.actualModel = cloner.Clone(original.actualModel);
this.trainingProblemData = cloner.Clone(original.trainingProblemData);
this.lossFunction = cloner.Clone(original.lossFunction);
this.seed = original.seed;
this.iterations = original.iterations;
this.maxSize = original.maxSize;
this.r = original.r;
this.m = original.m;
this.nu = original.nu;
}
public Network(Layer layer, IOptimizer optimizer, ILossFunction lossFunction)
{
this.random = RandomProvider.GetRandom();
this.inputLayer = layer;
this.layerCollection = new Collection <Layer>();
this.optimizer = optimizer;
this.lossFunction = lossFunction;
do
{
this.layerCollection.Add(layer);
layer = layer.Next;
} while (layer != null);
}
// create only the surrogate model without an actual model
private GradientBoostedTreesModelSurrogate(IRegressionProblemData trainingProblemData, uint seed,
ILossFunction lossFunction, int iterations, int maxSize, double r, double m, double nu)
: base(trainingProblemData.TargetVariable, "Gradient boosted tree model", string.Empty)
{
this.trainingProblemData = trainingProblemData;
this.seed = seed;
this.lossFunction = lossFunction;
this.iterations = iterations;
this.maxSize = maxSize;
this.r = r;
this.m = m;
this.nu = nu;
actualModel = new Lazy <IGradientBoostedTreesModel>(() => RecalculateModel());
}
private GradientBoostedTreesModelSurrogate(GradientBoostedTreesModelSurrogate original, Cloner cloner)
: base(original, cloner)
{
if (original.actualModel != null)
{
this.actualModel = cloner.Clone(original.actualModel);
}
this.trainingProblemData = cloner.Clone(original.trainingProblemData);
this.lossFunction = cloner.Clone(original.lossFunction);
this.seed = original.seed;
this.iterations = original.iterations;
this.maxSize = original.maxSize;
this.r = original.r;
this.m = original.m;
this.nu = original.nu;
}
public Model(IEnumerable <Layer> collection, IOptimizer optimizer, ILossFunction lossFunction)
{
this.random = RandomProvider.GetRandom();
this.layerCollection = new Collection <Layer>();
this.optimizer = optimizer;
this.lossFunction = lossFunction;
foreach (Layer layer in collection)
{
if (this.layerCollection.Count > 0)
{
var previousLayer = this.layerCollection[this.layerCollection.Count - 1];
previousLayer.Next = layer;
layer.Previous = previousLayer;
}
this.layerCollection.Add(layer);
}
}
private GradientBoostedTreesModelSurrogate(GradientBoostedTreesModelSurrogate original, Cloner cloner)
: base(original, cloner)
{
IGradientBoostedTreesModel clonedModel = null;
if (original.actualModel.IsValueCreated)
{
clonedModel = cloner.Clone(original.ActualModel);
}
actualModel = new Lazy <IGradientBoostedTreesModel>(CreateLazyInitFunc(clonedModel)); // only capture clonedModel in the closure
this.trainingProblemData = cloner.Clone(original.trainingProblemData);
this.lossFunction = cloner.Clone(original.lossFunction);
this.seed = original.seed;
this.iterations = original.iterations;
this.maxSize = original.maxSize;
this.r = original.r;
this.m = original.m;
this.nu = original.nu;
}
/// <summary>Defaults.</summary>
/// <param name="d">The Descriptor to process.</param>
/// <param name="x">The Vector to process.</param>
/// <param name="y">The Vector to process.</param>
/// <param name="activationFunction">The activation.</param>
/// <param name="outputFunction">The ouput function for hidden nodes (Optional).</param>
/// <param name="epsilon">epsilon</param>
/// <returns>A Network.</returns>
public static Network Create(this Network network, Descriptor d, Matrix x, Vector y, IFunction activationFunction, IFunction outputFunction = null,
double epsilon = double.NaN, ILossFunction lossFunction = null)
{
// set output to number of choices of available
// 1 if only two choices
int distinct = y.Distinct().Count();
int output = distinct > 2 ? distinct : 1;
// identity funciton for bias nodes
IFunction ident = new Ident();
// set number of hidden units to (Input + Hidden) * 2/3 as basic best guess.
int hidden = (int)System.Math.Ceiling((double)(x.Cols + output) * 2.0 / 3.0);
return(network.Create(x.Cols, output, activationFunction, outputFunction,
fnNodeInitializer: new Func <int, int, NodeType, Neuron>((l, i, type) =>
{
if (type == NodeType.Input)
{
return new Neuron(false)
{
Label = d.ColumnAt(i - 1), ActivationFunction = activationFunction, NodeId = i, LayerId = l
}
}
;
else if (type == NodeType.Output)
{
return new Neuron(false)
{
Label = Network.GetLabel(i, d), ActivationFunction = activationFunction, NodeId = i, LayerId = l
}
}
;
else
{
return new Neuron(false)
{
ActivationFunction = activationFunction, NodeId = i, LayerId = l
}
};
}), lossFunction: lossFunction, hiddenLayers: hidden));
}
public N(
IEnumerable <int> neuron_length_per_layers,
IEnumerable <IActivationFunction> actfuncs,
IOutputFunction output_layer_function,
ILossFunction loss_function
)
{
create_layers(neuron_length_per_layers, actfuncs);
init_links();
//foreach( var n in this.layers.Take( this.layers.Length - 1 ).SelectMany( l => l.neurons ) )
//{
// n.sign = UnityEngine.Random.value > 0.3f ? 1.0f : -1.0f;
//}
this.loss_func = loss_function;
this.output_layer_func = output_layer_function;
if (output_layer_function is SoftMax)
{
add_softmax_layer();
}
}
public NeuralNetwork(double learnRate, IActivationFunction activationFunction,
IWeightInitilizer weightInitilizer, ILossFunction lossFunction)
{
if (activationFunction == null)
{
throw new Exception("[NeuralNetwork] activationFunction is null.");
}
if (weightInitilizer == null)
{
throw new Exception("[NeuralNetwork] weightInitilizer is null.");
}
if (lossFunction == null)
{
throw new Exception("[NeuralNetwork] lossFunction is null.");
}
_learnRate = learnRate;
_activationFunction = activationFunction;
_weightInitilizer = weightInitilizer;
_lossFunction = lossFunction;
Layers = new List <Layer>();
}
public NeuralNetwork(int[] layers, IActivationFunction[] function, ILossFunction loss, IOptimizer optimizer)
{
this.layers = layers;
this.function = function;
this.lossFunc = loss;
this.Optimizer = optimizer;
this.Weights = new Matrix[layers.Length];
this.Biases = new Vector[layers.Length];
int w = layers[0];
Random rng = new Random(0);
for (int i = 1; i < Weights.Length; i++)
{
int h = layers[i];
Weights[i] = new Matrix(w, h);
Biases[i] = new Vector(h);
for (int j = 0; j < h; j++)
{
Biases[i][j] = (float)rng.NextGaussian();// 1e-10f;
for (int k = 0; k < w; k++)
{
Weights[i][k, j] = (float)rng.NextGaussian(0, System.Math.Pow(w, -0.5f));// * 1e-10f;
}
}
w = h;
}
this.Solver = new GradientSolver(this);
}
public GbmState(IRegressionProblemData problemData, ILossFunction lossFunction, uint randSeed, int maxSize, double r, double m, double nu)
{
// default settings for MaxSize, Nu and R
this.maxSize = maxSize;
this.nu = nu;
this.r = r;
this.m = m;
this.randSeed = randSeed;
random = new MersenneTwister(randSeed);
this.problemData = problemData;
this.trainingRows = problemData.TrainingIndices.ToArray();
this.testRows = problemData.TestIndices.ToArray();
this.lossFunction = lossFunction;
int nRows = trainingRows.Length;
y = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, trainingRows).ToArray();
treeBuilder = new RegressionTreeBuilder(problemData, random);
activeIdx = Enumerable.Range(0, nRows).ToArray();
var zeros = Enumerable.Repeat(0.0, nRows).ToArray();
double f0 = lossFunction.LineSearch(y, zeros, activeIdx, 0, nRows - 1); // initial constant value (mean for squared errors)
pred = Enumerable.Repeat(f0, nRows).ToArray();
predTest = Enumerable.Repeat(f0, testRows.Length).ToArray();
pseudoRes = new double[nRows];
models = new List <IRegressionModel>();
weights = new List <double>();
// add constant model
models.Add(new ConstantModel(f0, problemData.TargetVariable));
weights.Add(1.0);
}
public void SetLossFunction(ILossFunction lf)
{
_lf = lf;
}
// specific interface that allows to specify the target labels and the training rows which is necessary when for gradient boosted trees
public IRegressionModel CreateRegressionTreeForGradientBoosting(double[] y, double[] curPred, int maxSize, int[] idx, ILossFunction lossFunction, double r = 0.5, double m = 0.5) {
Debug.Assert(maxSize > 0);
Debug.Assert(r > 0);
Debug.Assert(r <= 1.0);
Debug.Assert(y.Count() == this.y.Length);
Debug.Assert(m > 0);
Debug.Assert(m <= 1.0);
// y and curPred are changed in gradient boosting
this.y = y;
this.curPred = curPred;
// shuffle row idx
HeuristicLab.Random.ListExtensions.ShuffleInPlace(idx, random);
int nRows = idx.Count();
// shuffle variable idx
HeuristicLab.Random.ListExtensions.ShuffleInPlace(allowedVariables, random);
// only select a part of the rows and columns randomly
effectiveRows = (int)Math.Ceiling(nRows * r);
effectiveVars = (int)Math.Ceiling(nCols * m);
// the which array is used for partitioing row idxs
Array.Clear(which, 0, which.Length);
// mark selected rows
for (int row = 0; row < effectiveRows; row++) {
which[idx[row]] = 1; // we use the which vector as a temporary variable here
internalIdx[row] = idx[row];
}
for (int col = 0; col < nCols; col++) {
int i = 0;
for (int row = 0; row < nRows; row++) {
if (which[sortedIdxAll[col][row]] > 0) {
Debug.Assert(i < effectiveRows);
sortedIdx[col][i] = sortedIdxAll[col][row];
i++;
}
}
}
this.tree = new RegressionTreeModel.TreeNode[maxSize];
this.queue.Clear();
this.curTreeNodeIdx = 0;
// start out with only one leaf node (constant prediction)
// and calculate the best split for this root node and enqueue it into a queue sorted by improvement throught the split
// start and end idx are inclusive
CreateLeafNode(0, effectiveRows - 1, lossFunction);
// process the priority queue to complete the tree
CreateRegressionTreeFromQueue(maxSize, lossFunction);
return new RegressionTreeModel(tree.ToArray());
}
// processes potential splits from the queue as long as splits are left and the maximum size of the tree is not reached
private void CreateRegressionTreeFromQueue(int maxNodes, ILossFunction lossFunction) {
while (queue.Any() && curTreeNodeIdx + 1 < maxNodes) { // two nodes are created in each loop
var f = queue[queue.Count - 1]; // last element has the largest improvement
queue.RemoveAt(queue.Count - 1);
var startIdx = f.StartIdx;
var endIdx = f.EndIndex;
Debug.Assert(endIdx - startIdx >= 0);
Debug.Assert(startIdx >= 0);
Debug.Assert(endIdx < internalIdx.Length);
// split partition into left and right
int splitIdx;
SplitPartition(f.StartIdx, f.EndIndex, f.SplittingVariable, f.SplittingThreshold, out splitIdx);
Debug.Assert(splitIdx + 1 <= endIdx);
Debug.Assert(startIdx <= splitIdx);
// create two leaf nodes (and enqueue best splits for both)
var leftTreeIdx = CreateLeafNode(startIdx, splitIdx, lossFunction);
var rightTreeIdx = CreateLeafNode(splitIdx + 1, endIdx, lossFunction);
// overwrite existing leaf node with an internal node
tree[f.ParentNodeIdx] = new RegressionTreeModel.TreeNode(f.SplittingVariable, f.SplittingThreshold, leftTreeIdx, rightTreeIdx);
}
}
// returns the index of the newly created tree node
private int CreateLeafNode(int startIdx, int endIdx, ILossFunction lossFunction) {
// write a leaf node
var val = lossFunction.LineSearch(originalY, curPred, internalIdx, startIdx, endIdx);
tree[curTreeNodeIdx] = new RegressionTreeModel.TreeNode(RegressionTreeModel.TreeNode.NO_VARIABLE, val);
EnqueuePartitionSplit(curTreeNodeIdx, startIdx, endIdx);
curTreeNodeIdx++;
return curTreeNodeIdx - 1;
}
// for custom stepping & termination
public static IGbmState CreateGbmState(IRegressionProblemData problemData, ILossFunction lossFunction, uint randSeed, int maxSize = 3, double r = 0.66, double m = 0.5, double nu = 0.01)
{
return(new GbmState(problemData, lossFunction, randSeed, maxSize, r, m, nu));
}
// wrap an actual model in a surrograte
public GradientBoostedTreesModelSurrogate(IRegressionProblemData trainingProblemData, uint seed,
ILossFunction lossFunction, int iterations, int maxSize, double r, double m, double nu,
IGradientBoostedTreesModel model)
: this(trainingProblemData, seed, lossFunction, iterations, maxSize, r, m, nu) {
this.actualModel = model;
}
// simple interface
public static GradientBoostedTreesSolution TrainGbm(IRegressionProblemData problemData, ILossFunction lossFunction, int maxSize, double nu, double r, double m, int maxIterations, uint randSeed = 31415)
{
Contract.Assert(r > 0);
Contract.Assert(r <= 1.0);
Contract.Assert(nu > 0);
Contract.Assert(nu <= 1.0);
var state = (GbmState)CreateGbmState(problemData, lossFunction, randSeed, maxSize, r, m, nu);
for (int iter = 0; iter < maxIterations; iter++)
{
MakeStep(state);
}
var model = state.GetModel();
return(new GradientBoostedTreesSolution(model, (IRegressionProblemData)problemData.Clone()));
}
// simple interface
public static GradientBoostedTreesSolution TrainGbm(IRegressionProblemData problemData, ILossFunction lossFunction, int maxSize, double nu, double r, double m, int maxIterations, uint randSeed = 31415) {
Contract.Assert(r > 0);
Contract.Assert(r <= 1.0);
Contract.Assert(nu > 0);
Contract.Assert(nu <= 1.0);
var state = (GbmState)CreateGbmState(problemData, lossFunction, randSeed, maxSize, r, m, nu);
for (int iter = 0; iter < maxIterations; iter++) {
MakeStep(state);
}
var model = state.GetModel();
return new GradientBoostedTreesSolution(model, (IRegressionProblemData)problemData.Clone());
}
// for custom stepping & termination
public static IGbmState CreateGbmState(IRegressionProblemData problemData, ILossFunction lossFunction, uint randSeed, int maxSize = 3, double r = 0.66, double m = 0.5, double nu = 0.01) {
return new GbmState(problemData, lossFunction, randSeed, maxSize, r, m, nu);
}
public double GetLoss(IEnumerable <ValueTuple <double[], double[]> > collection, ILossFunction lossFunction)
{
double sum = 0.0;
int size = collection.Count();
int outputs = this.layerCollection[this.layerCollection.Count - 1].Outputs;
foreach (var loss in from tuple in collection from loss in Forward(new Batch <double[]>(new double[][] { tuple.Item1 }), new Batch <double[]>(new double[][] { tuple.Item2 }), false, lossFunction).Item2[0] select loss)
{
sum += loss;
}
return(sum / size);
}
private Tuple <Batch <double[]>, Batch <double[]> > Forward(Batch <double[]> x, Batch <double[]> t, bool isTraining, ILossFunction lossFunction)
{
var layer = this.layerCollection[0];
var weightDecay = 0.0;
var vectorList1 = new List <double[]>();
var vectorList2 = new List <double[]>();
do
{
var updatable = layer as IUpdatable;
x = layer.Forward(x, isTraining);
if (updatable != null)
{
var sum = 0.0;
foreach (double weight in updatable.Weights)
{
sum += weight * weight;
}
weightDecay += 0.5 * this.weightDecayRate * sum;
}
layer = layer.Next;
} while (layer != null);
for (int i = 0; i < x.Size; i++)
{
var y = lossFunction.Forward(x[i], t[i]);
for (int j = 0; j < y.Item2.Length; j++)
{
y.Item2[j] += weightDecay;
}
vectorList1.Add(y.Item1);
vectorList2.Add(y.Item2);
}
return(Tuple.Create <Batch <double[]>, Batch <double[]> >(new Batch <double[]>(vectorList1), new Batch <double[]>(vectorList2)));
}
private IEnumerable <IUpdatable> Backward(Batch <double[]> y, Batch <double[]> t, ILossFunction lossFunction)
{
var layer = this.layerCollection[this.layerCollection.Count - 1];
var deltas = new Batch <double[]>(new double[t.Size][]);
var updatableList = new LinkedList <IUpdatable>();
for (int i = 0; i < t.Size; i++)
{
deltas[i] = lossFunction.Backward(y[i], t[i]);
}
do
{
var updatable = layer as IUpdatable;
deltas = layer.Backward(deltas);
if (updatable != null)
{
updatableList.AddFirst(updatable);
}
layer = layer.Previous;
} while (layer != null);
return(updatableList);
}
public void Fit(IEnumerable <ValueTuple <double[], double[]> > collection, int epochs, int batchSize, Func <IEnumerable <ValueTuple <double[], double[]> >, int, IEnumerable <ValueTuple <double[], double[]> > > func, IOptimizer optimizer, ILossFunction lossFunction)
{
// Backpropagation
int dataSize = collection.Count();
int t = 0;
// Stochastic gradient descent (SGD)
while (t < epochs)
{
// Mini-batch
int remaining = dataSize;
do
{
var dataTuple = func(collection, Math.Min(remaining, batchSize)).Aggregate <ValueTuple <double[], double[]>, Tuple <List <double[]>, List <double[]> > >(Tuple.Create <List <double[]>, List <double[]> >(new List <double[]>(), new List <double[]>()), (tuple1, tuple2) =>
{
tuple1.Item1.Add(tuple2.Item1);
tuple1.Item2.Add(tuple2.Item2);
return(tuple1);
});
int index = 0;
int identifier = 0;
var targets = new Batch <double[]>(dataTuple.Item2);
var layers = Backward(Forward(new Batch <double[]>(dataTuple.Item1), targets, true, lossFunction).Item1, targets, lossFunction);
// Weight decay
foreach (var layer in layers)
{
layer.SetGradients((x, y, z) => x ? y + this.weightDecayRate * layer.Weights[z] : y);
}
if (this.maxGradient.HasValue)
{
// Gradient clipping
var vectors = from tuple in layers let batch = tuple.GetGradients() from vector in batch select vector;
double sum = 0.0;
foreach (var gradient in from vector in vectors from gradient in vector select gradient)
{
sum += gradient * gradient;
}
double rate = this.maxGradient.Value / (Math.Sqrt(sum) + Math.Pow(10, -6));
if (rate < 1)
{
foreach (var vector in vectors)
{
for (int i = 0; i < vector.Length; i++)
{
vector[i] *= rate;
}
}
}
}
foreach (var layer in layers)
{
layer.Update(layer.GetGradients(), (weight, gradient) => optimizer.Optimize(identifier++, weight, gradient));
index++;
}
remaining -= batchSize;
} while (remaining > 0);
if (this.Stepped != null)
{
this.Stepped(this, new EventArgs());
}
t++;
}
}
public void Fit(IEnumerable <ValueTuple <double[], double[]> > collection, int epochs, int batchSize, IOptimizer optimizer, ILossFunction lossFunction)
{
Fit(collection, epochs, batchSize, (x, y) => x.Sample <ValueTuple <double[], double[]> >(this.random, y), optimizer, lossFunction);
}
