// wrap an actual model in a surrograte public GradientBoostedTreesModelSurrogate(IRegressionProblemData trainingProblemData, uint seed, ILossFunction lossFunction, int iterations, int maxSize, double r, double m, double nu, IGradientBoostedTreesModel model) : this(trainingProblemData, seed, lossFunction, iterations, maxSize, r, m, nu) { this.actualModel = model; }
public GbmState(IRegressionProblemData problemData, ILossFunction lossFunction, uint randSeed, int maxSize, double r, double m, double nu) { // default settings for MaxSize, Nu and R this.maxSize = maxSize; this.nu = nu; this.r = r; this.m = m; this.randSeed = randSeed; random = new MersenneTwister(randSeed); this.problemData = problemData; this.trainingRows = problemData.TrainingIndices.ToArray(); this.testRows = problemData.TestIndices.ToArray(); this.lossFunction = lossFunction; int nRows = trainingRows.Length; y = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, trainingRows).ToArray(); treeBuilder = new RegressionTreeBuilder(problemData, random); activeIdx = Enumerable.Range(0, nRows).ToArray(); var zeros = Enumerable.Repeat(0.0, nRows).ToArray(); double f0 = lossFunction.LineSearch(y, zeros, activeIdx, 0, nRows - 1); // initial constant value (mean for squared errors) pred = Enumerable.Repeat(f0, nRows).ToArray(); predTest = Enumerable.Repeat(f0, testRows.Length).ToArray(); pseudoRes = new double[nRows]; models = new List<IRegressionModel>(); weights = new List<double>(); // add constant model models.Add(new ConstantModel(f0, problemData.TargetVariable)); weights.Add(1.0); }
public LinearModel(ILossFunction loss, float learningRate, float L2, float L1) { _loss = loss; _learningRate = learningRate; _L2 = L2; _L1 = L1; }
public DenseLayerNoBias(int nbInput, int nbOutput, IActivation activation, ILossFunction lossFunction) : base(nbInput, nbOutput, activation, lossFunction) { errors = new double[nbOutput]; derivatives = new double[nbOutput]; weightsError = new double[nbInput, nbOutput]; }
public Network(List <Layer> layers, ILossFunction lossFunction) { Seed = 123; // TODO: temp Layers = layers; LossFunction = lossFunction; InitializeWeightMatricies(); }
/// <summary> /// Connect networks with 0th entry as input /// </summary> /// <param name="nets"></param> public NeuralNetwork(params NeuralNetwork[] nets) { int layerCount = 1; for (int i = 0; i < nets.Length; i++) { layerCount += (nets[i].LayerCount - 1); } this.layers = new int[layerCount]; this.function = new IActivationFunction[layerCount]; this.lossFunc = nets[0].LossFunction; this.Optimizer = nets[0].Optimizer; this.Weights = new Matrix[layerCount]; this.Biases = new Vector[layerCount]; int idx = 1; for (int i = 0; i < nets.Length; i++) { for (int j = 1; j < nets[i].LayerCount; j++) { Weights[idx] = nets[i].Weights[j]; Biases[idx] = nets[i].Biases[j]; layers[idx] = nets[i].Layers[j]; function[idx] = nets[i].ActivationFunctions[j]; idx++; } } this.Solver = new GradientSolver(this); }
// processes potential splits from the queue as long as splits are left and the maximum size of the tree is not reached private void CreateRegressionTreeFromQueue(int maxNodes, ILossFunction lossFunction) { while (queue.Any() && curTreeNodeIdx + 1 < maxNodes) // two nodes are created in each loop { var f = queue[queue.Count - 1]; // last element has the largest improvement queue.RemoveAt(queue.Count - 1); var startIdx = f.StartIdx; var endIdx = f.EndIndex; Debug.Assert(endIdx - startIdx >= 0); Debug.Assert(startIdx >= 0); Debug.Assert(endIdx < internalIdx.Length); // split partition into left and right int splitIdx; SplitPartition(f.StartIdx, f.EndIndex, f.SplittingVariable, f.SplittingThreshold, out splitIdx); Debug.Assert(splitIdx + 1 <= endIdx); Debug.Assert(startIdx <= splitIdx); // create two leaf nodes (and enqueue best splits for both) var leftTreeIdx = CreateLeafNode(startIdx, splitIdx, lossFunction); var rightTreeIdx = CreateLeafNode(splitIdx + 1, endIdx, lossFunction); // overwrite existing leaf node with an internal node tree[f.ParentNodeIdx] = new RegressionTreeModel.TreeNode(f.SplittingVariable, f.SplittingThreshold, leftTreeIdx, rightTreeIdx); } }
// returns the index of the newly created tree node private int CreateLeafNode(int startIdx, int endIdx, ILossFunction lossFunction) { // write a leaf node var val = lossFunction.LineSearch(originalY, curPred, internalIdx, startIdx, endIdx); tree[curTreeNodeIdx] = new RegressionTreeModel.TreeNode(RegressionTreeModel.TreeNode.NO_VARIABLE, val); EnqueuePartitionSplit(curTreeNodeIdx, startIdx, endIdx); curTreeNodeIdx++; return(curTreeNodeIdx - 1); }
// create only the surrogate model without an actual model public GradientBoostedTreesModelSurrogate(IRegressionProblemData trainingProblemData, uint seed, ILossFunction lossFunction, int iterations, int maxSize, double r, double m, double nu) : base("Gradient boosted tree model", string.Empty) { this.trainingProblemData = trainingProblemData; this.seed = seed; this.lossFunction = lossFunction; this.iterations = iterations; this.maxSize = maxSize; this.r = r; this.m = m; this.nu = nu; }
public LayerBase(int nbInput, int nbOutput, IActivation activation, ILossFunction lossFunction) { this.InputError = new double[nbInput]; this.NbOutput = nbOutput; this.NbInput = nbInput; this.Output = new double[nbOutput]; this.Weights = new NNMatrix(nbInput, nbOutput); this.NonActivatedOutput = new double[nbOutput]; this.Activation = activation; this.LossFunction = lossFunction; }
protected override void DoLossDerivative(Tensor correct, ILossFunction lossFunction, Tensor dy) { var tStorage = correct.Storage as GpuStorage ?? throw new UnsupportedStorageException(nameof(correct)); var dyStorage = dy.Storage as GpuStorage ?? throw new UnsupportedStorageException(nameof(dy)); var dO = _storage.DeviceStorage; var dT = tStorage.DeviceStorage; var dDy = dyStorage.DeviceStorage; _context.Methods.LossDerivative(dO, dT, dDy, lossFunction, dy.Storage.Descriptor); }
protected override void DoLoss(Tensor correct, ILossFunction lossFunction, Tensor loss) { var tStorage = correct.Storage as GpuStorage ?? throw new UnsupportedStorageException(nameof(correct)); var lossStorage = loss.Storage as GpuStorage ?? throw new UnsupportedStorageException(nameof(loss)); var dO = _storage.DeviceStorage; var dT = tStorage.DeviceStorage; var dLoss = lossStorage.DeviceStorage; _context.Methods.Loss(dO, dT, dLoss, lossFunction, _storage.Descriptor); }
// create only the surrogate model without an actual model public GradientBoostedTreesModelSurrogate(IRegressionProblemData trainingProblemData, uint seed, ILossFunction lossFunction, int iterations, int maxSize, double r, double m, double nu) : base("Gradient boosted tree model", string.Empty) { this.trainingProblemData = trainingProblemData; this.seed = seed; this.lossFunction = lossFunction; this.iterations = iterations; this.maxSize = maxSize; this.r = r; this.m = m; this.nu = nu; }
// for custom stepping & termination public static IGbmState CreateGbmState(IRegressionProblemData problemData, ILossFunction lossFunction, uint randSeed, int maxSize = 3, double r = 0.66, double m = 0.5, double nu = 0.01) { // check input variables. Only double variables are allowed. var invalidInputs = problemData.AllowedInputVariables.Where(name => !problemData.Dataset.VariableHasType <double>(name)); if (invalidInputs.Any()) { throw new NotSupportedException("Gradient tree boosting only supports real-valued variables. Unsupported inputs: " + string.Join(", ", invalidInputs)); } return(new GbmState(problemData, lossFunction, randSeed, maxSize, r, m, nu)); }
private GradientBoostedTreesModelSurrogate(GradientBoostedTreesModelSurrogate original, Cloner cloner) : base(original, cloner) { if (original.actualModel != null) this.actualModel = cloner.Clone(original.actualModel); this.trainingProblemData = cloner.Clone(original.trainingProblemData); this.lossFunction = cloner.Clone(original.lossFunction); this.seed = original.seed; this.iterations = original.iterations; this.maxSize = original.maxSize; this.r = original.r; this.m = original.m; this.nu = original.nu; }
public Network(Layer layer, IOptimizer optimizer, ILossFunction lossFunction) { this.random = RandomProvider.GetRandom(); this.inputLayer = layer; this.layerCollection = new Collection <Layer>(); this.optimizer = optimizer; this.lossFunction = lossFunction; do { this.layerCollection.Add(layer); layer = layer.Next; } while (layer != null); }
// create only the surrogate model without an actual model private GradientBoostedTreesModelSurrogate(IRegressionProblemData trainingProblemData, uint seed, ILossFunction lossFunction, int iterations, int maxSize, double r, double m, double nu) : base(trainingProblemData.TargetVariable, "Gradient boosted tree model", string.Empty) { this.trainingProblemData = trainingProblemData; this.seed = seed; this.lossFunction = lossFunction; this.iterations = iterations; this.maxSize = maxSize; this.r = r; this.m = m; this.nu = nu; actualModel = new Lazy <IGradientBoostedTreesModel>(() => RecalculateModel()); }
private GradientBoostedTreesModelSurrogate(GradientBoostedTreesModelSurrogate original, Cloner cloner) : base(original, cloner) { if (original.actualModel != null) { this.actualModel = cloner.Clone(original.actualModel); } this.trainingProblemData = cloner.Clone(original.trainingProblemData); this.lossFunction = cloner.Clone(original.lossFunction); this.seed = original.seed; this.iterations = original.iterations; this.maxSize = original.maxSize; this.r = original.r; this.m = original.m; this.nu = original.nu; }
public Model(IEnumerable <Layer> collection, IOptimizer optimizer, ILossFunction lossFunction) { this.random = RandomProvider.GetRandom(); this.layerCollection = new Collection <Layer>(); this.optimizer = optimizer; this.lossFunction = lossFunction; foreach (Layer layer in collection) { if (this.layerCollection.Count > 0) { var previousLayer = this.layerCollection[this.layerCollection.Count - 1]; previousLayer.Next = layer; layer.Previous = previousLayer; } this.layerCollection.Add(layer); } }
private GradientBoostedTreesModelSurrogate(GradientBoostedTreesModelSurrogate original, Cloner cloner) : base(original, cloner) { IGradientBoostedTreesModel clonedModel = null; if (original.actualModel.IsValueCreated) { clonedModel = cloner.Clone(original.ActualModel); } actualModel = new Lazy <IGradientBoostedTreesModel>(CreateLazyInitFunc(clonedModel)); // only capture clonedModel in the closure this.trainingProblemData = cloner.Clone(original.trainingProblemData); this.lossFunction = cloner.Clone(original.lossFunction); this.seed = original.seed; this.iterations = original.iterations; this.maxSize = original.maxSize; this.r = original.r; this.m = original.m; this.nu = original.nu; }
/// <summary>Defaults.</summary> /// <param name="d">The Descriptor to process.</param> /// <param name="x">The Vector to process.</param> /// <param name="y">The Vector to process.</param> /// <param name="activationFunction">The activation.</param> /// <param name="outputFunction">The ouput function for hidden nodes (Optional).</param> /// <param name="epsilon">epsilon</param> /// <returns>A Network.</returns> public static Network Create(this Network network, Descriptor d, Matrix x, Vector y, IFunction activationFunction, IFunction outputFunction = null, double epsilon = double.NaN, ILossFunction lossFunction = null) { // set output to number of choices of available // 1 if only two choices int distinct = y.Distinct().Count(); int output = distinct > 2 ? distinct : 1; // identity funciton for bias nodes IFunction ident = new Ident(); // set number of hidden units to (Input + Hidden) * 2/3 as basic best guess. int hidden = (int)System.Math.Ceiling((double)(x.Cols + output) * 2.0 / 3.0); return(network.Create(x.Cols, output, activationFunction, outputFunction, fnNodeInitializer: new Func <int, int, NodeType, Neuron>((l, i, type) => { if (type == NodeType.Input) { return new Neuron(false) { Label = d.ColumnAt(i - 1), ActivationFunction = activationFunction, NodeId = i, LayerId = l } } ; else if (type == NodeType.Output) { return new Neuron(false) { Label = Network.GetLabel(i, d), ActivationFunction = activationFunction, NodeId = i, LayerId = l } } ; else { return new Neuron(false) { ActivationFunction = activationFunction, NodeId = i, LayerId = l } }; }), lossFunction: lossFunction, hiddenLayers: hidden)); }
public N( IEnumerable <int> neuron_length_per_layers, IEnumerable <IActivationFunction> actfuncs, IOutputFunction output_layer_function, ILossFunction loss_function ) { create_layers(neuron_length_per_layers, actfuncs); init_links(); //foreach( var n in this.layers.Take( this.layers.Length - 1 ).SelectMany( l => l.neurons ) ) //{ // n.sign = UnityEngine.Random.value > 0.3f ? 1.0f : -1.0f; //} this.loss_func = loss_function; this.output_layer_func = output_layer_function; if (output_layer_function is SoftMax) { add_softmax_layer(); } }
public NeuralNetwork(double learnRate, IActivationFunction activationFunction, IWeightInitilizer weightInitilizer, ILossFunction lossFunction) { if (activationFunction == null) { throw new Exception("[NeuralNetwork] activationFunction is null."); } if (weightInitilizer == null) { throw new Exception("[NeuralNetwork] weightInitilizer is null."); } if (lossFunction == null) { throw new Exception("[NeuralNetwork] lossFunction is null."); } _learnRate = learnRate; _activationFunction = activationFunction; _weightInitilizer = weightInitilizer; _lossFunction = lossFunction; Layers = new List <Layer>(); }
public NeuralNetwork(int[] layers, IActivationFunction[] function, ILossFunction loss, IOptimizer optimizer) { this.layers = layers; this.function = function; this.lossFunc = loss; this.Optimizer = optimizer; this.Weights = new Matrix[layers.Length]; this.Biases = new Vector[layers.Length]; int w = layers[0]; Random rng = new Random(0); for (int i = 1; i < Weights.Length; i++) { int h = layers[i]; Weights[i] = new Matrix(w, h); Biases[i] = new Vector(h); for (int j = 0; j < h; j++) { Biases[i][j] = (float)rng.NextGaussian();// 1e-10f; for (int k = 0; k < w; k++) { Weights[i][k, j] = (float)rng.NextGaussian(0, System.Math.Pow(w, -0.5f));// * 1e-10f; } } w = h; } this.Solver = new GradientSolver(this); }
public GbmState(IRegressionProblemData problemData, ILossFunction lossFunction, uint randSeed, int maxSize, double r, double m, double nu) { // default settings for MaxSize, Nu and R this.maxSize = maxSize; this.nu = nu; this.r = r; this.m = m; this.randSeed = randSeed; random = new MersenneTwister(randSeed); this.problemData = problemData; this.trainingRows = problemData.TrainingIndices.ToArray(); this.testRows = problemData.TestIndices.ToArray(); this.lossFunction = lossFunction; int nRows = trainingRows.Length; y = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, trainingRows).ToArray(); treeBuilder = new RegressionTreeBuilder(problemData, random); activeIdx = Enumerable.Range(0, nRows).ToArray(); var zeros = Enumerable.Repeat(0.0, nRows).ToArray(); double f0 = lossFunction.LineSearch(y, zeros, activeIdx, 0, nRows - 1); // initial constant value (mean for squared errors) pred = Enumerable.Repeat(f0, nRows).ToArray(); predTest = Enumerable.Repeat(f0, testRows.Length).ToArray(); pseudoRes = new double[nRows]; models = new List <IRegressionModel>(); weights = new List <double>(); // add constant model models.Add(new ConstantModel(f0, problemData.TargetVariable)); weights.Add(1.0); }
public void SetLossFunction(ILossFunction lf) { _lf = lf; }
// specific interface that allows to specify the target labels and the training rows which is necessary when for gradient boosted trees public IRegressionModel CreateRegressionTreeForGradientBoosting(double[] y, double[] curPred, int maxSize, int[] idx, ILossFunction lossFunction, double r = 0.5, double m = 0.5) { Debug.Assert(maxSize > 0); Debug.Assert(r > 0); Debug.Assert(r <= 1.0); Debug.Assert(y.Count() == this.y.Length); Debug.Assert(m > 0); Debug.Assert(m <= 1.0); // y and curPred are changed in gradient boosting this.y = y; this.curPred = curPred; // shuffle row idx HeuristicLab.Random.ListExtensions.ShuffleInPlace(idx, random); int nRows = idx.Count(); // shuffle variable idx HeuristicLab.Random.ListExtensions.ShuffleInPlace(allowedVariables, random); // only select a part of the rows and columns randomly effectiveRows = (int)Math.Ceiling(nRows * r); effectiveVars = (int)Math.Ceiling(nCols * m); // the which array is used for partitioing row idxs Array.Clear(which, 0, which.Length); // mark selected rows for (int row = 0; row < effectiveRows; row++) { which[idx[row]] = 1; // we use the which vector as a temporary variable here internalIdx[row] = idx[row]; } for (int col = 0; col < nCols; col++) { int i = 0; for (int row = 0; row < nRows; row++) { if (which[sortedIdxAll[col][row]] > 0) { Debug.Assert(i < effectiveRows); sortedIdx[col][i] = sortedIdxAll[col][row]; i++; } } } this.tree = new RegressionTreeModel.TreeNode[maxSize]; this.queue.Clear(); this.curTreeNodeIdx = 0; // start out with only one leaf node (constant prediction) // and calculate the best split for this root node and enqueue it into a queue sorted by improvement throught the split // start and end idx are inclusive CreateLeafNode(0, effectiveRows - 1, lossFunction); // process the priority queue to complete the tree CreateRegressionTreeFromQueue(maxSize, lossFunction); return new RegressionTreeModel(tree.ToArray()); }
// processes potential splits from the queue as long as splits are left and the maximum size of the tree is not reached private void CreateRegressionTreeFromQueue(int maxNodes, ILossFunction lossFunction) { while (queue.Any() && curTreeNodeIdx + 1 < maxNodes) { // two nodes are created in each loop var f = queue[queue.Count - 1]; // last element has the largest improvement queue.RemoveAt(queue.Count - 1); var startIdx = f.StartIdx; var endIdx = f.EndIndex; Debug.Assert(endIdx - startIdx >= 0); Debug.Assert(startIdx >= 0); Debug.Assert(endIdx < internalIdx.Length); // split partition into left and right int splitIdx; SplitPartition(f.StartIdx, f.EndIndex, f.SplittingVariable, f.SplittingThreshold, out splitIdx); Debug.Assert(splitIdx + 1 <= endIdx); Debug.Assert(startIdx <= splitIdx); // create two leaf nodes (and enqueue best splits for both) var leftTreeIdx = CreateLeafNode(startIdx, splitIdx, lossFunction); var rightTreeIdx = CreateLeafNode(splitIdx + 1, endIdx, lossFunction); // overwrite existing leaf node with an internal node tree[f.ParentNodeIdx] = new RegressionTreeModel.TreeNode(f.SplittingVariable, f.SplittingThreshold, leftTreeIdx, rightTreeIdx); } }
// returns the index of the newly created tree node private int CreateLeafNode(int startIdx, int endIdx, ILossFunction lossFunction) { // write a leaf node var val = lossFunction.LineSearch(originalY, curPred, internalIdx, startIdx, endIdx); tree[curTreeNodeIdx] = new RegressionTreeModel.TreeNode(RegressionTreeModel.TreeNode.NO_VARIABLE, val); EnqueuePartitionSplit(curTreeNodeIdx, startIdx, endIdx); curTreeNodeIdx++; return curTreeNodeIdx - 1; }
// for custom stepping & termination public static IGbmState CreateGbmState(IRegressionProblemData problemData, ILossFunction lossFunction, uint randSeed, int maxSize = 3, double r = 0.66, double m = 0.5, double nu = 0.01) { return(new GbmState(problemData, lossFunction, randSeed, maxSize, r, m, nu)); }
// wrap an actual model in a surrograte public GradientBoostedTreesModelSurrogate(IRegressionProblemData trainingProblemData, uint seed, ILossFunction lossFunction, int iterations, int maxSize, double r, double m, double nu, IGradientBoostedTreesModel model) : this(trainingProblemData, seed, lossFunction, iterations, maxSize, r, m, nu) { this.actualModel = model; }
// simple interface public static GradientBoostedTreesSolution TrainGbm(IRegressionProblemData problemData, ILossFunction lossFunction, int maxSize, double nu, double r, double m, int maxIterations, uint randSeed = 31415) { Contract.Assert(r > 0); Contract.Assert(r <= 1.0); Contract.Assert(nu > 0); Contract.Assert(nu <= 1.0); var state = (GbmState)CreateGbmState(problemData, lossFunction, randSeed, maxSize, r, m, nu); for (int iter = 0; iter < maxIterations; iter++) { MakeStep(state); } var model = state.GetModel(); return(new GradientBoostedTreesSolution(model, (IRegressionProblemData)problemData.Clone())); }
// simple interface public static GradientBoostedTreesSolution TrainGbm(IRegressionProblemData problemData, ILossFunction lossFunction, int maxSize, double nu, double r, double m, int maxIterations, uint randSeed = 31415) { Contract.Assert(r > 0); Contract.Assert(r <= 1.0); Contract.Assert(nu > 0); Contract.Assert(nu <= 1.0); var state = (GbmState)CreateGbmState(problemData, lossFunction, randSeed, maxSize, r, m, nu); for (int iter = 0; iter < maxIterations; iter++) { MakeStep(state); } var model = state.GetModel(); return new GradientBoostedTreesSolution(model, (IRegressionProblemData)problemData.Clone()); }
// for custom stepping & termination public static IGbmState CreateGbmState(IRegressionProblemData problemData, ILossFunction lossFunction, uint randSeed, int maxSize = 3, double r = 0.66, double m = 0.5, double nu = 0.01) { return new GbmState(problemData, lossFunction, randSeed, maxSize, r, m, nu); }
public double GetLoss(IEnumerable <ValueTuple <double[], double[]> > collection, ILossFunction lossFunction) { double sum = 0.0; int size = collection.Count(); int outputs = this.layerCollection[this.layerCollection.Count - 1].Outputs; foreach (var loss in from tuple in collection from loss in Forward(new Batch <double[]>(new double[][] { tuple.Item1 }), new Batch <double[]>(new double[][] { tuple.Item2 }), false, lossFunction).Item2[0] select loss) { sum += loss; } return(sum / size); }
private Tuple <Batch <double[]>, Batch <double[]> > Forward(Batch <double[]> x, Batch <double[]> t, bool isTraining, ILossFunction lossFunction) { var layer = this.layerCollection[0]; var weightDecay = 0.0; var vectorList1 = new List <double[]>(); var vectorList2 = new List <double[]>(); do { var updatable = layer as IUpdatable; x = layer.Forward(x, isTraining); if (updatable != null) { var sum = 0.0; foreach (double weight in updatable.Weights) { sum += weight * weight; } weightDecay += 0.5 * this.weightDecayRate * sum; } layer = layer.Next; } while (layer != null); for (int i = 0; i < x.Size; i++) { var y = lossFunction.Forward(x[i], t[i]); for (int j = 0; j < y.Item2.Length; j++) { y.Item2[j] += weightDecay; } vectorList1.Add(y.Item1); vectorList2.Add(y.Item2); } return(Tuple.Create <Batch <double[]>, Batch <double[]> >(new Batch <double[]>(vectorList1), new Batch <double[]>(vectorList2))); }
private IEnumerable <IUpdatable> Backward(Batch <double[]> y, Batch <double[]> t, ILossFunction lossFunction) { var layer = this.layerCollection[this.layerCollection.Count - 1]; var deltas = new Batch <double[]>(new double[t.Size][]); var updatableList = new LinkedList <IUpdatable>(); for (int i = 0; i < t.Size; i++) { deltas[i] = lossFunction.Backward(y[i], t[i]); } do { var updatable = layer as IUpdatable; deltas = layer.Backward(deltas); if (updatable != null) { updatableList.AddFirst(updatable); } layer = layer.Previous; } while (layer != null); return(updatableList); }
public void Fit(IEnumerable <ValueTuple <double[], double[]> > collection, int epochs, int batchSize, Func <IEnumerable <ValueTuple <double[], double[]> >, int, IEnumerable <ValueTuple <double[], double[]> > > func, IOptimizer optimizer, ILossFunction lossFunction) { // Backpropagation int dataSize = collection.Count(); int t = 0; // Stochastic gradient descent (SGD) while (t < epochs) { // Mini-batch int remaining = dataSize; do { var dataTuple = func(collection, Math.Min(remaining, batchSize)).Aggregate <ValueTuple <double[], double[]>, Tuple <List <double[]>, List <double[]> > >(Tuple.Create <List <double[]>, List <double[]> >(new List <double[]>(), new List <double[]>()), (tuple1, tuple2) => { tuple1.Item1.Add(tuple2.Item1); tuple1.Item2.Add(tuple2.Item2); return(tuple1); }); int index = 0; int identifier = 0; var targets = new Batch <double[]>(dataTuple.Item2); var layers = Backward(Forward(new Batch <double[]>(dataTuple.Item1), targets, true, lossFunction).Item1, targets, lossFunction); // Weight decay foreach (var layer in layers) { layer.SetGradients((x, y, z) => x ? y + this.weightDecayRate * layer.Weights[z] : y); } if (this.maxGradient.HasValue) { // Gradient clipping var vectors = from tuple in layers let batch = tuple.GetGradients() from vector in batch select vector; double sum = 0.0; foreach (var gradient in from vector in vectors from gradient in vector select gradient) { sum += gradient * gradient; } double rate = this.maxGradient.Value / (Math.Sqrt(sum) + Math.Pow(10, -6)); if (rate < 1) { foreach (var vector in vectors) { for (int i = 0; i < vector.Length; i++) { vector[i] *= rate; } } } } foreach (var layer in layers) { layer.Update(layer.GetGradients(), (weight, gradient) => optimizer.Optimize(identifier++, weight, gradient)); index++; } remaining -= batchSize; } while (remaining > 0); if (this.Stepped != null) { this.Stepped(this, new EventArgs()); } t++; } }
public void Fit(IEnumerable <ValueTuple <double[], double[]> > collection, int epochs, int batchSize, IOptimizer optimizer, ILossFunction lossFunction) { Fit(collection, epochs, batchSize, (x, y) => x.Sample <ValueTuple <double[], double[]> >(this.random, y), optimizer, lossFunction); }