public FastNN(int[] numNeurons, int miniBatchSize, bool l2loss, float dropoutProb = 0) { this.l2loss = l2loss; layers = new DenseLayer[numNeurons.Length - 1]; dropouts = new DropoutLayer[numNeurons.Length - 1]; activations = new Matrix <float> [numNeurons.Length]; singleActivations = new Matrix <float> [numNeurons.Length]; preActivations = new Matrix <float> [numNeurons.Length - 1]; deltas = new Matrix <float> [numNeurons.Length - 1]; this.miniBatchSize = miniBatchSize; optimizer = new Adam(0.001F); IActivationFunc activationFunc = new Relu(); IInitialization initialization = new HeNormal(); for (int i = 0; i < numNeurons.Length; i++) { activations[i] = DenseMatrix.Create(miniBatchSize, numNeurons[i], 0); singleActivations[i] = DenseMatrix.Create(1, numNeurons[i], 0); if (i == 0) { continue; } if (i == numNeurons.Length - 1) { activationFunc = new Linear(); } preActivations[i - 1] = DenseMatrix.Create(miniBatchSize, numNeurons[i], 0); layers[i - 1] = new DenseLayer(numNeurons[i - 1], numNeurons[i], activationFunc, initialization); deltas[i - 1] = DenseMatrix.Create(miniBatchSize, numNeurons[i], 0); if (dropoutProb > 0 && i < numNeurons.Length - 1) { dropouts[i - 1] = new DropoutLayer(miniBatchSize, numNeurons[i], dropoutProb); } } computeSDOutput = false; if (numNeurons.Last() == 2) { computeSDOutput = true; } }
public DenseLayer CopyOnlyParams() { DenseLayer layer = new DenseLayer(); layer.weights = new Weights(this.weights); layer.bias = new Bias(this.bias); layer.activationFunc = this.activationFunc; return(layer); }
private void Backward(Matrix <float> X, Vector <float> y) { DenseLayer layer = layers.Last(); Matrix <float> delta = deltas.Last(); for (int i = layers.Length - 1; i >= 0; i--) { if (i == layers.Length - 1) { SetOutputDelta(X, y); continue; } DenseLayer prevLayer = layers[i]; Matrix <float> W = layer.Weights.Vals; Matrix <float> preActivation = preActivations[i]; DropoutLayer dropout = dropouts[i]; IActivationFunc activationFunc = prevLayer.ActivationFunc; Matrix <float> deltaNew = deltas[i]; delta.TransposeAndMultiply(W, deltaNew); preActivation.MapInplace(activationFunc.dF, Zeros.Include); deltaNew.PointwiseMultiply(preActivation, deltaNew); delta = deltaNew; layer = prevLayer; } for (int i = layers.Length - 1; i >= 0; i--) { layer = layers[i]; delta = deltas[i]; Matrix <float> gradWeights = layer.GradWeights.Vals; Matrix <float> gradBias = layer.GradBias.Vals; Matrix <float> a = activations[i]; a.TransposeThisAndMultiply(delta, gradWeights); gradWeights.Divide(miniBatchSize, gradWeights); Vector <float> gradBiasVect = delta.ColumnSums(); gradBiasVect.Divide(miniBatchSize, gradBiasVect); gradBias.SetRow(0, gradBiasVect); } }
public float[][] Forward(Matrix <float> X, bool pred) { Matrix <float>[] activations; if (pred == false) { activations = this.activations; } else { activations = this.singleActivations; } activations[0] = X; Matrix <float> a = activations[0]; for (int i = 0; i < layers.Length; i++) { DenseLayer layer = layers[i]; Matrix <float> W = layer.Weights.Vals; Matrix <float> b = layer.Bias.Vals; IActivationFunc activationFunc = layer.ActivationFunc; DropoutLayer dropout = null; if (dropouts != null) { dropout = dropouts[i]; } if (pred == false) { b = layer.Bias.Broadcast(miniBatchSize); } Matrix <float> aNext = activations[i + 1]; a.Multiply(W, aNext); aNext.Add(b, aNext); if (pred == false) { aNext.CopyTo(preActivations[i]); } if (activationFunc.GetType() != typeof(Linear)) { aNext.MapInplace(activationFunc.F, Zeros.Include); } if (pred == false && dropout != null) { dropout.Sample(); aNext.PointwiseMultiply(dropout.Vals, aNext); } a = aNext; } return(a.ToColumnArrays()); }