public FastNN(int[] numNeurons, int miniBatchSize, bool l2loss, float dropoutProb = 0)
{
this.l2loss = l2loss;
layers = new DenseLayer[numNeurons.Length - 1];
dropouts = new DropoutLayer[numNeurons.Length - 1];
activations = new Matrix <float> [numNeurons.Length];
singleActivations = new Matrix <float> [numNeurons.Length];
preActivations = new Matrix <float> [numNeurons.Length - 1];
deltas = new Matrix <float> [numNeurons.Length - 1];
this.miniBatchSize = miniBatchSize;
optimizer = new Adam(0.001F);
IActivationFunc activationFunc = new Relu();
IInitialization initialization = new HeNormal();
for (int i = 0; i < numNeurons.Length; i++)
{
activations[i] = DenseMatrix.Create(miniBatchSize, numNeurons[i], 0);
singleActivations[i] = DenseMatrix.Create(1, numNeurons[i], 0);
if (i == 0)
{
continue;
}
if (i == numNeurons.Length - 1)
{
activationFunc = new Linear();
}
preActivations[i - 1] = DenseMatrix.Create(miniBatchSize, numNeurons[i], 0);
layers[i - 1] = new DenseLayer(numNeurons[i - 1], numNeurons[i], activationFunc, initialization);
deltas[i - 1] = DenseMatrix.Create(miniBatchSize, numNeurons[i], 0);
if (dropoutProb > 0 && i < numNeurons.Length - 1)
{
dropouts[i - 1] = new DropoutLayer(miniBatchSize, numNeurons[i], dropoutProb);
}
}
computeSDOutput = false;
if (numNeurons.Last() == 2)
{
computeSDOutput = true;
}
}
public DenseLayer CopyOnlyParams()
{
DenseLayer layer = new DenseLayer();
layer.weights = new Weights(this.weights);
layer.bias = new Bias(this.bias);
layer.activationFunc = this.activationFunc;
return(layer);
}
private void Backward(Matrix <float> X, Vector <float> y)
{
DenseLayer layer = layers.Last();
Matrix <float> delta = deltas.Last();
for (int i = layers.Length - 1; i >= 0; i--)
{
if (i == layers.Length - 1)
{
SetOutputDelta(X, y);
continue;
}
DenseLayer prevLayer = layers[i];
Matrix <float> W = layer.Weights.Vals;
Matrix <float> preActivation = preActivations[i];
DropoutLayer dropout = dropouts[i];
IActivationFunc activationFunc = prevLayer.ActivationFunc;
Matrix <float> deltaNew = deltas[i];
delta.TransposeAndMultiply(W, deltaNew);
preActivation.MapInplace(activationFunc.dF, Zeros.Include);
deltaNew.PointwiseMultiply(preActivation, deltaNew);
delta = deltaNew;
layer = prevLayer;
}
for (int i = layers.Length - 1; i >= 0; i--)
{
layer = layers[i];
delta = deltas[i];
Matrix <float> gradWeights = layer.GradWeights.Vals;
Matrix <float> gradBias = layer.GradBias.Vals;
Matrix <float> a = activations[i];
a.TransposeThisAndMultiply(delta, gradWeights);
gradWeights.Divide(miniBatchSize, gradWeights);
Vector <float> gradBiasVect = delta.ColumnSums();
gradBiasVect.Divide(miniBatchSize, gradBiasVect);
gradBias.SetRow(0, gradBiasVect);
}
}
public float[][] Forward(Matrix <float> X, bool pred)
{
Matrix <float>[] activations;
if (pred == false)
{
activations = this.activations;
}
else
{
activations = this.singleActivations;
}
activations[0] = X;
Matrix <float> a = activations[0];
for (int i = 0; i < layers.Length; i++)
{
DenseLayer layer = layers[i];
Matrix <float> W = layer.Weights.Vals;
Matrix <float> b = layer.Bias.Vals;
IActivationFunc activationFunc = layer.ActivationFunc;
DropoutLayer dropout = null;
if (dropouts != null)
{
dropout = dropouts[i];
}
if (pred == false)
{
b = layer.Bias.Broadcast(miniBatchSize);
}
Matrix <float> aNext = activations[i + 1];
a.Multiply(W, aNext);
aNext.Add(b, aNext);
if (pred == false)
{
aNext.CopyTo(preActivations[i]);
}
if (activationFunc.GetType() != typeof(Linear))
{
aNext.MapInplace(activationFunc.F, Zeros.Include);
}
if (pred == false && dropout != null)
{
dropout.Sample();
aNext.PointwiseMultiply(dropout.Vals, aNext);
}
a = aNext;
}
return(a.ToColumnArrays());
}