public override void CalcGradients(List <double[]> inputs, Layer outputlayer)
{
for (int b = 0; b < NN.BatchSize; b++)
{
//var input = inputs[i];
//if (UsesTanh) { input = Maths.TanhDerriv(inputs[i]); }
double[,] Input = Pad(Maths.Convert(inputs[b]));
double[,] stochgradients;
if (DownOrUp)
{
stochgradients = Convolve(Maths.Convert(Errors[b]), Input);
}
else
{
stochgradients = Convolve(Input, Maths.Convert(Errors[b]));
}
//Gradients = stochgradients;
//Add the stochastic gradients to the batch gradients
for (int j = 0; j < Gradients.GetLength(0); j++)
{
for (int k = 0; k < Gradients.GetLength(1); k++)
{
Gradients[j, k] += stochgradients[j, k];
}
}
}
}
public override void Calculate(List <double[]> inputs, bool output)
{
ZVals = new List <double[]>();
for (int i = 0; i < NN.BatchSize; i++)
{
ZVals.Add(Maths.Convert(Pool(Maths.Convert(inputs[i]), output)));
}
//If normalizing, do so, but only if it won't return an all-zero matrix
if (NN.NormOutputs && ZVals[0].Length > 1)
{
ZVals = Maths.Normalize(ZVals);
}
//Use the specified type of activation function
if (ActivationFunction == 0)
{
Values = Maths.Tanh(ZVals); return;
}
if (ActivationFunction == 1)
{
Values = Maths.ReLu(ZVals); return;
}
Values = ZVals;
}
/// <summary>
/// Calculates the dot product of the kernel and input matrix.
/// Matrices should be size [x, y] and [y], respectively, where x is the output size and y is the latent space's size
/// </summary>
/// <param name="inputs">The input matrix</param>
/// <param name="isoutput">Whether to use hyperbolic tangent on the output</param>
/// <returns></returns>
public override void Calculate(List <double[]> inputs, bool isoutput)
{
ZVals = new List <double[]>();
for (int b = 0; b < NN.BatchSize; b++)
{
ZVals.Add(Maths.Convert(DownOrUp ? Convolve(Weights, Pad(Maths.Convert(inputs[b]))) : FullConvolve(Weights, Pad(Maths.Convert(inputs[b])))));
}
//If normalizing, do so, but only if it won't return an all-zero matrix
if (NN.NormOutputs && ZVals[0].Length > 1)
{
ZVals = Maths.Normalize(ZVals);
}
//Use the specified type of activation function
if (ActivationFunction == 0)
{
Values = Maths.Tanh(ZVals); return;
}
if (ActivationFunction == 1)
{
Values = Maths.ReLu(ZVals); return;
}
Values = ZVals;
}
/// <summary>
/// Test code to use the critic as a classifier
/// </summary>
public static void TestTrain(NN Critic, bool gradientnorm, int imgspeed, Form1 activeform)
{
int formupdateiterator = 0;
//Test code to generate a new layer with predefined qualities
//List<Layer> layers = new List<Layer>() { new ConvolutionLayer(4, 784) { DownOrUp = true, Stride = 1 }.Init(false), new ConvolutionLayer(3, 625){ DownOrUp = true, Stride = 1 }.Init(false),
// new ConvolutionLayer(2, 529){ DownOrUp = true, Stride = 1 }.Init(false), new FullyConnectedLayer(100, 484).Init(false), new FullyConnectedLayer(10, 100).Init(true) };
//List<bool> tans = new List<bool>() { true, true, true, true, true};
//List<bool> bns = new List<bool>() { false, false, false, false, false };
//List<bool> ress = new List<bool>() { false, false, false, false, false };
//NN Critic = new NN().Init(layers, tans, ress, bns);
while (Training)
{
double mean = 0;
double stddev = 0;
double score = 0;
double perccorrect = 0;
List <List <double[]> > nums = new List <List <double[]> >();
List <int> labels = new List <int>();
Random r = new Random();
for (int i = 0; i < 10; i++)
{
var temp = new List <double[]>();
for (int j = 0; j < BatchSize; j++)
{
temp.Add(Maths.Normalize(IO.FindNextNumber(i)));
//var tmpmean = Maths.CalcMean(temp[j]);
//mean += tmpmean;
//stddev += Maths.CalcStdDev(temp[j], tmpmean);
}
nums.Add(temp);
}
//Batch normalization
//mean /= 10 * batchsize; stddev /= 10 * batchsize;
//for (int i = 0; i < 10; i++)
//{
// nums[i] = Maths.BatchNormalize(nums[i], mean, stddev);
//}
//Foreach number
for (int i = 0; i < 10; i++)
{
Critic.Calculate(nums[i]);
//Foreach sample in the batch
for (int j = 0; j < BatchSize; j++)
{
double max = -99;
int guess = -1;
//Foreach output neuron
for (int k = 0; k < 10; k++)
{
var value = Critic.Layers[Critic.NumLayers - 1].Values[j][k];
score += Math.Pow(value - (k == i ? 1d : 0d), 2);
if (value > max)
{
max = value; guess = k;
}
}
perccorrect += guess == i ? 1d : 0d;
labels.Add(guess);
}
Critic.CalcGradients(nums[i], null, i, true);
}
score /= (10 * BatchSize);
perccorrect /= (10 * BatchSize);
score = Math.Sqrt(score);
Critic.Update();
//Report values to the front end
if (Clear)
{
Critic.Trials = 0; Critic.Error = 0; Critic.PercCorrect = 0; Clear = false;
}
Critic.Trials++;
Critic.Error = (Critic.Error * ((Critic.Trials) / (Critic.Trials + 1d))) + (score * (1d / (Critic.Trials)));
Critic.PercCorrect = (Critic.PercCorrect * ((Critic.Trials) / (Critic.Trials + 1d))) + (perccorrect * (1d / (Critic.Trials)));
//Update image (if applicable)
if (formupdateiterator >= imgspeed)
{
//Maths.Rescale(list8[0], mean8, stddev8);
int index = r.Next(0, 10);
var values = Form1.Rescale(Maths.Convert(nums[index][0]));
var image = new int[28, 28];
//Convert values to a 2d array
for (int i = 0; i < 28; i++)
{
for (int ii = 0; ii < 28; ii++)
{
image[ii, i] = (int)values[i, ii];
}
}
activeform.Invoke((Action) delegate
{
activeform.image = image;
activeform.CScore = Critic.Error.ToString();
activeform.CPerc = Critic.PercCorrect.ToString();
//Critic.Layers[Critic.NumLayers - 1].Values[0][index].ToString();
activeform.Label = labels[index].ToString();
if (Critic.Error > Form1.Cutoff)
{
Training = false;
}
if (IO.Reset)
{
IO.Reset = false;
activeform.Epoch++;
}
});
formupdateiterator = 0;
}
formupdateiterator++;
}
activeform.Invoke((Action) delegate
{
//Notify of being done training
activeform.DoneTraining = true;
//Reset errors
activeform.CScore = null;
activeform.GScore = null;
});
}
/// <summary>
/// Trains the GAN
/// </summary>
/// <param name="Critic">The network which criticises real and fake images</param>
/// <param name="Generator">The network which generates fake images</param>
/// <param name="LatentSize">How large the random noise input of the generator is</param>
/// <param name="ctg">The Critic To Generator training ratio</param>
/// <param name="num">The numerical digit to be learned (0-9)</param>
/// <param name="activeform">The form running this method</param>
/// <param name="imgspeed">How often generated images are pushed to the front-end</param>
public static void Train(NN Critic, NN Generator, int LatentSize, int ctg, Form1 activeform, int imgspeed)
{
int formupdateiterator = 0;
//The generator of the latentspace
Random r = new Random();
while (Training)
{
//Train critic x times per 1 of generator
for (int i = 0; i < ctg; i++)
{
//Batch norm stuff
double realmean = 0;
double realstddev = 0;
double AvgRealScore = 0;
double AvgFakeScore = 0;
//Generate samples
var realsamples = new List <double[]>();
var latentspaces = new List <double[]>();
for (int ii = 0; ii < BatchSize; ii++)
{
//Find next image
realsamples.Add(IO.FindNextNumber(NN.Number));
//Generate latent space for fake image
latentspaces.Add(Maths.RandomGaussian(r, LatentSize));
//Calculate values to help scale the fakes
var mean = Maths.CalcMean(realsamples[ii]);
realmean += mean;
realstddev += Maths.CalcStdDev(realsamples[ii], mean);
}
realmean /= BatchSize;
realstddev /= BatchSize;
//Batchnorm the samples
realsamples = Maths.BatchNormalize(realsamples, realmean, realstddev);
var fakesamples = Maths.BatchNormalize(Generator.GenerateSamples(latentspaces), realmean, realstddev);
//The RMSE of each network
double CError = 0;
double GError = 0;
//Critic's scores of each type of sample
List <double> rscores = new List <double>();
List <double> fscores = new List <double>();
//Real image calculations
double RealPercCorrect = 0;
Critic.Calculate(realsamples);
for (int j = 0; j < BatchSize; j++)
{
//The score is the value of the output (last) neuron of the critic
rscores.Add(Critic.Layers[Critic.NumLayers - 1].Values[j][0]);
AvgRealScore += rscores[j];
//Add the squared error
CError += Math.Pow(1d - Critic.Layers[Critic.NumLayers - 1].Values[j][0], 2);
GError += Math.Pow(-Critic.Layers[Critic.NumLayers - 1].Values[j][0], 2);
//Add whether it was correct or not to the total
RealPercCorrect += Critic.Layers[Critic.NumLayers - 1].Values[j][0] > 0 ? 1d : 0d;
}
AvgRealScore /= BatchSize;
RealPercCorrect /= BatchSize;
//Loss on real images = how accurate the critic is
Critic.CalcGradients(realsamples, null, RealPercCorrect, true);
//Fake image calculations
double FakePercIncorrect = 0;
Critic.Calculate(fakesamples);
for (int j = 0; j < BatchSize; j++)
{
//The score is the value of the output (last) neuron of the critic
fscores.Add(Critic.Layers[Critic.NumLayers - 1].Values[j][0]);
AvgFakeScore += fscores[j];
//Add the squared error
CError += Math.Pow(-Critic.Layers[Critic.NumLayers - 1].Values[j][0], 2);
GError += Math.Pow(1d - Critic.Layers[Critic.NumLayers - 1].Values[j][0], 2);
//Add whether it was correct or not to the total
FakePercIncorrect += Critic.Layers[Critic.NumLayers - 1].Values[j][0] > 0 ? 1d : 0d;
}
AvgFakeScore /= BatchSize;
FakePercIncorrect /= BatchSize;
//Wasserstein loss on fake images = real % correct - fake % correct
Critic.CalcGradients(fakesamples, null, RealPercCorrect - (1 - FakePercIncorrect), true);
//Update weights and biases
Critic.Update();
//Reset trial number if desired
if (Clear)
{
Critic.Trials = 0;
Generator.Trials = 0;
Clear = false;
}
//Critic processes 2 images per 1 the generator does
CError = Math.Sqrt(CError / (2 * BatchSize));
GError = Math.Sqrt(GError / BatchSize);
//Update errors and % correct values
Critic.Error = (Critic.Error * ((Critic.Trials) / (Critic.Trials + 1d))) + (CError * (1d / (Critic.Trials + 1d)));
Critic.PercCorrect = (Critic.PercCorrect * ((Critic.Trials) / (Critic.Trials + 1d))) + (RealPercCorrect * (1d / (Critic.Trials + 1d)));
Generator.Error = (Generator.Error * ((Generator.Trials) / (Generator.Trials + 1d))) + (GError * (1d / (Generator.Trials + 1)));
Generator.PercCorrect = (Generator.PercCorrect * ((Generator.Trials) / (Generator.Trials + 1d))) + (FakePercIncorrect * (1d / (Generator.Trials + 1d)));
//Iterate trial count
Critic.Trials++;
Generator.Trials++;
}
//Generate samples
List <double[]> testlatents = new List <double[]>();
for (int i = 0; i < BatchSize; i++)
{
testlatents.Add(Maths.RandomGaussian(r, LatentSize));
}
var tests = Generator.GenerateSamples(testlatents);
//Criticize generated samples
Critic.Calculate(tests);
//Compute generator's error on the critic's scores
double Score = 0;
for (int j = 0; j < BatchSize; j++)
{
Score += Critic.Layers[Critic.NumLayers - 1].Values[j][0] > 0 ? 1 : 0;
}
//Backprop through the critic to the generator
Critic.CalcGradients(tests, null, Score, false);
Generator.CalcGradients(testlatents, Critic.Layers[0], Score, true);
//Update the generator's weights and biases
Generator.Update();
//Update image (if applicable)
if (formupdateiterator >= imgspeed)
{
//Code that converts normalized generator outputs into an image
//Changes distribution of output values to 0-255 (brightness)
var values = Form1.Rescale(Maths.Convert(tests[0]));
var image = new int[28, 28];
//Convert values to a 2d int array
for (int i = 0; i < 28; i++)
{
for (int ii = 0; ii < 28; ii++)
{
image[ii, i] = (int)values[i, ii];
}
}
//Report values and image to the front end
activeform.Invoke((Action) delegate
{
activeform.image = image;
activeform.CScore = Critic.Error.ToString();
activeform.CPerc = Critic.PercCorrect.ToString();
activeform.GScore = Generator.Error.ToString();
activeform.GPerc = Generator.PercCorrect.ToString();
if (Critic.Error > Form1.Cutoff)
{
Training = false;
}
if (IO.Reset)
{
IO.Reset = false;
activeform.Epoch++;
}
});
formupdateiterator = 0;
}
formupdateiterator++;
}
if (Save)
{
//Save nns
IO.Write(Generator, false);
IO.Write(Critic, true);
}
activeform.Invoke((Action) delegate
{
//Notify of being done training
activeform.DoneTraining = true;
//Reset errors
activeform.CScore = null;
activeform.CPerc = null;
activeform.GScore = null;
activeform.GPerc = null;
});
}
/// <summary>
/// Computes the error signal of the layer, also gradients if applicable
/// </summary>
/// <param name="input">Previous layer's values</param>
/// <param name="output">Whether the layer is the output layer</param>
/// <param name="loss">The loss of the layer</param>
/// <param name="calcgradients">Whether or not to calculate gradients in the layer</param>
public void Backprop(List <double[]> inputs, Layer outputlayer, double loss, bool calcgradients)
{
//Reset errors
Errors = new List <double[]>();
//Calculate errors
if (outputlayer is null)
{
for (int j = 0; j < inputs.Count; j++)
{
Errors.Add(new double[Length]);
for (int i = 0; i < Length; i++)
{
//(i == loss ? 1d : 0d)
Errors[j][i] = 2d * (Values[j][i] - loss);
}
}
}
else
{
for (int i = 0; i < inputs.Count; i++)
{
Errors.Add(new double[outputlayer.InputLength]);
}
if (outputlayer is SumLayer)
{
//Errors with respect to the output of the convolution
//dl/do
for (int i = 0; i < outputlayer.ZVals.Count; i++)
{
for (int k = 0; k < outputlayer.Length; k++)
{
for (int j = 0; j < outputlayer.InputLength; j++)
{
Errors[i][j] += outputlayer.Errors[i][k];
}
}
}
}
//Apply tanhderriv, if applicable, to the output's zvals
var outputZVals = outputlayer.ZVals;
if (outputlayer.ActivationFunction == 0)
{
outputZVals = Maths.TanhDerriv(outputlayer.ZVals);
}
if (outputlayer.ActivationFunction == 1)
{
outputZVals = Maths.ReLuDerriv(outputlayer.ZVals);
}
if (outputlayer is FullyConnectedLayer)
{
var FCLOutput = outputlayer as FullyConnectedLayer;
for (int i = 0; i < outputlayer.ZVals.Count; i++)
{
for (int k = 0; k < FCLOutput.Length; k++)
{
for (int j = 0; j < FCLOutput.InputLength; j++)
{
Errors[i][j] += FCLOutput.Weights[k, j] * outputZVals[i][k] * FCLOutput.Errors[i][k];
}
}
}
}
if (outputlayer is ConvolutionLayer)
{
var CLOutput = outputlayer as ConvolutionLayer;
for (int i = 0; i < outputlayer.ZVals.Count; i++)
{
if ((outputlayer as ConvolutionLayer).DownOrUp)
{
Errors[i] = Maths.Convert(CLOutput.UnPad(CLOutput.FullConvolve(CLOutput.Weights, Maths.Convert(CLOutput.Errors[i]))));
}
else
{
Errors[i] = Maths.Convert(CLOutput.UnPad(CLOutput.Convolve(CLOutput.Weights, Maths.Convert(CLOutput.Errors[i]))));
}
}
//Errors = Maths.Convert(CLOutput.UnPad(CLOutput.FullConvolve(CLOutput.Weights, Maths.Convert(CLOutput.Errors))));
}
if (outputlayer is PoolingLayer)
{
var PLOutput = outputlayer as PoolingLayer;
for (int b = 0; b < NN.BatchSize; b++)
{
if (PLOutput.DownOrUp)
{
int iterator = 0;
var wets = Maths.Convert(PLOutput.Weights);
for (int i = 0; i < Length; i++)
{
if (wets[i] == 0)
{
continue;
}
Errors[b][i] = PLOutput.Errors[b][iterator];
iterator++;
}
}
else
{
//Sum the errors
double[,] outputerrors = Maths.Convert(PLOutput.Errors[b]);
int oel = outputerrors.GetLength(0);
int oew = outputerrors.GetLength(1);
double[,] errors = new double[oel / PLOutput.PoolSize, oew / PLOutput.PoolSize];
for (int i = 0; i < oel; i++)
{
for (int ii = 0; ii < oew; ii++)
{
errors[i / PLOutput.PoolSize, ii / PLOutput.PoolSize] += outputerrors[i, ii];
}
}
Errors[b] = Maths.Convert(errors);
}
}
}
}
//Normalize errors (if applicable)
if (NN.NormErrors && Errors[0].Length > 1)
{
Errors = Maths.Normalize(Errors);
}
if (calcgradients)
{
if (this is FullyConnectedLayer)
{
(this as FullyConnectedLayer).CalcGradients(inputs, outputlayer);
}
if (this is ConvolutionLayer)
{
(this as ConvolutionLayer).CalcGradients(inputs, outputlayer);
}
if (this is PoolingLayer)
{
return;
}
if (this is SumLayer)
{
return;
}
}
}