/// <summary>
/// Test code to use the critic as a classifier
/// </summary>
public static void TestTrain(NN Critic, bool gradientnorm, int imgspeed, Form1 activeform)
{
int formupdateiterator = 0;
//Test code to generate a new layer with predefined qualities
//List<Layer> layers = new List<Layer>() { new ConvolutionLayer(4, 784) { DownOrUp = true, Stride = 1 }.Init(false), new ConvolutionLayer(3, 625){ DownOrUp = true, Stride = 1 }.Init(false),
// new ConvolutionLayer(2, 529){ DownOrUp = true, Stride = 1 }.Init(false), new FullyConnectedLayer(100, 484).Init(false), new FullyConnectedLayer(10, 100).Init(true) };
//List<bool> tans = new List<bool>() { true, true, true, true, true};
//List<bool> bns = new List<bool>() { false, false, false, false, false };
//List<bool> ress = new List<bool>() { false, false, false, false, false };
//NN Critic = new NN().Init(layers, tans, ress, bns);
while (Training)
{
double mean = 0;
double stddev = 0;
double score = 0;
double perccorrect = 0;
List <List <double[]> > nums = new List <List <double[]> >();
List <int> labels = new List <int>();
Random r = new Random();
for (int i = 0; i < 10; i++)
{
var temp = new List <double[]>();
for (int j = 0; j < BatchSize; j++)
{
temp.Add(Maths.Normalize(IO.FindNextNumber(i)));
//var tmpmean = Maths.CalcMean(temp[j]);
//mean += tmpmean;
//stddev += Maths.CalcStdDev(temp[j], tmpmean);
}
nums.Add(temp);
}
//Batch normalization
//mean /= 10 * batchsize; stddev /= 10 * batchsize;
//for (int i = 0; i < 10; i++)
//{
// nums[i] = Maths.BatchNormalize(nums[i], mean, stddev);
//}
//Foreach number
for (int i = 0; i < 10; i++)
{
Critic.Calculate(nums[i]);
//Foreach sample in the batch
for (int j = 0; j < BatchSize; j++)
{
double max = -99;
int guess = -1;
//Foreach output neuron
for (int k = 0; k < 10; k++)
{
var value = Critic.Layers[Critic.NumLayers - 1].Values[j][k];
score += Math.Pow(value - (k == i ? 1d : 0d), 2);
if (value > max)
{
max = value; guess = k;
}
}
perccorrect += guess == i ? 1d : 0d;
labels.Add(guess);
}
Critic.CalcGradients(nums[i], null, i, true);
}
score /= (10 * BatchSize);
perccorrect /= (10 * BatchSize);
score = Math.Sqrt(score);
Critic.Update();
//Report values to the front end
if (Clear)
{
Critic.Trials = 0; Critic.Error = 0; Critic.PercCorrect = 0; Clear = false;
}
Critic.Trials++;
Critic.Error = (Critic.Error * ((Critic.Trials) / (Critic.Trials + 1d))) + (score * (1d / (Critic.Trials)));
Critic.PercCorrect = (Critic.PercCorrect * ((Critic.Trials) / (Critic.Trials + 1d))) + (perccorrect * (1d / (Critic.Trials)));
//Update image (if applicable)
if (formupdateiterator >= imgspeed)
{
//Maths.Rescale(list8[0], mean8, stddev8);
int index = r.Next(0, 10);
var values = Form1.Rescale(Maths.Convert(nums[index][0]));
var image = new int[28, 28];
//Convert values to a 2d array
for (int i = 0; i < 28; i++)
{
for (int ii = 0; ii < 28; ii++)
{
image[ii, i] = (int)values[i, ii];
}
}
activeform.Invoke((Action) delegate
{
activeform.image = image;
activeform.CScore = Critic.Error.ToString();
activeform.CPerc = Critic.PercCorrect.ToString();
//Critic.Layers[Critic.NumLayers - 1].Values[0][index].ToString();
activeform.Label = labels[index].ToString();
if (Critic.Error > Form1.Cutoff)
{
Training = false;
}
if (IO.Reset)
{
IO.Reset = false;
activeform.Epoch++;
}
});
formupdateiterator = 0;
}
formupdateiterator++;
}
activeform.Invoke((Action) delegate
{
//Notify of being done training
activeform.DoneTraining = true;
//Reset errors
activeform.CScore = null;
activeform.GScore = null;
});
}
/// <summary>
/// Trains the GAN
/// </summary>
/// <param name="Critic">The network which criticises real and fake images</param>
/// <param name="Generator">The network which generates fake images</param>
/// <param name="LatentSize">How large the random noise input of the generator is</param>
/// <param name="ctg">The Critic To Generator training ratio</param>
/// <param name="num">The numerical digit to be learned (0-9)</param>
/// <param name="activeform">The form running this method</param>
/// <param name="imgspeed">How often generated images are pushed to the front-end</param>
public static void Train(NN Critic, NN Generator, int LatentSize, int ctg, Form1 activeform, int imgspeed)
{
int formupdateiterator = 0;
//The generator of the latentspace
Random r = new Random();
while (Training)
{
//Train critic x times per 1 of generator
for (int i = 0; i < ctg; i++)
{
//Batch norm stuff
double realmean = 0;
double realstddev = 0;
double AvgRealScore = 0;
double AvgFakeScore = 0;
//Generate samples
var realsamples = new List <double[]>();
var latentspaces = new List <double[]>();
for (int ii = 0; ii < BatchSize; ii++)
{
//Find next image
realsamples.Add(IO.FindNextNumber(NN.Number));
//Generate latent space for fake image
latentspaces.Add(Maths.RandomGaussian(r, LatentSize));
//Calculate values to help scale the fakes
var mean = Maths.CalcMean(realsamples[ii]);
realmean += mean;
realstddev += Maths.CalcStdDev(realsamples[ii], mean);
}
realmean /= BatchSize;
realstddev /= BatchSize;
//Batchnorm the samples
realsamples = Maths.BatchNormalize(realsamples, realmean, realstddev);
var fakesamples = Maths.BatchNormalize(Generator.GenerateSamples(latentspaces), realmean, realstddev);
//The RMSE of each network
double CError = 0;
double GError = 0;
//Critic's scores of each type of sample
List <double> rscores = new List <double>();
List <double> fscores = new List <double>();
//Real image calculations
double RealPercCorrect = 0;
Critic.Calculate(realsamples);
for (int j = 0; j < BatchSize; j++)
{
//The score is the value of the output (last) neuron of the critic
rscores.Add(Critic.Layers[Critic.NumLayers - 1].Values[j][0]);
AvgRealScore += rscores[j];
//Add the squared error
CError += Math.Pow(1d - Critic.Layers[Critic.NumLayers - 1].Values[j][0], 2);
GError += Math.Pow(-Critic.Layers[Critic.NumLayers - 1].Values[j][0], 2);
//Add whether it was correct or not to the total
RealPercCorrect += Critic.Layers[Critic.NumLayers - 1].Values[j][0] > 0 ? 1d : 0d;
}
AvgRealScore /= BatchSize;
RealPercCorrect /= BatchSize;
//Loss on real images = how accurate the critic is
Critic.CalcGradients(realsamples, null, RealPercCorrect, true);
//Fake image calculations
double FakePercIncorrect = 0;
Critic.Calculate(fakesamples);
for (int j = 0; j < BatchSize; j++)
{
//The score is the value of the output (last) neuron of the critic
fscores.Add(Critic.Layers[Critic.NumLayers - 1].Values[j][0]);
AvgFakeScore += fscores[j];
//Add the squared error
CError += Math.Pow(-Critic.Layers[Critic.NumLayers - 1].Values[j][0], 2);
GError += Math.Pow(1d - Critic.Layers[Critic.NumLayers - 1].Values[j][0], 2);
//Add whether it was correct or not to the total
FakePercIncorrect += Critic.Layers[Critic.NumLayers - 1].Values[j][0] > 0 ? 1d : 0d;
}
AvgFakeScore /= BatchSize;
FakePercIncorrect /= BatchSize;
//Wasserstein loss on fake images = real % correct - fake % correct
Critic.CalcGradients(fakesamples, null, RealPercCorrect - (1 - FakePercIncorrect), true);
//Update weights and biases
Critic.Update();
//Reset trial number if desired
if (Clear)
{
Critic.Trials = 0;
Generator.Trials = 0;
Clear = false;
}
//Critic processes 2 images per 1 the generator does
CError = Math.Sqrt(CError / (2 * BatchSize));
GError = Math.Sqrt(GError / BatchSize);
//Update errors and % correct values
Critic.Error = (Critic.Error * ((Critic.Trials) / (Critic.Trials + 1d))) + (CError * (1d / (Critic.Trials + 1d)));
Critic.PercCorrect = (Critic.PercCorrect * ((Critic.Trials) / (Critic.Trials + 1d))) + (RealPercCorrect * (1d / (Critic.Trials + 1d)));
Generator.Error = (Generator.Error * ((Generator.Trials) / (Generator.Trials + 1d))) + (GError * (1d / (Generator.Trials + 1)));
Generator.PercCorrect = (Generator.PercCorrect * ((Generator.Trials) / (Generator.Trials + 1d))) + (FakePercIncorrect * (1d / (Generator.Trials + 1d)));
//Iterate trial count
Critic.Trials++;
Generator.Trials++;
}
//Generate samples
List <double[]> testlatents = new List <double[]>();
for (int i = 0; i < BatchSize; i++)
{
testlatents.Add(Maths.RandomGaussian(r, LatentSize));
}
var tests = Generator.GenerateSamples(testlatents);
//Criticize generated samples
Critic.Calculate(tests);
//Compute generator's error on the critic's scores
double Score = 0;
for (int j = 0; j < BatchSize; j++)
{
Score += Critic.Layers[Critic.NumLayers - 1].Values[j][0] > 0 ? 1 : 0;
}
//Backprop through the critic to the generator
Critic.CalcGradients(tests, null, Score, false);
Generator.CalcGradients(testlatents, Critic.Layers[0], Score, true);
//Update the generator's weights and biases
Generator.Update();
//Update image (if applicable)
if (formupdateiterator >= imgspeed)
{
//Code that converts normalized generator outputs into an image
//Changes distribution of output values to 0-255 (brightness)
var values = Form1.Rescale(Maths.Convert(tests[0]));
var image = new int[28, 28];
//Convert values to a 2d int array
for (int i = 0; i < 28; i++)
{
for (int ii = 0; ii < 28; ii++)
{
image[ii, i] = (int)values[i, ii];
}
}
//Report values and image to the front end
activeform.Invoke((Action) delegate
{
activeform.image = image;
activeform.CScore = Critic.Error.ToString();
activeform.CPerc = Critic.PercCorrect.ToString();
activeform.GScore = Generator.Error.ToString();
activeform.GPerc = Generator.PercCorrect.ToString();
if (Critic.Error > Form1.Cutoff)
{
Training = false;
}
if (IO.Reset)
{
IO.Reset = false;
activeform.Epoch++;
}
});
formupdateiterator = 0;
}
formupdateiterator++;
}
if (Save)
{
//Save nns
IO.Write(Generator, false);
IO.Write(Critic, true);
}
activeform.Invoke((Action) delegate
{
//Notify of being done training
activeform.DoneTraining = true;
//Reset errors
activeform.CScore = null;
activeform.CPerc = null;
activeform.GScore = null;
activeform.GPerc = null;
});
}