/// <summary>
/// Saves a specified NN to a file
/// </summary>
/// <param name="nn">The specified NN</param>
/// <param name="COG">[C]ritic [O]r [G]enerator</param>
public static void Write(NN nn, bool COG)
{
StreamWriter sw = new StreamWriter(new FileStream(COG ? CWBPath : GWBPath, FileMode.Create, FileAccess.Write, FileShare.None));
sw.Write(NN.Number + "," + nn.NumLayers + ",");
for (int i = 0; i < nn.NumLayers; i++)
{
if (nn.Layers[i] is FullyConnectedLayer)
{
sw.Write("0,");
}
if (nn.Layers[i] is SumLayer)
{
sw.Write("1,");
}
if (nn.Layers[i] is ConvolutionLayer)
{
var conv = nn.Layers[i] as ConvolutionLayer;
sw.Write("2," + conv.KernelSize.ToString() + "," + conv.PadSize.ToString()
+ "," + conv.Stride.ToString() + "," + (conv.DownOrUp ? "1," : "0,"));
}
if (nn.Layers[i] is PoolingLayer)
{
var pool = nn.Layers[i] as PoolingLayer;
sw.Write("3," + (pool.DownOrUp ? "1," : "0,") + pool.PoolSize + ",");
}
sw.Write(nn.Layers[i].Length + "," + nn.Layers[i].InputLength + ","
+ (nn.ResidualLayers[i] ? "1," : "0,") + (nn.BatchNormLayers[i] ? "1," : "0,")
+ nn.Activations[i].ToString() + ",");
//Sum layers have no weights
if (nn.Layers[i] is SumLayer || nn.Layers[i] is PoolingLayer)
{
continue;
}
for (int j = 0; j < nn.Layers[i].Weights.GetLength(0); j++)
{
for (int jj = 0; jj < nn.Layers[i].Weights.GetLength(1); jj++)
{
sw.Write(Math.Round(nn.Layers[i].Weights[j, jj], 5) + ",");
}
if (i != nn.NumLayers - 1 && nn.Layers[i] is FullyConnectedLayer)
{
sw.Write(Math.Round((nn.Layers[i] as FullyConnectedLayer).Biases[j], 5) + ",");
}
}
}
sw.Close();
}
private void TrainBtn_Click(object sender, EventArgs e)
{
if (NN.Training)
{
NN.Training = false; TrainBtn.Enabled = false; return;
}
NN.Training = true;
var thread = new Thread(() =>
{
NN.Train(Critic, Generator, latentsize, ctogratio, this, imgspeed);
//NN.TestTrain(Critic, GradientNormCB.Checked, imgspeed, this);
});
thread.IsBackground = true;
thread.Start();
}
public Form1()
{
InitializeComponent();
//Layer types combobox
LayerTypeCB.Items.Add("Fully Connected");
LayerTypeCB.Items.Add("Convolution");
LayerTypeCB.Items.Add("Sum");
LayerTypeCB.Items.Add("Pool");
LayerTypeCB.SelectedIndex = 0;
Epoch = 0;
//InputNormCB.Checked = true;
//GradientNormCB.Checked = true;
ClipTxt.Text = NN.ClipParameter.ToString();
AlphaTxt.Text = NN.LearningRate.ToString();
RMSDTxt.Text = NN.RMSDecay.ToString();
MTxt.Text = NN.BatchSize.ToString();
NormErrorsCB.Checked = false;
CTGTxt.Text = ctogratio.ToString();
try
{
Critic = IO.Read(true);
RefreshList(Critic, false);
Generator = IO.Read(false);
RefreshList(Generator, true);
}
catch
{
ActiveLayers = Default(false);
InactiveLayers = Default(true);
ResetBtn_Click(this, new EventArgs());
}
RefreshLayerLB();
//Only want this shown if a conv layer is selected
UpDownCB.Hide();
NumberTxt.Text = NN.Number.ToString();
}
/// <summary>
/// Test code to use the critic as a classifier
/// </summary>
public static void TestTrain(NN Critic, bool gradientnorm, int imgspeed, Form1 activeform)
{
int formupdateiterator = 0;
//Test code to generate a new layer with predefined qualities
//List<Layer> layers = new List<Layer>() { new ConvolutionLayer(4, 784) { DownOrUp = true, Stride = 1 }.Init(false), new ConvolutionLayer(3, 625){ DownOrUp = true, Stride = 1 }.Init(false),
// new ConvolutionLayer(2, 529){ DownOrUp = true, Stride = 1 }.Init(false), new FullyConnectedLayer(100, 484).Init(false), new FullyConnectedLayer(10, 100).Init(true) };
//List<bool> tans = new List<bool>() { true, true, true, true, true};
//List<bool> bns = new List<bool>() { false, false, false, false, false };
//List<bool> ress = new List<bool>() { false, false, false, false, false };
//NN Critic = new NN().Init(layers, tans, ress, bns);
while (Training)
{
double mean = 0;
double stddev = 0;
double score = 0;
double perccorrect = 0;
List <List <double[]> > nums = new List <List <double[]> >();
List <int> labels = new List <int>();
Random r = new Random();
for (int i = 0; i < 10; i++)
{
var temp = new List <double[]>();
for (int j = 0; j < BatchSize; j++)
{
temp.Add(Maths.Normalize(IO.FindNextNumber(i)));
//var tmpmean = Maths.CalcMean(temp[j]);
//mean += tmpmean;
//stddev += Maths.CalcStdDev(temp[j], tmpmean);
}
nums.Add(temp);
}
//Batch normalization
//mean /= 10 * batchsize; stddev /= 10 * batchsize;
//for (int i = 0; i < 10; i++)
//{
// nums[i] = Maths.BatchNormalize(nums[i], mean, stddev);
//}
//Foreach number
for (int i = 0; i < 10; i++)
{
Critic.Calculate(nums[i]);
//Foreach sample in the batch
for (int j = 0; j < BatchSize; j++)
{
double max = -99;
int guess = -1;
//Foreach output neuron
for (int k = 0; k < 10; k++)
{
var value = Critic.Layers[Critic.NumLayers - 1].Values[j][k];
score += Math.Pow(value - (k == i ? 1d : 0d), 2);
if (value > max)
{
max = value; guess = k;
}
}
perccorrect += guess == i ? 1d : 0d;
labels.Add(guess);
}
Critic.CalcGradients(nums[i], null, i, true);
}
score /= (10 * BatchSize);
perccorrect /= (10 * BatchSize);
score = Math.Sqrt(score);
Critic.Update();
//Report values to the front end
if (Clear)
{
Critic.Trials = 0; Critic.Error = 0; Critic.PercCorrect = 0; Clear = false;
}
Critic.Trials++;
Critic.Error = (Critic.Error * ((Critic.Trials) / (Critic.Trials + 1d))) + (score * (1d / (Critic.Trials)));
Critic.PercCorrect = (Critic.PercCorrect * ((Critic.Trials) / (Critic.Trials + 1d))) + (perccorrect * (1d / (Critic.Trials)));
//Update image (if applicable)
if (formupdateiterator >= imgspeed)
{
//Maths.Rescale(list8[0], mean8, stddev8);
int index = r.Next(0, 10);
var values = Form1.Rescale(Maths.Convert(nums[index][0]));
var image = new int[28, 28];
//Convert values to a 2d array
for (int i = 0; i < 28; i++)
{
for (int ii = 0; ii < 28; ii++)
{
image[ii, i] = (int)values[i, ii];
}
}
activeform.Invoke((Action) delegate
{
activeform.image = image;
activeform.CScore = Critic.Error.ToString();
activeform.CPerc = Critic.PercCorrect.ToString();
//Critic.Layers[Critic.NumLayers - 1].Values[0][index].ToString();
activeform.Label = labels[index].ToString();
if (Critic.Error > Form1.Cutoff)
{
Training = false;
}
if (IO.Reset)
{
IO.Reset = false;
activeform.Epoch++;
}
});
formupdateiterator = 0;
}
formupdateiterator++;
}
activeform.Invoke((Action) delegate
{
//Notify of being done training
activeform.DoneTraining = true;
//Reset errors
activeform.CScore = null;
activeform.GScore = null;
});
}
/// <summary>
/// Trains the GAN
/// </summary>
/// <param name="Critic">The network which criticises real and fake images</param>
/// <param name="Generator">The network which generates fake images</param>
/// <param name="LatentSize">How large the random noise input of the generator is</param>
/// <param name="ctg">The Critic To Generator training ratio</param>
/// <param name="num">The numerical digit to be learned (0-9)</param>
/// <param name="activeform">The form running this method</param>
/// <param name="imgspeed">How often generated images are pushed to the front-end</param>
public static void Train(NN Critic, NN Generator, int LatentSize, int ctg, Form1 activeform, int imgspeed)
{
int formupdateiterator = 0;
//The generator of the latentspace
Random r = new Random();
while (Training)
{
//Train critic x times per 1 of generator
for (int i = 0; i < ctg; i++)
{
//Batch norm stuff
double realmean = 0;
double realstddev = 0;
double AvgRealScore = 0;
double AvgFakeScore = 0;
//Generate samples
var realsamples = new List <double[]>();
var latentspaces = new List <double[]>();
for (int ii = 0; ii < BatchSize; ii++)
{
//Find next image
realsamples.Add(IO.FindNextNumber(NN.Number));
//Generate latent space for fake image
latentspaces.Add(Maths.RandomGaussian(r, LatentSize));
//Calculate values to help scale the fakes
var mean = Maths.CalcMean(realsamples[ii]);
realmean += mean;
realstddev += Maths.CalcStdDev(realsamples[ii], mean);
}
realmean /= BatchSize;
realstddev /= BatchSize;
//Batchnorm the samples
realsamples = Maths.BatchNormalize(realsamples, realmean, realstddev);
var fakesamples = Maths.BatchNormalize(Generator.GenerateSamples(latentspaces), realmean, realstddev);
//The RMSE of each network
double CError = 0;
double GError = 0;
//Critic's scores of each type of sample
List <double> rscores = new List <double>();
List <double> fscores = new List <double>();
//Real image calculations
double RealPercCorrect = 0;
Critic.Calculate(realsamples);
for (int j = 0; j < BatchSize; j++)
{
//The score is the value of the output (last) neuron of the critic
rscores.Add(Critic.Layers[Critic.NumLayers - 1].Values[j][0]);
AvgRealScore += rscores[j];
//Add the squared error
CError += Math.Pow(1d - Critic.Layers[Critic.NumLayers - 1].Values[j][0], 2);
GError += Math.Pow(-Critic.Layers[Critic.NumLayers - 1].Values[j][0], 2);
//Add whether it was correct or not to the total
RealPercCorrect += Critic.Layers[Critic.NumLayers - 1].Values[j][0] > 0 ? 1d : 0d;
}
AvgRealScore /= BatchSize;
RealPercCorrect /= BatchSize;
//Loss on real images = how accurate the critic is
Critic.CalcGradients(realsamples, null, RealPercCorrect, true);
//Fake image calculations
double FakePercIncorrect = 0;
Critic.Calculate(fakesamples);
for (int j = 0; j < BatchSize; j++)
{
//The score is the value of the output (last) neuron of the critic
fscores.Add(Critic.Layers[Critic.NumLayers - 1].Values[j][0]);
AvgFakeScore += fscores[j];
//Add the squared error
CError += Math.Pow(-Critic.Layers[Critic.NumLayers - 1].Values[j][0], 2);
GError += Math.Pow(1d - Critic.Layers[Critic.NumLayers - 1].Values[j][0], 2);
//Add whether it was correct or not to the total
FakePercIncorrect += Critic.Layers[Critic.NumLayers - 1].Values[j][0] > 0 ? 1d : 0d;
}
AvgFakeScore /= BatchSize;
FakePercIncorrect /= BatchSize;
//Wasserstein loss on fake images = real % correct - fake % correct
Critic.CalcGradients(fakesamples, null, RealPercCorrect - (1 - FakePercIncorrect), true);
//Update weights and biases
Critic.Update();
//Reset trial number if desired
if (Clear)
{
Critic.Trials = 0;
Generator.Trials = 0;
Clear = false;
}
//Critic processes 2 images per 1 the generator does
CError = Math.Sqrt(CError / (2 * BatchSize));
GError = Math.Sqrt(GError / BatchSize);
//Update errors and % correct values
Critic.Error = (Critic.Error * ((Critic.Trials) / (Critic.Trials + 1d))) + (CError * (1d / (Critic.Trials + 1d)));
Critic.PercCorrect = (Critic.PercCorrect * ((Critic.Trials) / (Critic.Trials + 1d))) + (RealPercCorrect * (1d / (Critic.Trials + 1d)));
Generator.Error = (Generator.Error * ((Generator.Trials) / (Generator.Trials + 1d))) + (GError * (1d / (Generator.Trials + 1)));
Generator.PercCorrect = (Generator.PercCorrect * ((Generator.Trials) / (Generator.Trials + 1d))) + (FakePercIncorrect * (1d / (Generator.Trials + 1d)));
//Iterate trial count
Critic.Trials++;
Generator.Trials++;
}
//Generate samples
List <double[]> testlatents = new List <double[]>();
for (int i = 0; i < BatchSize; i++)
{
testlatents.Add(Maths.RandomGaussian(r, LatentSize));
}
var tests = Generator.GenerateSamples(testlatents);
//Criticize generated samples
Critic.Calculate(tests);
//Compute generator's error on the critic's scores
double Score = 0;
for (int j = 0; j < BatchSize; j++)
{
Score += Critic.Layers[Critic.NumLayers - 1].Values[j][0] > 0 ? 1 : 0;
}
//Backprop through the critic to the generator
Critic.CalcGradients(tests, null, Score, false);
Generator.CalcGradients(testlatents, Critic.Layers[0], Score, true);
//Update the generator's weights and biases
Generator.Update();
//Update image (if applicable)
if (formupdateiterator >= imgspeed)
{
//Code that converts normalized generator outputs into an image
//Changes distribution of output values to 0-255 (brightness)
var values = Form1.Rescale(Maths.Convert(tests[0]));
var image = new int[28, 28];
//Convert values to a 2d int array
for (int i = 0; i < 28; i++)
{
for (int ii = 0; ii < 28; ii++)
{
image[ii, i] = (int)values[i, ii];
}
}
//Report values and image to the front end
activeform.Invoke((Action) delegate
{
activeform.image = image;
activeform.CScore = Critic.Error.ToString();
activeform.CPerc = Critic.PercCorrect.ToString();
activeform.GScore = Generator.Error.ToString();
activeform.GPerc = Generator.PercCorrect.ToString();
if (Critic.Error > Form1.Cutoff)
{
Training = false;
}
if (IO.Reset)
{
IO.Reset = false;
activeform.Epoch++;
}
});
formupdateiterator = 0;
}
formupdateiterator++;
}
if (Save)
{
//Save nns
IO.Write(Generator, false);
IO.Write(Critic, true);
}
activeform.Invoke((Action) delegate
{
//Notify of being done training
activeform.DoneTraining = true;
//Reset errors
activeform.CScore = null;
activeform.CPerc = null;
activeform.GScore = null;
activeform.GPerc = null;
});
}
private void RefreshList(NN desired, bool ActiveOrInactive)
{
var layers = new List <string>();
if (ActiveOrInactive)
{
LayerLB.Items.Clear();
}
int index = 0;
for (int i = 0; i < desired.Layers.Count; i++)
{
string layer = "";
//Filter by layer type
if (desired.Layers[i] is FullyConnectedLayer)
{
layer += "f,"; layer += desired.Layers[i].Length + ",";
if (!(desired.ResidualLayers is null))
{
layer += desired.ResidualLayers[i] ? "1," : "0,";
layer += desired.BatchNormLayers[i] ? "1," : "0,";
layer += desired.Activations[i].ToString() + ",";
}
}
if (desired.Layers[i] is ConvolutionLayer)
{
var conv = desired.Layers[i] as ConvolutionLayer;
layer += "c";
layer += conv.DownOrUp ? "," : "u,";
layer += conv.KernelSize + ",";
layer += desired.ResidualLayers[i] ? "1," : "0,";
layer += desired.BatchNormLayers[i] ? "1," : "0,";
layer += desired.Activations[i].ToString() + ",";
layer += conv.PadSize.ToString() + ",";
layer += conv.Stride.ToString();
}
if (desired.Layers[i] is SumLayer)
{
layer += "s";
}
if (desired.Layers[i] is PoolingLayer)
{
var pool = desired.Layers[i] as PoolingLayer;
layer += "p"; layer += pool.DownOrUp ? "," : "u,";
layer += pool.PoolSize + ",";
if (!(desired.ResidualLayers is null))
{
layer += desired.ResidualLayers[i] ? "1," : "0,";
layer += desired.BatchNormLayers[i] ? "1," : "0,";
layer += desired.Activations[i].ToString() + ",";
}
}
layers.Add(layer);
index++;
}
if (ActiveOrInactive)
{
ActiveLayers = layers;
RefreshLayerLB();
}
else
{
InactiveLayers = layers;
}
}
/*
* Apologies that this code is indecipherable; it was made during a flury of changes and bugfixes.
* Documenting it would have been purposeless at the time of writing due to how quickly it was being changed.
* And it's too large to easily document now.
*
* Its purpose is to read/write NNs to a file in CSV format
* If you misalign something, it will silently fail
*
* Should you choose to ignore this warning, good luck to you, a brave soul!
*/
/// <summary>
/// Returns a NN from a file
/// </summary>
/// <param name="COG">[C]ritic [O]r [G]enerator</param>
/// <returns></returns>
public static NN Read(bool COG)
{
NN nn = new NN();
nn.Layers = new List <Layer>();
nn.ResidualLayers = new List <bool>();
nn.BatchNormLayers = new List <bool>();
nn.Activations = new List <int>();
string[] text;
using (StreamReader sr = File.OpenText(COG ? CWBPath : GWBPath))
{
text = sr.ReadToEnd().Split(',');
}
NN.Number = int.Parse(text[0]);
nn.NumLayers = int.Parse(text[1]);
int iterator = 2;
for (int i = 0; i < nn.NumLayers; i++)
{
string type = text[iterator]; iterator++;
int kernelsize = 0;
int padsize = 0;
int stride = 0;
bool downorup = false;
if (type == "2")
{
kernelsize = int.Parse(text[iterator]); iterator++;
padsize = int.Parse(text[iterator]); iterator++;
stride = int.Parse(text[iterator]); iterator++;
if (text[iterator] == "1")
{
downorup = true;
}
iterator++;
}
if (type == "3")
{
downorup = text[iterator] == "1"; iterator++;
kernelsize = int.Parse(text[iterator]); iterator++;
}
int LayerCount = int.Parse(text[iterator]); iterator++;
int InputLayerCount = int.Parse(text[iterator]); iterator++;
nn.ResidualLayers.Add(text[iterator] == "1"); iterator++;
nn.BatchNormLayers.Add(text[iterator] == "1"); iterator++;
nn.Activations.Add(int.Parse(text[iterator])); iterator++;
if (type == "0")
{
nn.Layers.Add(new FullyConnectedLayer(LayerCount, InputLayerCount)); nn.Layers[i].ActivationFunction = nn.Activations[i];
}
//No weights exist in a sum layer
if (type == "1")
{
nn.Layers.Add(new SumLayer(LayerCount, InputLayerCount)); nn.Layers[i].ActivationFunction = nn.Activations[i]; continue;
}
if (type == "2")
{
nn.Layers.Add(new ConvolutionLayer(kernelsize, InputLayerCount));
var conv = nn.Layers[i] as ConvolutionLayer;
nn.Layers[i].Length = LayerCount;
nn.Layers[i].ActivationFunction = nn.Activations[i];
conv.PadSize = padsize; conv.Stride = stride; conv.DownOrUp = downorup;
}
if (type == "3")
{
nn.Layers.Add(new PoolingLayer(downorup, kernelsize, InputLayerCount));
nn.Layers[i].ActivationFunction = nn.Activations[i];
//No weights exist in a pooling layer
continue;
}
for (int j = 0; j < nn.Layers[i].Weights.GetLength(0); j++)
{
for (int jj = 0; jj < nn.Layers[i].Weights.GetLength(1); jj++)
{
nn.Layers[i].Weights[j, jj] = double.Parse(text[iterator]); iterator++;
}
if (i != nn.NumLayers - 1 && nn.Layers[i] is FullyConnectedLayer)
{
(nn.Layers[i] as FullyConnectedLayer).Biases[j] = double.Parse(text[iterator]); iterator++;
}
}
}
return(nn);
}