public void Train()
{
string dir = "ND_OPT_ConvAutoencoder_Data";
Directory.CreateDirectory($@"{dir}");
Directory.CreateDirectory($@"{dir}\Results");
Directory.CreateDirectory($@"{dir}\Sources");
AnimeDatasets a_dataset = new AnimeDatasets(StartSide, @"I:\Datasets\VAE_Dataset\White", @"I:\Datasets\VAE_Dataset\White\conv");//@"I:\Datasets\anime-faces\combined", @"I:\Datasets\anime-faces\combined_small");
a_dataset.InitializeDataset();
AnimeDatasets b_dataset = new AnimeDatasets(EndSide, @"I:\Datasets\VAE_Dataset\White", @"I:\Datasets\VAE_Dataset\White\conv");//@"I:\Datasets\anime-faces\combined", @"I:\Datasets\anime-faces\combined_small");
b_dataset.InitializeDataset();
Adam sgd = new Adam(0.001f);
Quadratic quadratic = new Quadratic();
NRandom r = new NRandom(0);
NRandom r2 = new NRandom(0);
Matrix loss_deriv = new Matrix(OutputSize, 1, MemoryFlags.ReadWrite, true);
#region Setup Database
Matrix data_vec = new Matrix(LatentSize, 1, MemoryFlags.ReadOnly, false);
Matrix[] a_dataset_vec = new Matrix[a_dataset.TrainingFiles.Count];
float[][] a_dataset_f = new float[a_dataset.TrainingFiles.Count][];
Matrix[] b_dataset_vec = new Matrix[a_dataset.TrainingFiles.Count];
float[][] b_dataset_f = new float[a_dataset.TrainingFiles.Count][];
for (int i = 0; i < a_dataset.TrainingFiles.Count; i++)
{
a_dataset_f[i] = new float[InputSize];
a_dataset_vec[i] = new Matrix(InputSize, 1, MemoryFlags.ReadOnly, false);
a_dataset.LoadImage(a_dataset.TrainingFiles[i], a_dataset_f[i]);
a_dataset_vec[i].Write(a_dataset_f[i]);
b_dataset_f[i] = new float[OutputSize];
b_dataset_vec[i] = new Matrix(OutputSize, 1, MemoryFlags.ReadOnly, false);
b_dataset.LoadImage(b_dataset.TrainingFiles[i], b_dataset_f[i]);
b_dataset_vec[i].Write(b_dataset_f[i]);
}
#endregion
for (int i0 = 000; i0 < 20000 * BatchSize; i0++)
{
int idx = (r.Next() % (a_dataset.TrainingFiles.Count / 2));
var out_img = superres_enc_front.ForwardPropagate(a_dataset_vec[idx]);
quadratic.LossDeriv(out_img[0], b_dataset_vec[idx], loss_deriv, 0);
superres_dec_back.ResetLayerErrors();
superres_dec_back.ComputeGradients(loss_deriv);
superres_dec_back.ComputeLayerErrors(loss_deriv);
superres_dec_back.UpdateLayers(sgd);
loss_deriv.Clear();
if (i0 % BatchSize == 0)
{
a_dataset.SaveImage($@"{dir}\Sources\{i0 / BatchSize}.png", a_dataset_f[idx]);
b_dataset.SaveImage($@"{dir}\Results\{i0 / BatchSize}.png", out_img[0].Read());
}
Console.Clear();
Console.Write($"Iteration: {i0 / BatchSize}, Sub-Batch: {i0 % BatchSize}");
}
superres_enc_front.Save($@"{dir}\network_final.bin");
Console.WriteLine("DONE.");
}
public void Check()
{
front = InputLayer.Create(3, 1);
back = ActivationLayer.Create <Sigmoid>();
conv = ConvLayer.Create(2, 1);
fc = FCLayer.Create(1, 1);
front.Append(
conv.Append(
ConvLayer.Create(3, 3, 2).Append(
ActivationLayer.Create <LeakyReLU>().Append(
fc.Append(
FCLayer.Create(1, 1).Append(
back
))))));
front.SetupInternalState();
front.InitializeWeights(new UniformWeightInitializer(0, 0)); //ConstantWeightInitializer());
var lossFunc = new Quadratic();
var optimizer = new SGD(0.7f);
Matrix x0 = new Matrix(9, 1, MemoryFlags.ReadWrite, false);
x0.Write(new float[] { 0, 0, 0, 0, 0, 0, 0, 0, 0 });
Matrix x1 = new Matrix(9, 1, MemoryFlags.ReadWrite, false);
x1.Write(new float[] { 0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f, 0.7f, 0.8f, 0.9f });
Matrix x2 = new Matrix(9, 1, MemoryFlags.ReadWrite, false);
x2.Write(new float[] { 0, 0, 0, 1, 1, 1, 1, 1, 1 });
Matrix x3 = new Matrix(9, 1, MemoryFlags.ReadWrite, false);
x3.Write(new float[] { 1, 1, 1, 1, 1, 1, 1, 1, 1 });
var loss_vec = new Matrix(1, 1, MemoryFlags.ReadWrite, true);
var x = new Matrix[] { x0, x1, x2, x3 };
var y = new Matrix[]
{
new Matrix(1, 1, MemoryFlags.ReadWrite, true),
new Matrix(1, 1, MemoryFlags.ReadWrite, true),
new Matrix(1, 1, MemoryFlags.ReadWrite, true),
new Matrix(1, 1, MemoryFlags.ReadWrite, true),
};
y[0].Write(new float[] { 0.53f });
y[1].Write(new float[] { 0.77f });
y[2].Write(new float[] { 0.88f });
y[3].Write(new float[] { 1.1f });
float delta = 1e-1f;
float orig_loss_deriv = 0;
float norm_conv = 0.0f, norm_conv_net = 0.0f;
float norm_fc = 0.0f, norm_fc_net = 0.0f;
for (int epoch = 0; epoch < 1; epoch++)
{
for (int idx = 1; idx < x.Length - 2; idx++)
{
{
var output = front.ForwardPropagate(x[idx]);
//Compute loss deriv
loss_vec.Clear();
lossFunc.LossDeriv(output[0], y[idx], loss_vec, 0);
orig_loss_deriv = loss_vec.Memory[0];
back.ComputeGradients(loss_vec);
back.ComputeLayerErrors(loss_vec);
}
{
//Save weights and apply deltas
var conv_l = conv.CurrentLayer as ConvLayer;
for (int f_i = 0; f_i < conv_l.FilterCnt; f_i++)
{
for (int i_i = 0; i_i < conv_l.InputDepth; i_i++)
{
for (int f_y = 0; f_y < conv_l.FilterSz; f_y++)
{
for (int f_x = 0; f_x < conv_l.FilterSz; f_x++)
{
var w_delta = conv_l.WeightErrors[f_i][i_i].Memory[f_y * conv_l.FilterSz + f_x];
conv_l.Weights[f_i][i_i].Memory[f_y * conv_l.FilterSz + f_x] += delta;
var output = front.ForwardPropagate(x[idx]);
loss_vec.Clear();
lossFunc.Loss(output[0], y[idx], loss_vec, 0);
var y1 = loss_vec.Memory[0];
conv_l.Weights[f_i][i_i].Memory[f_y * conv_l.FilterSz + f_x] -= 2 * delta;
output = front.ForwardPropagate(x[idx]);
loss_vec.Clear();
lossFunc.Loss(output[0], y[idx], loss_vec, 0);
var y0 = loss_vec.Memory[0];
conv_l.Weights[f_i][i_i].Memory[f_y * conv_l.FilterSz + f_x] += delta;
var deriv = (y1 - y0) / (2 * delta);
var norm = ((w_delta - deriv) * (w_delta - deriv)) / ((w_delta + deriv) * (w_delta + deriv));
norm_conv += norm;
norm_conv_net++;
}
}
}
}
var fc_l = fc.CurrentLayer as FCLayer;
for (int i = 0; i < fc_l.Weights.Rows * fc_l.Weights.Columns; i++)
{
var w_delta = fc_l.WeightDelta.Memory[i];
fc_l.Weights.Memory[i] += delta;
var output = front.ForwardPropagate(x[idx]);
loss_vec.Clear();
lossFunc.Loss(output[0], y[idx], loss_vec, 0);
var y1 = loss_vec.Memory[0];
fc_l.Weights.Memory[i] -= 2 * delta;
output = front.ForwardPropagate(x[idx]);
loss_vec.Clear();
lossFunc.Loss(output[0], y[idx], loss_vec, 0);
var y0 = loss_vec.Memory[0];
fc_l.Weights.Memory[i] += delta;
var deriv = (y1 - y0) / (2 * delta);
var norm = ((w_delta - deriv) * (w_delta - deriv)) / ((w_delta + deriv) * (w_delta + deriv));
norm_fc += norm;
norm_fc_net++;
}
}
{
back.ResetLayerErrors();
back.ComputeGradients(loss_vec);
back.ComputeLayerErrors(loss_vec);
back.UpdateLayers(optimizer);
}
}
}
Console.WriteLine($"Conv Norm {norm_conv / norm_conv_net}");
Console.WriteLine($"FC Norm {norm_fc / norm_fc_net}");
Console.ReadLine();
}
public void Train()
{
string dir = "GAN_Data";
Directory.CreateDirectory($@"{dir}");
Directory.CreateDirectory($@"{dir}\Results");
Directory.CreateDirectory($@"{dir}\Sources");
Directory.CreateDirectory($@"{dir}\ResultsPRE");
Directory.CreateDirectory($@"{dir}\SourcesPRE");
#region GAN Variables
var sgd_disc = new Adam(0.0002f, 1e-6f);
var sgd_gen = new Adam(0.0002f, 1e-6f);
var sgd_dec = new Adam(0.0002f);
var fake_loss = new NamedLossFunction(NamedLossFunction.GANDiscFake, NamedLossFunction.GANDiscFake);
var real_loss = new NamedLossFunction(NamedLossFunction.GANDiscReal, NamedLossFunction.GANDiscReal);
var gen_loss = new NamedLossFunction(NamedLossFunction.GANGen, NamedLossFunction.GANGen);
var enc_loss = new Quadratic();
NRandom r_dataset = new NRandom(0);
NRandom r_latent = new NRandom(0);
Matrix data_vec = new Matrix(LatentSize, 1, MemoryFlags.ReadOnly, false);
Matrix d_real_loss = new Matrix(1, 1, MemoryFlags.ReadWrite, true);
Matrix d_fake_loss = new Matrix(1, 1, MemoryFlags.ReadWrite, true);
Matrix g_loss = new Matrix(1, 1, MemoryFlags.ReadWrite, true);
Matrix e_loss = new Matrix(OutputSize, 1, MemoryFlags.ReadWrite, true);
Matrix loss_reader = new Matrix(1, 1, MemoryFlags.ReadWrite, true);
float d_real_loss_f = 0;
float d_fake_loss_f = 0;
float g_loss_f = 0;
float d_real_class_f = 0, d_fake_class_f = 0, g_class_f = 0;
Matrix zero = new Matrix(1, 1, MemoryFlags.ReadWrite, true);
zero.Memory[0] = 0;
Matrix one = new Matrix(1, 1, MemoryFlags.ReadWrite, true);
one.Memory[0] = 1;
#endregion
#region Setup Database
AnimeDatasets dataset = new AnimeDatasets(StartSide, /*@"I:\Datasets\anime-faces\combined", @"I:\Datasets\anime-faces\combined_small");*/ @"I:\Datasets\VAE_Dataset\White", @"I:\Datasets\VAE_Dataset\White\conv");
dataset.InitializeDataset();
Matrix[] dataset_vec = new Matrix[dataset.TrainingFiles.Count];
float[][] dataset_f = new float[dataset.TrainingFiles.Count][];
for (int i = 0; i < dataset.TrainingFiles.Count; i++)
{
dataset_f[i] = new float[InputSize];
dataset_vec[i] = new Matrix(InputSize, 1, MemoryFlags.ReadOnly, false);
dataset.LoadImage(dataset.TrainingFiles[i], dataset_f[i]);
dataset_vec[i].Write(dataset_f[i]);
}
#endregion
//Pretrain generator
for (int i0 = 0; i0 < 5000; i0++)
{
int idx = 0;
idx = (r_dataset.Next() % dataset.TrainingFiles.Count);
var latent = encoder.ForwardPropagate(dataset_vec[idx]);
var res = generator.ForwardPropagate(latent);
e_loss.Clear();
enc_loss.LossDeriv(res[0], dataset_vec[idx], e_loss, 0.0f * sgd_dec.L2Val / sgd_dec.Net);
var enc_loss_v = generator_back.ComputeGradients(e_loss);
generator_back.ComputeLayerErrors(e_loss);
encoder_back.ComputeGradients(enc_loss_v);
encoder_back.ComputeLayerErrors(enc_loss_v);
sgd_dec.Update(0);
generator_back.UpdateLayers(sgd_gen);
encoder_back.UpdateLayers(sgd_dec);
if (i0 % BatchSize == 0)
{
dataset.SaveImage($@"{dir}\SourcesPRE\{i0 / BatchSize}.png", dataset_f[idx]);
dataset.SaveImage($@"{dir}\ResultsPRE\{i0 / BatchSize}.png", res[0].Read());
generator.Save($@"{dir}\pretrained_generator_fc.bin");
encoder.Save($@"{dir}\trained_encoder_fc.bin");
}
Console.Clear();
Console.WriteLine($"Iteration: {i0 / BatchSize} Sub-batch: {i0 % BatchSize}");
}
for (int i0 = 000; i0 < 5000 * BatchSize; i0++)
{
int idx = 0;
idx = (r_dataset.Next() % dataset.TrainingFiles.Count);
//Generate the fake data
for (int i1 = 0; i1 < LatentSize; i1++)
{
data_vec.Memory[i1] = (float)r_latent.NextGaussian(0, 1);//LatentSize;
}
var fake_result = generator.ForwardPropagate(data_vec);
if (i0 % BatchSize == 0)
{
dataset.SaveImage($@"{dir}\Sources\{i0 / BatchSize}.png", dataset_f[idx]);
dataset.SaveImage($@"{dir}\Results\{i0 / BatchSize}.png", fake_result[0].Read());
}
Console.Clear();
Console.WriteLine($"Iteration: {i0 / BatchSize} Sub-batch: {i0 % BatchSize}");
Console.WriteLine($"Discriminator Real Loss: {d_real_loss_f}\nDiscriminator Fake Loss: {d_fake_loss_f}\nGenerator Loss: {g_loss_f}\n");
Console.WriteLine($"Discriminator Real Prediction: {d_real_class_f}\nDiscriminator Fake Prediction: {d_fake_class_f}\nGenerator Prediction: {g_class_f}");
d_fake_loss.Clear();
d_real_loss.Clear();
g_loss.Clear();
zero.Memory[0] = (r_latent.Next() % 1000) / 10000f;
one.Memory[0] = 1 - (r_latent.Next() % 1000) / 10000f;
//Discriminator feed forward for real data
{
var d_real_class = discriminator.ForwardPropagate(dataset_vec[idx]);
real_loss.LossDeriv(d_real_class[0], one, d_real_loss, 0.01f * sgd_disc.L2Val / sgd_disc.Net);
var d_real_prop = discriminator_back.ComputeGradients(d_real_loss);
discriminator_back.ComputeLayerErrors(d_real_loss);
d_real_class_f = d_real_class[0].Memory[0];
real_loss.Loss(d_real_class[0], one, loss_reader, 0.01f * sgd_disc.L2Val / sgd_disc.Net);
d_real_loss_f = loss_reader.Memory[0];
loss_reader.Memory[0] = 0;
}
//Discriminator feed forward for fake data
{
var d_fake_class = discriminator.ForwardPropagate(fake_result);
fake_loss.LossDeriv(d_fake_class[0], zero, d_fake_loss, 0.01f * sgd_disc.L2Val / sgd_disc.Net);
var d_fake_prop = discriminator_back.ComputeGradients(d_fake_loss);
discriminator_back.ComputeLayerErrors(d_fake_loss);
d_fake_class_f = d_fake_class[0].Memory[0];
fake_loss.Loss(d_fake_class[0], zero, loss_reader, 0.01f * sgd_disc.L2Val / sgd_disc.Net);
d_fake_loss_f = loss_reader.Memory[0];
loss_reader.Memory[0] = 0;
}
//Update and reset discriminator
sgd_disc.Update(0);
discriminator_back.UpdateLayers(sgd_disc);
discriminator_back.ResetLayerErrors();
//Generate the fake data again
{
for (int i1 = 0; i1 < LatentSize; i1++)
{
data_vec.Memory[i1] = (float)r_latent.NextGaussian(0, 1);//LatentSize;
}
fake_result = generator.ForwardPropagate(data_vec);
var d_gen_class = discriminator.ForwardPropagate(fake_result);
//Compute discriminator crossentropy loss assuming fake is real and propagate
gen_loss.LossDeriv(d_gen_class[0], one, g_loss, 0.01f * sgd_gen.L2Val / sgd_gen.Net);
var d_err = discriminator_back.ComputeGradients(g_loss);
generator_back.ComputeGradients(d_err);
generator_back.ComputeLayerErrors(d_err);
g_class_f = d_gen_class[0].Memory[0];
gen_loss.Loss(d_gen_class[0], one, loss_reader, 0.01f * sgd_gen.L2Val / sgd_gen.Net);
g_loss_f = loss_reader.Memory[0];
loss_reader.Memory[0] = 0;
//Update generator
sgd_gen.Update(0);
generator_back.UpdateLayers(sgd_gen);
generator_back.ResetLayerErrors();
discriminator.ResetLayerErrors();
}
}
discriminator.Save($@"{dir}\network_final.bin");
Console.WriteLine("DONE.");
}
public void Train()
{
string dir = "GAN_Data";
Directory.CreateDirectory($@"{dir}");
Directory.CreateDirectory($@"{dir}\Results");
Directory.CreateDirectory($@"{dir}\Sources");
#region GAN Variables
var sgd_disc = new Adam(0.0002f, 1e-6f);
var sgd_gen = new Adam(0.0002f, 1e-6f);
var fake_loss = new NamedLossFunction(NamedLossFunction.GANDiscFake, NamedLossFunction.GANDiscFake);
var real_loss = new NamedLossFunction(NamedLossFunction.GANDiscReal, NamedLossFunction.GANDiscReal);
var gen_loss = new NamedLossFunction(NamedLossFunction.GANGen, NamedLossFunction.GANGen);
NRandom r_dataset = new NRandom(0);
NRandom r_latent = new NRandom(0);
Matrix data_vec = new Matrix(LatentSize, 1, MemoryFlags.ReadOnly, false);
Matrix d_real_loss = new Matrix(1, 1, MemoryFlags.ReadWrite, true);
Matrix d_fake_loss = new Matrix(1, 1, MemoryFlags.ReadWrite, true);
Matrix g_loss = new Matrix(1, 1, MemoryFlags.ReadWrite, true);
Matrix loss_reader = new Matrix(1, 1, MemoryFlags.ReadWrite, true);
float d_real_loss_f = 0;
float d_fake_loss_f = 0;
float g_loss_f = 0;
float d_real_class_f = 0, d_fake_class_f = 0, g_class_f = 0;
Matrix zero = new Matrix(1, 1, MemoryFlags.ReadWrite, true);
zero.Memory[0] = 0;
Matrix one = new Matrix(1, 1, MemoryFlags.ReadWrite, true);
one.Memory[0] = 1;
#endregion
#region Setup Database
Reader dataset = new Reader();
dataset.InitializeTraining();
Matrix[] imgs = dataset.TrainingImages;
#endregion
for (int i0 = 000; i0 < 1000 * BatchSize; i0++)
{
int idx = 0;
idx = (r_dataset.Next() % imgs.Length);
//Generate the fake data
for (int i1 = 0; i1 < LatentSize; i1++)
{
data_vec.Memory[i1] = (float)r_latent.NextGaussian(0, 1);//LatentSize;
}
var fake_result = generator.ForwardPropagate(data_vec);
if (i0 % BatchSize == 0)
{
SaveImage($@"{dir}\Sources\{i0 / BatchSize}.png", imgs[idx].Read(), 28);
SaveImage($@"{dir}\Results\{i0 / BatchSize}.png", fake_result[0].Read(), 28);
}
Console.Clear();
Console.WriteLine($"Iteration: {i0 / BatchSize} Sub-batch: {i0 % BatchSize}");
Console.WriteLine($"Discriminator Real Loss: {d_real_loss_f}\nDiscriminator Fake Loss: {d_fake_loss_f}\nGenerator Loss: {g_loss_f}\n");
Console.WriteLine($"Discriminator Real Prediction: {d_real_class_f}\nDiscriminator Fake Prediction: {d_fake_class_f}\nGenerator Prediction: {g_class_f}");
d_fake_loss.Clear();
d_real_loss.Clear();
g_loss.Clear();
zero.Memory[0] = (r_latent.Next() % 100) / 1000f;
one.Memory[0] = 1 - (r_latent.Next() % 100) / 1000f;
//Discriminator feed forward for real data
{
var d_real_class = discriminator.ForwardPropagate(imgs[idx]);
real_loss.LossDeriv(d_real_class[0], one, d_real_loss, 0.01f * sgd_disc.L2Val / sgd_disc.Net);
var d_real_prop = discriminator_back.ComputeGradients(d_real_loss);
discriminator_back.ComputeLayerErrors(d_real_loss);
d_real_class_f = d_real_class[0].Memory[0];
real_loss.Loss(d_real_class[0], one, loss_reader, 0.01f * sgd_disc.L2Val);
d_real_loss_f = loss_reader.Memory[0];
loss_reader.Memory[0] = 0;
}
//Discriminator feed forward for fake data
{
var d_fake_class = discriminator.ForwardPropagate(fake_result);
fake_loss.LossDeriv(d_fake_class[0], zero, d_fake_loss, 0.01f * sgd_disc.L2Val / sgd_disc.Net);
var d_fake_prop = discriminator_back.ComputeGradients(d_fake_loss);
discriminator_back.ComputeLayerErrors(d_fake_loss);
d_fake_class_f = d_fake_class[0].Memory[0];
fake_loss.Loss(d_fake_class[0], zero, loss_reader, 0.01f * sgd_disc.L2Val / sgd_disc.Net);
d_fake_loss_f = loss_reader.Memory[0];
loss_reader.Memory[0] = 0;
}
//Update and reset discriminator
sgd_disc.Update(0);
discriminator_back.UpdateLayers(sgd_disc);
discriminator_back.ResetLayerErrors();
//Generate the fake data again
{
for (int i1 = 0; i1 < LatentSize; i1++)
{
data_vec.Memory[i1] = (float)r_latent.NextGaussian(0, 1);//LatentSize;
}
fake_result = generator.ForwardPropagate(data_vec);
var d_gen_class = discriminator.ForwardPropagate(fake_result);
//Compute discriminator crossentropy loss assuming fake is real and propagate
gen_loss.LossDeriv(d_gen_class[0], one, g_loss, 0.01f * sgd_gen.L2Val / sgd_gen.Net);
var d_err = discriminator_back.ComputeGradients(g_loss);
generator_back.ComputeGradients(d_err);
generator_back.ComputeLayerErrors(d_err);
g_class_f = d_gen_class[0].Memory[0];
gen_loss.Loss(d_gen_class[0], one, loss_reader, 0.01f * sgd_gen.L2Val / sgd_gen.Net);
g_loss_f = loss_reader.Memory[0];
loss_reader.Memory[0] = 0;
//Update generator
sgd_gen.Update(0);
generator_back.UpdateLayers(sgd_gen);
generator_back.ResetLayerErrors();
discriminator.ResetLayerErrors();
}
}
discriminator.Save($@"{dir}\network_final.bin");
Console.WriteLine("DONE.");
}