public KeyValuePair <double[, , ], double[, , ]> GetRandom()
{
var pair = enumer.GetRandom();
var data = network.GetOutput(pair.Key);
return(new KeyValuePair <double[, , ], double[, , ]>(data, pair.Value));
}
public override double[] TrainBatchContinue(IDataEnumerator data, int batch, int count)
{
double[] errors = new double[count];
for (int c = 0; c < count; c++)
{
int max_found = 0;
double[] loc_errors = new double[batch];
int[] loc_found = new int[batch];
DateTime start = DateTime.Now;
double err = 0;
Parallel.For(0, batch, b =>
{
var pair = data.GetRandom(ref network);
var res = network.ParallelWriteG(b, pair.Key, pair.Value, k);
loc_errors[b] = res.Key;
});
for (int b = 0; b < batch; b++)
{
network.ActionTwoG(b, batch, batch, (v1, v2) => v1 + v2);
err += loc_errors[b];
max_found += loc_found[b];
}
network.ActionG(batch, batch, val => val / batch);
network.GradWeights(batch);
network.ActionG(batch, batch, val => 0);
errors[c] = err / batch;
}
return(errors);
}
public IOPair GetRandom(ref Network network)
{
var pair = enumer.GetRandom(ref network);
var data = this.network.GetOutput(pair.Key);
return(new IOPair(data, pair.Value));
}
public override double[] TrainBatch(IDataEnumerator data, int batch, int count)
{
double[] errors = new double[count];
for (int c = 0; c < count; c++)
{
int max_found = 0;
double max = 0;
double err = 0;
for (int j = 0; j < batch; j++)
{
max = -1;
var pair = data.GetRandom(ref network);
int max_arg = -1;
pair.Value.ForEach((val, z, y, x) => {
if (val > max)
{
max_arg = x;
max = val;
}
});
max = 0;
int max_arg2 = -2;
var res = network.WriteG(0, pair.Key, pair.Value, k);
res.Value.ForEach((val, z, y, x) =>
{
if (val > max)
{
max_arg2 = x;
max = val;
}
});
err += res.Key;
network.ActionTwoG(0, 1, 1, (x, y) => x + y);
if (max_arg == max_arg2)
{
max_found++;
}
}
network.ActionG(1, 1, val => val / batch);
network.GradWeights(1);
network.ActionG(1, 1, x => 0);
errors[c] = err / batch;
Console.WriteLine(err / batch);
}
return(errors);
}
public override double[] TrainBatch(IDataEnumerator data, int batch, int count)
{
double[] errors = new double[count];
for (int c = 0; c < count; c++)
{
double err = 0;
for (int j = 0; j < batch; j++)
{
var pair = data.GetRandom(ref network);
var res = network.WriteG(0, pair.Key, pair.Value, 1);
err += res.Key;
network.ActionTwoG(0, 1, 1, (x, y) => x + y);
}
network.ActionG(1, 1, val => val / batch);
double norm = 0;
int norm_count = 0;
network.ActionG(1, 1, val => {
norm += val * val;
norm_count++;
return(val);
});
norm /= norm_count;
r += norm;
double k = 1e-1 / (d + Math.Sqrt(r));
network.ActionG(1, 1, val => k * val);
network.GradWeights(1);
network.ActionG(1, 1, x => 0);
errors[c] = err / batch;
Console.WriteLine(err / batch);
}
return(errors);
}
public IOPair GetRandom(ref Network network)
{
var pair = enumer.GetRandom(ref network);
return(new IOPair(pair.Key, pair.Key));
}
public override double[] TrainBatch(IDataEnumerator data, int batch, int count)
{
double[] errors = new double[count];
network.CreateGradients(batch + 2);
for (int c = 0; c < count; c++)
{
int max_found = 0;
double[] loc_errors = new double[batch];
int[] loc_found = new int[batch];
DateTime start = DateTime.Now;
double err = 0;
//for(int b = 0;b < batch;b++)
Parallel.For(0, batch, b =>
{
var pair = data.GetRandom(ref network);
var res = network.ParallelWriteG(b, pair.Key, pair.Value, k, 0);
loc_errors[b] = res.Key;
if (need_max)
{
double max = -1;
int max_arg = -1;
pair.Value.ForEach((val, z, y, x) =>
{
if (val > max)
{
max_arg = x;
max = val;
}
});
max = -1;
int max_arg2 = -2;
res.Value.ForEach((val, z, y, x) =>
{
if (val > max)
{
max_arg2 = x;
max = val;
}
});
if (max_arg == max_arg2)
{
loc_found[b] = 1;
}
}
});
for (int b = 0; b < batch; b++)
{
network.ActionTwoG(b, batch, batch, (v1, v2) => v1 + v2);
err += loc_errors[b];
max_found += loc_found[b];
}
network.ActionG(batch, batch, val => val / batch);
if (firstSet)
{
network.ActionTwoG(batch, batch + 1, batch + 1, (g, v) => g);
firstSet = false;
}
else
{
network.ActionTwoG(batch, batch + 1, batch + 1, (g, v) => a * v + g);
}
network.GradWeights(batch + 1);
network.ActionG(batch, batch, val => 0);
errors[c] = err / batch;
double dw = 0;
int w_count = 0;
//network.ActionG(batch + 1, batch + 1, x =>
//{
// dw += x * x;
// w_count++;
// return x;
//});
//Console.WriteLine();
//Console.WriteLine("{2}: {0} : {1}", err / batch, Math.Sqrt(dw / w_count), c);
//Console.WriteLine("A: {0} ", (float)(DateTime.Now - start).TotalSeconds);
//if(need_max)
// Console.WriteLine("Found {0} of {1} {2}%", max_found, batch, 100.0f * max_found / batch);
//if (c % 10 == 0)
// File.WriteAllText(c + "_" + err / batch, network.SaveJSON());
if (err / batch <= 1e-7)
{
acc++;
if (acc > 10)
{
Console.WriteLine("Stopped at {0}", c);
return(errors);
}
}
else
{
acc = 0;
}
//Console.WriteLine("Norm: {0}", network.layers[0].NormG(2));
}
return(errors);
}
public List <KeyValuePair <double, double> > TrainBatchPercent(IDataEnumerator data, int batch, int count)
{
List <KeyValuePair <double, double> > pairs = new List <KeyValuePair <double, double> >();
network.CreateGradients(batch + 2);
for (int c = 0; c < count; c++)
{
int max_found = 0;
double[] loc_errors = new double[batch];
int[] loc_found = new int[batch];
DateTime start = DateTime.Now;
double err = 0;
Parallel.For(0, batch, b =>
{
var pair = data.GetRandom(ref network);
var res = network.ParallelWriteG(b, pair.Key, pair.Value, k);
loc_errors[b] = res.Key;
double max = -1;
int max_arg = -1;
pair.Value.ForEach((val, z, y, x) =>
{
if (val > max)
{
max_arg = x;
max = val;
}
});
max = -1;
int max_arg2 = -2;
res.Value.ForEach((val, z, y, x) =>
{
if (val > max)
{
max_arg2 = x;
max = val;
}
});
if (max_arg == max_arg2)
{
loc_found[b] = 1;
}
});
for (int b = 0; b < batch; b++)
{
network.ActionTwoG(b, batch, batch, (v1, v2) => v1 + v2);
err += loc_errors[b];
max_found += loc_found[b];
}
network.ActionG(batch, batch, val => val / batch);
network.ActionTwoG(batch, batch + 1, batch + 1, (g, v) => a * v + g);
network.GradWeights(batch + 1);
network.ActionG(batch, batch, val => 0);
pairs.Add(new KeyValuePair <double, double>(err / batch, 100.0f * max_found / batch));
//double dw = 0;
//int w_count = 0;
//network.ActionG(batch + 1, batch + 1, x =>
//{
// dw += x * x;
// w_count++;
// return x;
//});
//Console.WriteLine();
//Console.WriteLine("{2}: {0} : {1}", err / batch, Math.Sqrt(dw / w_count), c);
//Console.WriteLine("A: {0} ", (float)(DateTime.Now - start).TotalSeconds);
//Console.WriteLine("Found {0} of {1} {2}%", max_found, batch, 100.0f * max_found / batch);
//Console.WriteLine("Norm: {0}", network.layers[0].NormG(2));
}
return(pairs);
}
public override double[] TrainBatch(IDataEnumerator data, int batch, int count)
{
double[] errors = new double[count];
for (int c = 0; c < count; c++)
{
DateTime start = DateTime.Now;
double err = 0;
for (int b = 0; b < batch; b++)
{
var pair = data.GetRandom(ref network);
var res = network.WriteG(0, pair.Key, pair.Value, 1);
err += res.Key;
network.ActionTwoG(0, 1, 1, (v1, v2) => v1 + v2);
}
double norm = 0;
int norm_count = 0;
network.ActionG(1, 1, val => {
norm += val * val;
norm_count++;
return(val);
});
norm = norm / norm_count;
r = r * gamma + (1 - gamma) * norm;
double k2 = k / (Math.Sqrt(r) + d) / batch;
network.ActionG(1, 1, val => val * k2);
network.ActionTwoG(1, 2, 2, (g, v) => a * v + g);
network.GradWeights(2);
network.ActionG(1, 1, val => 0);
errors[c] = err / batch;
Console.WriteLine("{0} : {1}", err / batch, Math.Sqrt(norm));
//Console.WriteLine("A: {0}", (DateTime.Now - start).TotalSeconds);
if (err / batch <= 1e-7)
{
acc++;
if (acc > 10)
{
Console.WriteLine("Stopped at {0}", c);
return(errors);
}
}
else
{
acc = 0;
}
//Console.WriteLine("Norm: {0}", network.layers[0].NormG(2));
}
return(errors);
}
public override double[] TrainBatch(IDataEnumerator data, int batch, int count)
{
double[] errors = new double[count];
network.CreateGradients(batch + 2);
for (int c = 0; c < count; c++)
{
int max_found = 0;
double[] loc_errors = new double[batch];
int[] loc_found = new int[batch];
DateTime start = DateTime.Now;
double err = 0;
//Parallel.For(0, batch, b =>
for (int b = 0; b < batch; b++)
{
var pair = data.GetRandom();
var res = network.ParallelWriteG(b, pair.Key, pair.Value, 1);
loc_errors[b] = res.Key;
double max = -1;
int max_arg = -1;
pair.Value.ForEach((val, z, y, x) =>
{
if (val > max)
{
max_arg = x;
max = val;
}
});
max = -1;
int max_arg2 = -2;
res.Value.ForEach((val, z, y, x) =>
{
if (val > max)
{
max_arg2 = x;
max = val;
}
});
if (max_arg == max_arg2)
{
loc_found[b] = 1;
}
}//);
for (int b = 0; b < batch; b++)
{
network.ActionTwoG(b, batch, batch, (v1, v2) => v1 + v2);
err += loc_errors[b];
max_found += loc_found[b];
}
network.ActionG(batch, batch, val => val / batch);
double dw = 0;
int w_count = 0;
network.ActionG(batch, batch, x =>
{
dw += x * x;
w_count++;
return(x);
});
dw /= w_count;
dw *= 1000;
k = ro * k + (1 - ro) * e / Math.Sqrt(1 + dw);
network.ActionTwoG(batch, batch + 1, batch + 1, (g, v) => a * v + g * k);
network.GradWeights(batch + 1);
network.ActionG(batch, batch, val => 0);
errors[c] = err / batch;
Console.WriteLine();
Console.WriteLine("{2}: {0} : {1}", err / batch, k, c);
Console.WriteLine("A: {0} ", (float)(DateTime.Now - start).TotalSeconds);
Console.WriteLine("Found {0} of {1} {2}%", max_found, batch, 100.0f * max_found / batch);
//Console.WriteLine("Norm: {0}", network.layers[0].NormG(2));
}
return(errors);
}
public KeyValuePair <double[, , ], double[, , ]> GetRandom()
{
var pair = enumer.GetRandom();
return(new KeyValuePair <double[, , ], double[, , ]>(pair.Key, pair.Key));
}
public override double[] TrainBatch(IDataEnumerator data, int batch, int count)
{
double[] errors = new double[count];
for (int c = 0; c < count; c++)
{
int max_found = 0;
double max = 0;
var g_write = new TimeSpan();
var a_time = new TimeSpan();
DateTime start;
double err = 0;
for (int b = 0; b < batch; b++)
{
var pair = data.GetRandom(ref network);
start = DateTime.Now;
var res = network.ParallelWriteG(0, pair.Key, pair.Value, k);
g_write += (DateTime.Now - start);
err += res.Key;
start = DateTime.Now;
network.ActionTwoG(0, 1, 1, (v1, v2) => v1 + v2);
a_time += (DateTime.Now - start);
max = -1;
int max_arg = -1;
pair.Value.ForEach((val, z, y, x) => {
if (val > max)
{
max_arg = x;
max = val;
}
});
max = -1;
int max_arg2 = -2;
res.Value.ForEach((val, z, y, x) =>
{
if (val > max)
{
max_arg2 = x;
max = val;
}
});
if (max_arg == max_arg2)
{
max_found++;
}
}
start = DateTime.Now;
network.ActionG(1, 1, val => val / batch);
network.ActionTwoG(1, 2, 2, (g, v) => a * v + g * (1 - a));
a_time += (DateTime.Now - start);
network.GradWeights(2);
start = DateTime.Now;
network.ActionG(1, 1, val => 0);
a_time += (DateTime.Now - start);
errors[c] = err / batch;
double dw = 0;
int w_count = 0;
network.ActionG(2, 2, x =>
{
dw += x * x;
w_count++;
return(x);
});
Console.WriteLine();
Console.WriteLine("{2}: {0} : {1}", err / batch, Math.Sqrt(dw / w_count), c);
Console.WriteLine("A: {0} G:{1}", a_time.TotalSeconds, g_write.TotalSeconds);
Console.WriteLine("Found {0} of {1} {2}%", max_found, batch, 100.0f * max_found / batch);
if (err / batch <= 1e-7)
{
acc++;
if (acc > 10)
{
Console.WriteLine("Stopped at {0}", c);
return(errors);
}
}
else
{
acc = 0;
}
//Console.WriteLine("Norm: {0}", network.layers[0].NormG(2));
}
return(errors);
}