private static void StandardCopyTask(DataStream reportStream)
{
double[] errors = new double[100];
long[] times = new long[100];
for (int i = 0; i < 100; i++)
{
errors[i] = 1;
}
const int seed = 32702;
Console.WriteLine(seed);
//TODO args parsing shit
Random rand = new Random(seed);
const int vectorSize = 8;
const int controllerSize = 100;
const int headsCount = 1;
const int memoryN = 128;
const int memoryM = 20;
const int inputSize = vectorSize + 2;
const int outputSize = vectorSize;
//TODO remove rand
NeuralTuringMachine machine = new NeuralTuringMachine(vectorSize + 2, vectorSize, controllerSize, headsCount, memoryN, memoryM, new RandomWeightInitializer(rand));
//TODO extract weight count calculation
int headUnitSize = Head.GetUnitSize(memoryM);
var weightsCount =
(headsCount * memoryN) +
(memoryN * memoryM) +
(controllerSize * headsCount * memoryM) +
(controllerSize * inputSize) +
(controllerSize) +
(outputSize * (controllerSize + 1)) +
(headsCount * headUnitSize * (controllerSize + 1));
Console.WriteLine(weightsCount);
RMSPropWeightUpdater rmsPropWeightUpdater = new RMSPropWeightUpdater(weightsCount, 0.95, 0.5, 0.001, 0.001);
//NeuralTuringMachine machine = NeuralTuringMachine.Load(@"NTM_0.000583637804331003_2015-04-18T223455");
BPTTTeacher teacher = new BPTTTeacher(machine, rmsPropWeightUpdater);
//for (int i = 1; i < 256; i++)
//{
// var sequence = SequenceGenerator.GenerateSequence(i, vectorSize);
// double[][] machineOutput = teacher.Train(sequence.Item1, sequence.Item2);
// double error = CalculateLoss(sequence.Item2, machineOutput);
// Console.WriteLine("{0},{1}", i, error);
//}
int minSeqLen = 200;
int maxSeqLen = 200;
double savingThreshold = 0.0005;
for (int i = 1; i < 10000000; i++)
{
//var sequence = SequenceGenerator.GenerateSequence(rand.Next(20) + 1, vectorSize);
var sequence = SequenceGenerator.GenerateSequence(rand.Next(minSeqLen, maxSeqLen), vectorSize);
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
double[][] headAddressings;
double[][] machinesOutput = teacher.TrainVerbose(sequence.Item1, sequence.Item2, out headAddressings);
stopwatch.Stop();
times[i % 100] = stopwatch.ElapsedMilliseconds;
double error = CalculateLoss(sequence.Item2, machinesOutput);
double averageError = error / (sequence.Item2.Length * sequence.Item2[0].Length);
errors[i % 100] = error;
if (reportStream != null)
{
reportStream.Set("Iteration", i);
reportStream.Set("Average data loss", averageError);
reportStream.Set("Training time", stopwatch.ElapsedMilliseconds);
reportStream.Set("Sequence length", (sequence.Item1.Length - 2) / 2);
reportStream.Set("Input", sequence.Item1);
reportStream.Set("Known output", sequence.Item2);
reportStream.Set("Real output", machinesOutput);
reportStream.Set("Head addressings", headAddressings);
reportStream.SendData();
}
if (i % 100 == 0)
{
double averageError2 = errors.Average();
Console.WriteLine(
"Iteration: {0}, error: {1}, iterations per second: {2:0.0} MinSeqLen: {3} MaxSeqLen: {4}", i,
averageError2, 1000 / times.Average(), minSeqLen, maxSeqLen);
if (averageError2 < savingThreshold)
{
savingThreshold /= 2;
machine.Save("NTM_" + averageError2 + "_" + DateTime.Now.ToString("s").Replace(":", ""));
maxSeqLen++;
minSeqLen++;
}
}
if (i % 100000 == 0)
{
machine.Save("NTM_" + i + DateTime.Now.ToString("s").Replace(":", ""));
}
}
}
static void Main()
{
DataStream reportStream = null;
try
{
YoVisionClientHelper yoVisionClientHelper = new YoVisionClientHelper();
yoVisionClientHelper.Connect(EndpointType.NetTcp, 8081, "localhost", "YoVisionServer");
reportStream = yoVisionClientHelper.RegisterDataStream("NGram task training",
new Int32DataType("Iteration"),
new DoubleDataType("Average data loss"),
new Double2DArrayType("Input"),
new Double2DArrayType("Known output"),
new Double2DArrayType("Real output"),
new Double2DArrayType("Head addressings"));
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
const int controllerSize = 100;
const int headsCount = 1;
const int memoryN = 128;
const int memoryM = 20;
const int inputSize = 1;
const int outputSize = 1;
Random rand = new Random(42);
NeuralTuringMachine machine = new NeuralTuringMachine(inputSize, outputSize, controllerSize, headsCount, memoryN, memoryM, new RandomWeightInitializer(rand));
int headUnitSize = Head.GetUnitSize(memoryM);
var weightsCount =
(headsCount * memoryN) +
(memoryN * memoryM) +
(controllerSize * headsCount * memoryM) +
(controllerSize * inputSize) +
(controllerSize) +
(outputSize * (controllerSize + 1)) +
(headsCount * headUnitSize * (controllerSize + 1));
Console.WriteLine(weightsCount);
RMSPropWeightUpdater rmsPropWeightUpdater = new RMSPropWeightUpdater(weightsCount, 0.95, 0.5, 0.001, 0.001);
BPTTTeacher teacher = new BPTTTeacher(machine, rmsPropWeightUpdater);
long[] times = new long[100];
for (int i = 1; i < 10000000; i++)
{
Tuple<double[][], double[][]> data = SequenceGenerator.GenerateSequence(SequenceGenerator.GeneratePropabilities());
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
double[][] headAddressings;
double[][] output = teacher.TrainVerbose(data.Item1, data.Item2, out headAddressings);
stopwatch.Stop();
times[i % 100] = stopwatch.ElapsedMilliseconds;
if (i%10 == 0)
{
double loss = CalculateLogLoss(output, data.Item2);
if (reportStream != null)
{
reportStream.Set("Iteration", i);
reportStream.Set("Average data loss", loss);
reportStream.Set("Input", data.Item1);
reportStream.Set("Known output", data.Item2);
reportStream.Set("Real output", output);
reportStream.Set("Head addressings", headAddressings);
reportStream.SendData();
}
}
if (i%100 == 0)
{
Console.WriteLine("Iteration: {0}, iterations per second: {1:0.0}", i, 1000 / times.Average());
}
if (i%1000 == 0)
{
double[] props = SequenceGenerator.GeneratePropabilities();
const int sampleCount = 100;
double[] losses = new double[sampleCount];
for (int j = 0; j < sampleCount; j++)
{
Tuple<double[][], double[][]> sequence = SequenceGenerator.GenerateSequence(props);
var machineOutput = teacher.Train(sequence.Item1, sequence.Item2);
double[][] knownOutput = sequence.Item2;
double loss = CalculateLogLoss(machineOutput, knownOutput);
losses[j] = -loss;
}
Console.WriteLine("Loss [bits per sequence]: {0}", losses.Average());
}
if (i % 1000 == 0)
{
machine.Save("NTM_" + i + DateTime.Now.ToString("s").Replace(":", ""));
}
}
}