private static void ClassicIterations(int controllerSize, int headCount, int memoryN, int memoryM, Random rand, List <double[][]> inputs, List <double[][]> outputs)
{
int weightsCount;
var machine = GetRandomMachine(out weightsCount, controllerSize, headCount, memoryN, memoryM, rand);
RMSPropWeightUpdater rmsPropWeightUpdater = new RMSPropWeightUpdater(weightsCount, 0.95, 0.5, 0.001);
BPTTTeacher teacher = new BPTTTeacher(machine, rmsPropWeightUpdater);
double[][] errors = new double[100][];
long[] times = new long[100];
for (int i = 0; i < 100; i++)
{
errors[i] = new double[4];
for (int j = 0; j < 4; j++)
{
errors[i][j] = 1;
}
}
int count = inputs.Count;
for (int i = 1; i < 10000000; i++)
{
int index = rand.Next(count);
double[][] input = inputs[index];
double[][] output = outputs[index];
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
double[][] machinesOutput = teacher.Train(input, output);
stopwatch.Stop();
times[i % 100] = stopwatch.ElapsedMilliseconds;
double[] error = CalculateLoss(output, machinesOutput);
errors[i % 100][0] = error[0];
errors[i % 100][1] = error[1];
errors[i % 100][2] = error[2];
errors[i % 100][3] = error[3];
double averageError = errors.Average(doubles => doubles.Average());
if (i % 100 == 0)
{
WriteExample(i, times, errors, averageError, output, machinesOutput);
}
if (i % 2500 == 0)
{
string directoryName = string.Format("{0}_{1}_{2}_{3}", controllerSize, headCount, memoryM, memoryN);
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
string filename = string.Format("NTM_{0}_{1}_{2}.ntm", i, DateTime.Now.ToString("s").Replace(":", ""),
averageError);
machine.Save(Path.Combine(directoryName, filename));
}
}
}
private static void Iterate(BPTTTeacher teacher, double[] errors, long[] times, int id)
{
const int vectorSize = 8;
const int minSeqLen = 1;
const int maxSeqLen = 20;
//double savingThreshold = 0.0005;
Random rand = new Random(DateTime.Now.Millisecond);
for (int i = 1; i <= 100; i++)
{
var sequence = SequenceGenerator.GenerateSequence(rand.Next(minSeqLen, maxSeqLen), vectorSize);
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
double[][] machinesOutput = teacher.Train(sequence.Item1, sequence.Item2);
stopwatch.Stop();
times[i % 100] = stopwatch.ElapsedMilliseconds;
double error = CalculateLoss(sequence.Item2, machinesOutput);
errors[i % 100] = error;
if (i % 100 == 0)
{
double averageError2 = errors.Average();
Console.WriteLine("Iteration: {0}, error: {1}, id: {2}", i, averageError2, id);
//if (averageError2 < savingThreshold)
//{
// savingThreshold /= 2;
// machine.Save("NTM_" + averageError2 + "_" + DateTime.Now.ToString("s").Replace(":", ""));
// maxSeqLen++;
// minSeqLen++;
//}
}
//if (i % 100000 == 0)
//{
// machine.Save("NTM_" + i + DateTime.Now.ToString("s").Replace(":", ""));
//}
}
}
public void Run()
{
const int vectorSize = 8;
const int minSeqLen = 1;
const int maxSeqLen = 20;
for (int j = 0; j < Priority; j++)
{
for (int i = 0; i < EpochLength; i++)
{
var sequence = SequenceGenerator.GenerateSequence(_rand.Next(minSeqLen, maxSeqLen), vectorSize);
double[][] machinesOutput = _teacher.Train(sequence.Item1, sequence.Item2);
double error = CalculateLoss(sequence.Item2, machinesOutput);
_longTermAverageErrors.Add(error);
if (_longTermAverageErrors.Count > 1000)
{
_longTermAverageErrors.RemoveAt(0);
}
_iterations++;
}
}
}
public void Run()
{
Console.WriteLine("Running task {0} with priority {1}", ID, Priority);
double[][] lastKnown = null;
double[][] lastComputed = null;
double[] lastErrors = null;
for (int j = 0; j < Priority; j++)
{
for (int i = 0; i < EpochLength; i++)
{
var sequence = _exampleGenerator();
double[][] machinesOutput = _teacher.Train(sequence.Item1, sequence.Item2);
double[] errors = CalculateLoss(sequence.Item2, machinesOutput);
_longTermAverageErrors.Add(errors.Average());
if (_longTermAverageErrors.Count > 1000)
{
_longTermAverageErrors.RemoveAt(0);
}
_iterations++;
if (_iterations % 2500 == 0)
{
if (!Directory.Exists(_directoryName))
{
Directory.CreateDirectory(_directoryName);
}
string filename = string.Format("NTM_{0}_{1}_{2}.ntm", _iterations, DateTime.Now.ToString("s").Replace(":", ""), GetLongTermErrorAverage());
_machine.Save(Path.Combine(_directoryName, filename));
}
lastKnown = sequence.Item2;
lastComputed = machinesOutput;
lastErrors = errors;
}
}
if (lastErrors != null && lastKnown != null && lastComputed != null)
{
WriteExample(lastErrors, lastKnown, lastComputed);
}
}
static void Main()
{
DataStream reportStream = null;
try
{
YoVisionClientHelper yoVisionClientHelper = new YoVisionClientHelper();
yoVisionClientHelper.Connect(EndpointType.NetTcp, 8081, "localhost", "YoVisionServer");
reportStream = yoVisionClientHelper.RegisterDataStream("NGram task training",
new Int32DataType("Iteration"),
new DoubleDataType("Average data loss"),
new Double2DArrayType("Input"),
new Double2DArrayType("Known output"),
new Double2DArrayType("Real output"),
new Double2DArrayType("Head addressings"));
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
const int controllerSize = 100;
const int headsCount = 1;
const int memoryN = 128;
const int memoryM = 20;
const int inputSize = 1;
const int outputSize = 1;
Random rand = new Random(42);
NeuralTuringMachine machine = new NeuralTuringMachine(inputSize, outputSize, controllerSize, headsCount, memoryN, memoryM, new RandomWeightInitializer(rand));
int headUnitSize = Head.GetUnitSize(memoryM);
var weightsCount =
(headsCount * memoryN) +
(memoryN * memoryM) +
(controllerSize * headsCount * memoryM) +
(controllerSize * inputSize) +
(controllerSize) +
(outputSize * (controllerSize + 1)) +
(headsCount * headUnitSize * (controllerSize + 1));
Console.WriteLine(weightsCount);
RMSPropWeightUpdater rmsPropWeightUpdater = new RMSPropWeightUpdater(weightsCount, 0.95, 0.5, 0.001, 0.001);
BPTTTeacher teacher = new BPTTTeacher(machine, rmsPropWeightUpdater);
long[] times = new long[100];
for (int i = 1; i < 10000000; i++)
{
Tuple<double[][], double[][]> data = SequenceGenerator.GenerateSequence(SequenceGenerator.GeneratePropabilities());
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
double[][] headAddressings;
double[][] output = teacher.TrainVerbose(data.Item1, data.Item2, out headAddressings);
stopwatch.Stop();
times[i % 100] = stopwatch.ElapsedMilliseconds;
if (i%10 == 0)
{
double loss = CalculateLogLoss(output, data.Item2);
if (reportStream != null)
{
reportStream.Set("Iteration", i);
reportStream.Set("Average data loss", loss);
reportStream.Set("Input", data.Item1);
reportStream.Set("Known output", data.Item2);
reportStream.Set("Real output", output);
reportStream.Set("Head addressings", headAddressings);
reportStream.SendData();
}
}
if (i%100 == 0)
{
Console.WriteLine("Iteration: {0}, iterations per second: {1:0.0}", i, 1000 / times.Average());
}
if (i%1000 == 0)
{
double[] props = SequenceGenerator.GeneratePropabilities();
const int sampleCount = 100;
double[] losses = new double[sampleCount];
for (int j = 0; j < sampleCount; j++)
{
Tuple<double[][], double[][]> sequence = SequenceGenerator.GenerateSequence(props);
var machineOutput = teacher.Train(sequence.Item1, sequence.Item2);
double[][] knownOutput = sequence.Item2;
double loss = CalculateLogLoss(machineOutput, knownOutput);
losses[j] = -loss;
}
Console.WriteLine("Loss [bits per sequence]: {0}", losses.Average());
}
if (i % 1000 == 0)
{
machine.Save("NTM_" + i + DateTime.Now.ToString("s").Replace(":", ""));
}
}
}