public void Train()
{
Trainer trainer = BuildTrainer();
// start the training
Console.WriteLine($"Running {Options.Epochs} epochs with {Vocab.Count} batches per epoch");
Console.WriteLine();
for (int i = 0; i < Options.Epochs; i++)
{
DateTime startTime = DateTime.Now;
IOPair <float[]> sequenceData;
Arguments arguments = new Arguments();
Console.WriteLine($"Running training on epoch {i + 1} of {Options.Epochs}");
for (int j = 0; j < Vocab.Count; j++)
{
sequenceData = GetSequenceData(j);
arguments[Inputs.Input] = Value.CreateSequence(Inputs.Input.Shape, sequenceData.Input, Device);
arguments[Inputs.Output] = Value.CreateSequence(Inputs.Output.Shape, sequenceData.Output, Device);
trainer.TrainMinibatch(arguments, false, Device);
// sample
if (j % Options.SampleFrequency == 0)
{
string sample = Sample(Options.SampleSize);
FileValidator.Validate(sample);
Console.WriteLine(sample);
}
// print current state
if (j % 100 == 0)
{
Console.WriteLine(string.Format("Epoch {0}: Batch [{1}-{2}] Cross Entropy = {3}, Evaluation = {4}",
(i + 1).ToString("000"),
(j + 1).ToString("000000"),
(j + 100).ToString("000000"),
trainer.PreviousMinibatchLossAverage().ToString("F6"),
trainer.PreviousMinibatchEvaluationAverage().ToString("F3")
));
}
}
// log completion
string epochTime = (DateTime.Now - startTime).ToString(@"hh\:mm\:ss\.fff");
Console.WriteLine($"Finished epoch {i + 1}: {epochTime}");
// save the model
string modelFileName = Path.Combine(ModelDirectory, $"{Options.ModelPrefix}_epoch{++Revision}.dnn");
Model.Save(modelFileName);
Console.WriteLine($"Saved model to {modelFileName}");
}
}
public IEnumerable <string> Sample()
{
for (int i = 0; i < Options.SampleCount; i++)
{
string sample = Sample(Options.SampleSize, Options.SamplePrime);
// the sample is filled to the samplesize with garbage so needs some minimal validation
// ' ' is used as the one hot trains the model all filenames must end with it
// these are then further filtered by the supplied prime
string[] results = sample.Split(' ', StringSplitOptions.RemoveEmptyEntries);
foreach (string result in results)
{
if (result.StartsWith(Options.SamplePrime))
{
FileValidator.Validate(result);
yield return(result);
}
}
}
}