public void PrepareTrainingSets(string path, IProgressWriter progressWriter = null)
{
Vocab = File.ReadAllText(path).Distinct().OrderBy(x => x).ToList();
var text = File.ReadAllText(path);
var batcher = new SequenceBatcher <float, char>(Vocab.Count, (ch, i) => Vocab[i] == ch ? 1 : 0);
var matrices = batcher.BatchSamples(text.ToList(), new BatchDimension(BatchDimensionType.BatchSize, _batchSize), progressWriter);
TrainingSet = new SequentialDataSet <float>(matrices);
TestSet = new SequentialDataSet <float>(new List <Matrix <float> >()); // No testing yet.
}
public void Load(string path)
{
using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read))
using (var reader = new BinaryReader(stream, Encoding.UTF8, true))
{
int vocabLen = reader.ReadInt32();
Vocab = reader.ReadChars(vocabLen).ToList();
TrainingSet = SequentialDataSet <float> .Load(stream);
TestSet = SequentialDataSet <float> .Load(stream);
}
}
