public void PrepareTrainingSets(string path, IProgressWriter progressWriter = null) { Vocab = File.ReadAllText(path).Distinct().OrderBy(x => x).ToList(); var text = File.ReadAllText(path); var batcher = new SequenceBatcher <float, char>(Vocab.Count, (ch, i) => Vocab[i] == ch ? 1 : 0); var matrices = batcher.BatchSamples(text.ToList(), new BatchDimension(BatchDimensionType.BatchSize, _batchSize), progressWriter); TrainingSet = new SequentialDataSet <float>(matrices); TestSet = new SequentialDataSet <float>(new List <Matrix <float> >()); // No testing yet. }
public void Load(string path) { using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read)) using (var reader = new BinaryReader(stream, Encoding.UTF8, true)) { int vocabLen = reader.ReadInt32(); Vocab = reader.ReadChars(vocabLen).ToList(); TrainingSet = SequentialDataSet <float> .Load(stream); TestSet = SequentialDataSet <float> .Load(stream); } }