public void TestUnifiedIteratorYield() { string filename = ".unittestfile" + nameof(TestUnifiedIteratorYield); CreateCsvTempFile(filename); SigmaEnvironment.Clear(); FileSource source = new FileSource(filename, Path.GetTempPath()); CsvRecordExtractor extractor = (CsvRecordExtractor) new CsvRecordReader(source).Extractor(new CsvRecordExtractor(new Dictionary <string, int[][]> { ["inputs"] = new[] { new[] { 0 } } })); ExtractedDataset dataset = new ExtractedDataset("test", 2, new DiskCacheProvider(Path.GetTempPath() + "/" + nameof(TestUnifiedIteratorYield)), true, extractor); UnifiedIterator iterator = new UnifiedIterator(dataset); SigmaEnvironment sigma = SigmaEnvironment.Create("test"); IComputationHandler handler = new CpuFloat32Handler(); foreach (var block in iterator.Yield(handler, sigma)) { Assert.AreEqual(new[] { 5.1f, 4.9f, 4.7f }, block["inputs"].GetDataAs <float>().GetValuesArrayAs <float>(0, 3)); } dataset.Dispose(); DeleteTempFile(filename); }
public void TestUndividedIteratorCreate() { string filename = ".unittestfile" + nameof(TestUndividedIteratorCreate); CreateCsvTempFile(filename); FileSource source = new FileSource(filename, Path.GetTempPath()); CsvRecordExtractor extractor = (CsvRecordExtractor) new CsvRecordReader(source).Extractor(new CsvRecordExtractor(new Dictionary <string, int[][]> { ["inputs"] = new[] { new[] { 0 } } })); ExtractedDataset dataset = new ExtractedDataset("test", 1, new DiskCacheProvider(Path.GetTempPath() + "/" + nameof(TestUndividedIteratorCreate)), true, extractor); Assert.Throws <ArgumentNullException>(() => new UndividedIterator(null)); dataset.Dispose(); DeleteTempFile(filename); }
public void TestCsvRecordExtractorCreate() { string filename = ".unittestfile" + nameof(TestCsvRecordExtractorCreate); CreateCsvTempFile(filename); FileSource source = new FileSource(filename, Path.GetTempPath()); CsvRecordExtractor extractor = (CsvRecordExtractor) new CsvRecordReader(source).Extractor(new CsvRecordExtractor(new Dictionary <string, int[][]> { ["inputs"] = new[] { new[] { 0 } } })); Assert.AreSame(source, extractor.Reader.Source); Assert.AreEqual(new[] { "inputs" }, extractor.SectionNames); source.Dispose(); DeleteTempFile(filename); }
public void TestUndividedIteratorYield() { string filename = ".unittestfile" + nameof(TestUndividedIteratorCreate); CreateCsvTempFile(filename); SigmaEnvironment.Clear(); FileSource source = new FileSource(filename, Path.GetTempPath()); CsvRecordExtractor extractor = (CsvRecordExtractor) new CsvRecordReader(source).Extractor(new CsvRecordExtractor(new Dictionary <string, int[][]> { ["inputs"] = new[] { new[] { 0 } } })); ExtractedDataset dataset = new ExtractedDataset("test", 2, new DiskCacheProvider(Path.GetTempPath() + "/" + nameof(TestUndividedIteratorCreate)), true, extractor); UndividedIterator iterator = new UndividedIterator(dataset); SigmaEnvironment sigma = SigmaEnvironment.Create("test"); IComputationHandler handler = new CpuFloat32Handler(); int index = 0; foreach (var block in iterator.Yield(handler, sigma)) { if (index == 0) { Assert.AreEqual(new float[] { 5.1f, 4.9f }, block["inputs"].GetDataAs <float>().GetValuesArrayAs <float>(0, 2)); } else if (index == 1) { Assert.AreEqual(new float[] { 4.7f }, block["inputs"].GetDataAs <float>().GetValuesArrayAs <float>(0, 1)); } else { Assert.Fail("There can be a maximum of two iterations, but this is yield iteration 3 (index 2)."); } index++; } dataset.Dispose(); DeleteTempFile(filename); }
public void TestMinibatchIteratorYield(int minibatchSize) { string filename = ".unittestfile" + nameof(TestMinibatchIteratorYield); CreateCsvTempFile(filename); SigmaEnvironment.Clear(); FileSource source = new FileSource(filename, Path.GetTempPath()); CsvRecordExtractor extractor = (CsvRecordExtractor) new CsvRecordReader(source).Extractor(new CsvRecordExtractor(new Dictionary <string, int[][]> { ["inputs"] = new[] { new[] { 0 } } })); ExtractedDataset dataset = new ExtractedDataset("test", 1, new DiskCacheProvider(Path.GetTempPath() + "/" + nameof(TestMinibatchIteratorYield)), true, extractor); MinibatchIterator iterator = new MinibatchIterator(minibatchSize, dataset); IComputationHandler handler = new CpuFloat32Handler(); SigmaEnvironment sigma = SigmaEnvironment.Create("test"); Assert.Throws <ArgumentNullException>(() => iterator.Yield(null, null).GetEnumerator().MoveNext()); Assert.Throws <ArgumentNullException>(() => iterator.Yield(handler, null).GetEnumerator().MoveNext()); Assert.Throws <ArgumentNullException>(() => iterator.Yield(null, sigma).GetEnumerator().MoveNext()); int index = 0; foreach (var block in iterator.Yield(handler, sigma)) { //pass through each more than 5 times to ensure consistency if (index++ > 20) { break; } Assert.Contains(block["inputs"].GetValue <float>(0, 0, 0), new float[] { 5.1f, 4.9f, 4.7f }); } dataset.Dispose(); DeleteTempFile(filename); }
public void TestCsvRecordExtractorExtract() { string filename = ".unittestfile" + nameof(TestCsvRecordExtractorCreate); CreateCsvTempFile(filename); FileSource source = new FileSource(filename, Path.GetTempPath()); CsvRecordExtractor extractor = (CsvRecordExtractor) new CsvRecordReader(source).Extractor(new CsvRecordExtractor(new Dictionary <string, IList <int> > { ["inputs"] = new[] { 0, 1, 2 } })); Assert.Throws <InvalidOperationException>(() => extractor.ExtractDirect(3, new CpuFloat32Handler())); extractor.Prepare(); Assert.Throws <ArgumentNullException>(() => extractor.ExtractDirect(3, null)); Assert.Throws <ArgumentException>(() => extractor.ExtractDirect(-1, new CpuFloat32Handler())); Assert.AreEqual(5, extractor.ExtractDirect(3, new CpuFloat32Handler())["inputs"].GetValue <int>(0, 0, 0)); source.Dispose(); DeleteTempFile(filename); }
public CsvRecordExtractor Extractor(params object[] parameters) { return(Extractor(columnMappings: CsvRecordExtractor.ParseExtractorParameters(parameters))); }