public void TestUnifiedIteratorYield()
{
string filename = ".unittestfile" + nameof(TestUnifiedIteratorYield);
CreateCsvTempFile(filename);
SigmaEnvironment.Clear();
FileSource source = new FileSource(filename, Path.GetTempPath());
CsvRecordExtractor extractor = (CsvRecordExtractor) new CsvRecordReader(source).Extractor(new CsvRecordExtractor(new Dictionary <string, int[][]> {
["inputs"] = new[] { new[] { 0 } }
}));
ExtractedDataset dataset = new ExtractedDataset("test", 2, new DiskCacheProvider(Path.GetTempPath() + "/" + nameof(TestUnifiedIteratorYield)), true, extractor);
UnifiedIterator iterator = new UnifiedIterator(dataset);
SigmaEnvironment sigma = SigmaEnvironment.Create("test");
IComputationHandler handler = new CpuFloat32Handler();
foreach (var block in iterator.Yield(handler, sigma))
{
Assert.AreEqual(new[] { 5.1f, 4.9f, 4.7f }, block["inputs"].GetDataAs <float>().GetValuesArrayAs <float>(0, 3));
}
dataset.Dispose();
DeleteTempFile(filename);
}
public void TestUndividedIteratorCreate()
{
string filename = ".unittestfile" + nameof(TestUndividedIteratorCreate);
CreateCsvTempFile(filename);
FileSource source = new FileSource(filename, Path.GetTempPath());
CsvRecordExtractor extractor = (CsvRecordExtractor) new CsvRecordReader(source).Extractor(new CsvRecordExtractor(new Dictionary <string, int[][]> {
["inputs"] = new[] { new[] { 0 } }
}));
ExtractedDataset dataset = new ExtractedDataset("test", 1, new DiskCacheProvider(Path.GetTempPath() + "/" + nameof(TestUndividedIteratorCreate)), true, extractor);
Assert.Throws <ArgumentNullException>(() => new UndividedIterator(null));
dataset.Dispose();
DeleteTempFile(filename);
}
public void TestDatasetBlockwiseSliceFetch()
{
RedirectGlobalsToTempPath();
string filename = nameof(TestDatasetBlockwiseSliceFetch) + "test.dat";
CreateCsvTempFile(filename);
CsvRecordExtractor extractor = new CsvRecordReader(new FileSource(filename)).Extractor("inputs", 0, "targets", 3);
ExtractedDataset dataset = new ExtractedDataset("name", 1, extractor);
DatasetBlockwiseSlice slice = new DatasetBlockwiseSlice(dataset, 1, 2, 3);
Assert.AreEqual(new float[] { 4.9f }, slice.FetchBlock(0, new CpuFloat32Handler())["inputs"].GetDataAs <float>().GetValuesArrayAs <float>(0, 1));
extractor.Reader?.Dispose();
dataset.Dispose();
DeleteTempFile(filename);
}
public void TestUndividedIteratorYield()
{
string filename = ".unittestfile" + nameof(TestUndividedIteratorCreate);
CreateCsvTempFile(filename);
SigmaEnvironment.Clear();
FileSource source = new FileSource(filename, Path.GetTempPath());
CsvRecordExtractor extractor = (CsvRecordExtractor) new CsvRecordReader(source).Extractor(new CsvRecordExtractor(new Dictionary <string, int[][]> {
["inputs"] = new[] { new[] { 0 } }
}));
ExtractedDataset dataset = new ExtractedDataset("test", 2, new DiskCacheProvider(Path.GetTempPath() + "/" + nameof(TestUndividedIteratorCreate)), true, extractor);
UndividedIterator iterator = new UndividedIterator(dataset);
SigmaEnvironment sigma = SigmaEnvironment.Create("test");
IComputationHandler handler = new CpuFloat32Handler();
int index = 0;
foreach (var block in iterator.Yield(handler, sigma))
{
if (index == 0)
{
Assert.AreEqual(new float[] { 5.1f, 4.9f }, block["inputs"].GetDataAs <float>().GetValuesArrayAs <float>(0, 2));
}
else if (index == 1)
{
Assert.AreEqual(new float[] { 4.7f }, block["inputs"].GetDataAs <float>().GetValuesArrayAs <float>(0, 1));
}
else
{
Assert.Fail("There can be a maximum of two iterations, but this is yield iteration 3 (index 2).");
}
index++;
}
dataset.Dispose();
DeleteTempFile(filename);
}
public void TestDatasetFetchBlockSequential()
{
RedirectGlobalsToTempPath();
string filename = $"test{nameof(TestDatasetFetchBlockSequential)}.dat";
CreateCsvTempFile(filename);
CsvRecordExtractor extractor = new CsvRecordReader(new FileSource(filename, Path.GetTempPath())).Extractor("inputs", 1, 2, "targets", 3);
ExtractedDataset dataset = new ExtractedDataset(name: "name", blockSizeRecords: 1, recordExtractors: extractor);
CpuFloat32Handler handler = new CpuFloat32Handler();
IDictionary <string, INDArray> namedArrays = dataset.FetchBlock(0, handler, false);
Assert.AreEqual(new[] { 3.5f, 1.4f }, namedArrays["inputs"].GetDataAs <float>().GetValuesArrayAs <float>(0, 2));
//fetch the same thing twice to check for same block
namedArrays = dataset.FetchBlock(0, handler, false);
Assert.AreEqual(new[] { 3.5f, 1.4f }, namedArrays["inputs"].GetDataAs <float>().GetValuesArrayAs <float>(0, 2));
//skipping second block (index 1)
namedArrays = dataset.FetchBlock(2, handler, false);
Assert.AreEqual(new[] { 3.2f, 1.3f }, namedArrays["inputs"].GetDataAs <float>().GetValuesArrayAs <float>(0, 2));
namedArrays = dataset.FetchBlock(1, handler, false);
Assert.AreEqual(new[] { 3.0f, 1.4f }, namedArrays["inputs"].GetDataAs <float>().GetValuesArrayAs <float>(0, 2));
namedArrays = dataset.FetchBlock(3, handler, false);
Assert.IsNull(namedArrays);
dataset.Dispose();
DeleteTempFile(filename);
}
public void TestDatasetFreeBlockSequential()
{
RedirectGlobalsToTempPath();
string filename = $"test{nameof(TestDatasetFetchBlockSequential)}.dat";
CreateCsvTempFile(filename);
CsvRecordExtractor extractor = new CsvRecordReader(new FileSource(filename, Path.GetTempPath())).Extractor("inputs", 1, 2, "targets", 3);
ExtractedDataset dataset = new ExtractedDataset(name: "name", blockSizeRecords: 1, recordExtractors: extractor);
CpuFloat32Handler handler = new CpuFloat32Handler();
dataset.FetchBlock(0, handler, false);
dataset.FetchBlock(1, handler, false);
dataset.FetchBlock(2, handler, false);
Assert.AreEqual(3, dataset.ActiveBlockRegionCount);
dataset.FreeBlock(1, handler);
dataset.FreeBlock(2, handler);
Assert.AreEqual(1, dataset.ActiveBlockRegionCount);
var namedArrays = dataset.FetchBlock(0, handler, false);
Assert.AreEqual(new[] { 3.5f, 1.4f }, namedArrays["inputs"].GetDataAs <float>().GetValuesArrayAs <float>(0, 2));
namedArrays = dataset.FetchBlock(1, handler, false);
Assert.AreEqual(new[] { 3.0f, 1.4f }, namedArrays["inputs"].GetDataAs <float>().GetValuesArrayAs <float>(0, 2));
namedArrays = dataset.FetchBlock(2, handler, false);
Assert.AreEqual(new[] { 3.2f, 1.3f }, namedArrays["inputs"].GetDataAs <float>().GetValuesArrayAs <float>(0, 2));
dataset.Dispose();
DeleteTempFile(filename);
}
public void TestMinibatchIteratorYield(int minibatchSize)
{
string filename = ".unittestfile" + nameof(TestMinibatchIteratorYield);
CreateCsvTempFile(filename);
SigmaEnvironment.Clear();
FileSource source = new FileSource(filename, Path.GetTempPath());
CsvRecordExtractor extractor = (CsvRecordExtractor) new CsvRecordReader(source).Extractor(new CsvRecordExtractor(new Dictionary <string, int[][]> {
["inputs"] = new[] { new[] { 0 } }
}));
ExtractedDataset dataset = new ExtractedDataset("test", 1, new DiskCacheProvider(Path.GetTempPath() + "/" + nameof(TestMinibatchIteratorYield)), true, extractor);
MinibatchIterator iterator = new MinibatchIterator(minibatchSize, dataset);
IComputationHandler handler = new CpuFloat32Handler();
SigmaEnvironment sigma = SigmaEnvironment.Create("test");
Assert.Throws <ArgumentNullException>(() => iterator.Yield(null, null).GetEnumerator().MoveNext());
Assert.Throws <ArgumentNullException>(() => iterator.Yield(handler, null).GetEnumerator().MoveNext());
Assert.Throws <ArgumentNullException>(() => iterator.Yield(null, sigma).GetEnumerator().MoveNext());
int index = 0;
foreach (var block in iterator.Yield(handler, sigma))
{
//pass through each more than 5 times to ensure consistency
if (index++ > 20)
{
break;
}
Assert.Contains(block["inputs"].GetValue <float>(0, 0, 0), new float[] { 5.1f, 4.9f, 4.7f });
}
dataset.Dispose();
DeleteTempFile(filename);
}
public async Task TestDatasetFetchAsync()
{
RedirectGlobalsToTempPath();
string filename = $"test{nameof(TestDatasetFetchAsync)}.dat";
CreateCsvTempFile(filename);
CsvRecordExtractor extractor = new CsvRecordReader(new FileSource(filename, Path.GetTempPath())).Extractor("inputs", 1, 2, "targets", 3);
ExtractedDataset dataset = new ExtractedDataset(name: "name", blockSizeRecords: 1, recordExtractors: extractor);
CpuFloat32Handler handler = new CpuFloat32Handler();
var block0 = dataset.FetchBlockAsync(0, handler);
var block2 = dataset.FetchBlockAsync(2, handler);
var block1 = dataset.FetchBlockAsync(1, handler);
//mock a free block request to freak out the dataset controller
dataset.FreeBlock(1, handler);
IDictionary <string, INDArray> namedArrays0 = await block0;
IDictionary <string, INDArray> namedArrays1 = await block1;
IDictionary <string, INDArray> namedArrays2 = await block2;
Assert.IsNotNull(namedArrays1);
Assert.AreEqual(new[] { 3.0f, 1.4f }, namedArrays1["inputs"].GetDataAs <float>().GetValuesArrayAs <float>(0, 2));
Assert.IsNotNull(namedArrays2);
Assert.AreEqual(new[] { 3.2f, 1.3f }, namedArrays2["inputs"].GetDataAs <float>().GetValuesArrayAs <float>(0, 2));
Assert.IsNotNull(namedArrays0);
Assert.AreEqual(new[] { 3.5f, 1.4f }, namedArrays0["inputs"].GetDataAs <float>().GetValuesArrayAs <float>(0, 2));
dataset.Dispose();
DeleteTempFile(filename);
}