public override IEnumerable <IDictionary <string, INDArray> > Yield(IComputationHandler handler, SigmaEnvironment environment)
{
CheckNotNull(handler, environment);
int currentIndex = 0;
// TODO populate registry with relevant parameters
while (true)
{
RequireBlocks(handler, currentIndex);
if (_fetchedBlocks[currentIndex] == null)
{
break;
}
PrepareBlocksAsync(handler, currentIndex + 1);
var currentBlock = _fetchedBlocks[currentIndex];
_logger.Debug($"Yielding undivided block at index {currentIndex}.");
yield return(currentBlock);
UnderlyingDataset.FreeBlock(currentIndex, handler);
currentIndex++;
}
}
protected void FreeBlocks(IComputationHandler handler, params int[] indices)
{
foreach (int index in indices)
{
if (!_fetchedBlocks.ContainsKey(index))
{
continue;
}
var block = _fetchedBlocks[index];
UnderlyingDataset.FreeBlock(index, handler);
_fetchedBlocks.Remove(index);
}
}
private IDictionary <string, INDArray> FetchAndMergeFromDataset(IComputationHandler handler)
{
Dictionary <string, INDArray> unifiedBlock = new Dictionary <string, INDArray>();
Dictionary <string, IList <INDArray> > allFetchedBlocks = new Dictionary <string, IList <INDArray> >();
int currentBlockIndex = 0;
while (true)
{
IDictionary <string, INDArray> currentBlock = UnderlyingDataset.FetchBlock(currentBlockIndex, handler);
if (currentBlock != null)
{
foreach (string section in currentBlock.Keys)
{
if (!allFetchedBlocks.ContainsKey(section))
{
allFetchedBlocks.Add(section, new List <INDArray>());
}
allFetchedBlocks[section].Add(currentBlock[section]);
}
UnderlyingDataset.FreeBlock(currentBlockIndex, handler);
}
if (!UnderlyingDataset.CanFetchBlocksAfter(currentBlockIndex))
{
break;
}
currentBlockIndex++;
}
if (allFetchedBlocks.Count == 0)
{
throw new InvalidOperationException($"Cannot fetch and merge an empty block list, no blocks could be fetched from the dataset {UnderlyingDataset} for handler {handler}.");
}
foreach (string section in allFetchedBlocks.Keys)
{
unifiedBlock.Add(section, handler.MergeBatch(allFetchedBlocks[section].ToArray()));
}
return(unifiedBlock);
}
public void FreeBlock(int blockIndex, IComputationHandler handler)
{
UnderlyingDataset.FreeBlock(blockIndex, handler);
}