private IDictionary <string, INDArray> FetchAndMergeFromDataset(IComputationHandler handler)
{
Dictionary <string, INDArray> unifiedBlock = new Dictionary <string, INDArray>();
Dictionary <string, IList <INDArray> > allFetchedBlocks = new Dictionary <string, IList <INDArray> >();
int currentBlockIndex = 0;
while (true)
{
IDictionary <string, INDArray> currentBlock = UnderlyingDataset.FetchBlock(currentBlockIndex, handler);
if (currentBlock != null)
{
foreach (string section in currentBlock.Keys)
{
if (!allFetchedBlocks.ContainsKey(section))
{
allFetchedBlocks.Add(section, new List <INDArray>());
}
allFetchedBlocks[section].Add(currentBlock[section]);
}
UnderlyingDataset.FreeBlock(currentBlockIndex, handler);
}
if (!UnderlyingDataset.CanFetchBlocksAfter(currentBlockIndex))
{
break;
}
currentBlockIndex++;
}
if (allFetchedBlocks.Count == 0)
{
throw new InvalidOperationException($"Cannot fetch and merge an empty block list, no blocks could be fetched from the dataset {UnderlyingDataset} for handler {handler}.");
}
foreach (string section in allFetchedBlocks.Keys)
{
unifiedBlock.Add(section, handler.MergeBatch(allFetchedBlocks[section].ToArray()));
}
return(unifiedBlock);
}
public bool CanFetchBlocksAfter(int blockIndex)
{
return(UnderlyingDataset.CanFetchBlocksAfter(blockIndex));
}