Exemple #1
0
        private IDictionary <string, INDArray> FetchAndMergeFromDataset(IComputationHandler handler)
        {
            Dictionary <string, INDArray> unifiedBlock = new Dictionary <string, INDArray>();

            Dictionary <string, IList <INDArray> > allFetchedBlocks = new Dictionary <string, IList <INDArray> >();

            int currentBlockIndex = 0;

            while (true)
            {
                IDictionary <string, INDArray> currentBlock = UnderlyingDataset.FetchBlock(currentBlockIndex, handler);

                if (currentBlock != null)
                {
                    foreach (string section in currentBlock.Keys)
                    {
                        if (!allFetchedBlocks.ContainsKey(section))
                        {
                            allFetchedBlocks.Add(section, new List <INDArray>());
                        }

                        allFetchedBlocks[section].Add(currentBlock[section]);
                    }

                    UnderlyingDataset.FreeBlock(currentBlockIndex, handler);
                }

                if (!UnderlyingDataset.CanFetchBlocksAfter(currentBlockIndex))
                {
                    break;
                }

                currentBlockIndex++;
            }

            if (allFetchedBlocks.Count == 0)
            {
                throw new InvalidOperationException($"Cannot fetch and merge an empty block list, no blocks could be fetched from the dataset {UnderlyingDataset} for handler {handler}.");
            }

            foreach (string section in allFetchedBlocks.Keys)
            {
                unifiedBlock.Add(section, handler.MergeBatch(allFetchedBlocks[section].ToArray()));
            }

            return(unifiedBlock);
        }
 public bool CanFetchBlocksAfter(int blockIndex)
 {
     return(UnderlyingDataset.CanFetchBlocksAfter(blockIndex));
 }