コード例 #1
0
        public override IEnumerable <IDictionary <string, INDArray> > Yield(IComputationHandler handler, SigmaEnvironment environment)
        {
            CheckNotNull(handler, environment);

            int currentIndex = 0;

            // TODO populate registry with relevant parameters
            while (true)
            {
                RequireBlocks(handler, currentIndex);

                if (_fetchedBlocks[currentIndex] == null)
                {
                    break;
                }

                PrepareBlocksAsync(handler, currentIndex + 1);

                var currentBlock = _fetchedBlocks[currentIndex];

                _logger.Debug($"Yielding undivided block at index {currentIndex}.");

                yield return(currentBlock);

                UnderlyingDataset.FreeBlock(currentIndex, handler);
                currentIndex++;
            }
        }
コード例 #2
0
        protected void FreeBlocks(IComputationHandler handler, params int[] indices)
        {
            foreach (int index in indices)
            {
                if (!_fetchedBlocks.ContainsKey(index))
                {
                    continue;
                }

                var block = _fetchedBlocks[index];

                UnderlyingDataset.FreeBlock(index, handler);

                _fetchedBlocks.Remove(index);
            }
        }
コード例 #3
0
        private IDictionary <string, INDArray> FetchAndMergeFromDataset(IComputationHandler handler)
        {
            Dictionary <string, INDArray> unifiedBlock = new Dictionary <string, INDArray>();

            Dictionary <string, IList <INDArray> > allFetchedBlocks = new Dictionary <string, IList <INDArray> >();

            int currentBlockIndex = 0;

            while (true)
            {
                IDictionary <string, INDArray> currentBlock = UnderlyingDataset.FetchBlock(currentBlockIndex, handler);

                if (currentBlock != null)
                {
                    foreach (string section in currentBlock.Keys)
                    {
                        if (!allFetchedBlocks.ContainsKey(section))
                        {
                            allFetchedBlocks.Add(section, new List <INDArray>());
                        }

                        allFetchedBlocks[section].Add(currentBlock[section]);
                    }

                    UnderlyingDataset.FreeBlock(currentBlockIndex, handler);
                }

                if (!UnderlyingDataset.CanFetchBlocksAfter(currentBlockIndex))
                {
                    break;
                }

                currentBlockIndex++;
            }

            if (allFetchedBlocks.Count == 0)
            {
                throw new InvalidOperationException($"Cannot fetch and merge an empty block list, no blocks could be fetched from the dataset {UnderlyingDataset} for handler {handler}.");
            }

            foreach (string section in allFetchedBlocks.Keys)
            {
                unifiedBlock.Add(section, handler.MergeBatch(allFetchedBlocks[section].ToArray()));
            }

            return(unifiedBlock);
        }
コード例 #4
0
 public void FreeBlock(int blockIndex, IComputationHandler handler)
 {
     UnderlyingDataset.FreeBlock(blockIndex, handler);
 }