예제 #1
0
        private void EnqueueTasks(ConcurrentDictionary <Task, Task> tasks, BulkImport <TIndexed> bulk, bool uncompletePartitions, TaskScheduler scheduler)
        {
            if (!uncompletePartitions && !bulk.HasFullPartition)
            {
                return;
            }
            if (uncompletePartitions)
            {
                bulk.FlushUncompletePartitions();
            }

            while (bulk._ReadyPartitions.Count != 0)
            {
                var item = bulk._ReadyPartitions.Dequeue();
                var task = retry.Do(() => IndexCore(item.Item1, item.Item2), scheduler);
                tasks.TryAdd(task, task);
                task.ContinueWith(prev =>
                {
                    _Exception = prev.Exception ?? _Exception;
                    tasks.TryRemove(prev, out prev);
                });
                if (tasks.Count > MaxQueued)
                {
                    WaitFinished(tasks, MaxQueued / 2);
                }
            }
        }
 protected override void ProcessBlock(BlockInfo block, BulkImport <TransactionEntry.Entity> bulk)
 {
     foreach (var transaction in block.Block.Transactions)
     {
         var indexed = new TransactionEntry.Entity(null, transaction, block.BlockId);
         bulk.Add(indexed.PartitionKey, indexed);
     }
 }
        protected override void ProcessBlock(BlockInfo block, BulkImport <OrderedBalanceChange> bulk)
        {
            foreach (var tx in block.Block.Transactions)
            {
                var txId = tx.GetHash();

                var entries = extract(txId, tx, block.BlockId, block.Block.Header, block.Height);
                foreach (var entry in entries)
                {
                    bulk.Add(entry.PartitionKey, entry);
                }
            }
        }
예제 #4
0
 protected override void ProcessBlock(BlockInfo block, BulkImport <BlockInfo> bulk)
 {
     bulk.Add("o", block);
 }
 protected override void ProcessBlock(BlockInfo block, BulkImport <ITableEntity> bulk)
 {
     throw new NotSupportedException();
 }
예제 #6
0
        public void Index(BlockFetcher blockFetcher, TaskScheduler scheduler)
        {
            ConcurrentDictionary <Task, Task> tasks = new ConcurrentDictionary <Task, Task>();

            try
            {
                SetThrottling();
                if (EnsureIsSetup)
                {
                    EnsureSetup().Wait();
                }

                BulkImport <TIndexed> bulk = new BulkImport <TIndexed>(PartitionSize);
                if (!SkipToEnd)
                {
                    try
                    {
                        foreach (var block in blockFetcher)
                        {
                            ThrowIfException();
                            if (blockFetcher.NeedSave)
                            {
                                if (SaveProgression)
                                {
                                    EnqueueTasks(tasks, bulk, true, scheduler);
                                    Save(tasks, blockFetcher, bulk);
                                }
                            }
                            ProcessBlock(block, bulk);
                            if (bulk.HasFullPartition)
                            {
                                EnqueueTasks(tasks, bulk, false, scheduler);
                            }
                        }
                        EnqueueTasks(tasks, bulk, true, scheduler);
                    }
                    catch (OperationCanceledException ex)
                    {
                        if (ex.CancellationToken != blockFetcher.CancellationToken)
                        {
                            throw;
                        }
                    }
                }
                else
                {
                    blockFetcher.SkipToEnd();
                }
                if (SaveProgression)
                {
                    Save(tasks, blockFetcher, bulk);
                }
                WaitFinished(tasks);
                ThrowIfException();
            }
            catch (AggregateException aex)
            {
                ExceptionDispatchInfo.Capture(aex.InnerException).Throw();
                throw;
            }
        }
예제 #7
0
 protected abstract void ProcessBlock(BlockInfo block, BulkImport <TIndexed> bulk);
예제 #8
0
 private void Save(ConcurrentDictionary <Task, Task> tasks, BlockFetcher fetcher, BulkImport <TIndexed> bulk)
 {
     WaitFinished(tasks);
     ThrowIfException();
     fetcher.SaveCheckpoint();
 }