private void EnqueueTasks(ConcurrentDictionary <Task, Task> tasks, BulkImport <TIndexed> bulk, bool uncompletePartitions, TaskScheduler scheduler) { if (!uncompletePartitions && !bulk.HasFullPartition) { return; } if (uncompletePartitions) { bulk.FlushUncompletePartitions(); } while (bulk._ReadyPartitions.Count != 0) { var item = bulk._ReadyPartitions.Dequeue(); var task = retry.Do(() => IndexCore(item.Item1, item.Item2), scheduler); tasks.TryAdd(task, task); task.ContinueWith(prev => { _Exception = prev.Exception ?? _Exception; tasks.TryRemove(prev, out prev); }); if (tasks.Count > MaxQueued) { WaitFinished(tasks, MaxQueued / 2); } } }
protected override void ProcessBlock(BlockInfo block, BulkImport <TransactionEntry.Entity> bulk) { foreach (var transaction in block.Block.Transactions) { var indexed = new TransactionEntry.Entity(null, transaction, block.BlockId); bulk.Add(indexed.PartitionKey, indexed); } }
protected override void ProcessBlock(BlockInfo block, BulkImport <OrderedBalanceChange> bulk) { foreach (var tx in block.Block.Transactions) { var txId = tx.GetHash(); var entries = extract(txId, tx, block.BlockId, block.Block.Header, block.Height); foreach (var entry in entries) { bulk.Add(entry.PartitionKey, entry); } } }
protected override void ProcessBlock(BlockInfo block, BulkImport <BlockInfo> bulk) { bulk.Add("o", block); }
protected override void ProcessBlock(BlockInfo block, BulkImport <ITableEntity> bulk) { throw new NotSupportedException(); }
public void Index(BlockFetcher blockFetcher, TaskScheduler scheduler) { ConcurrentDictionary <Task, Task> tasks = new ConcurrentDictionary <Task, Task>(); try { SetThrottling(); if (EnsureIsSetup) { EnsureSetup().Wait(); } BulkImport <TIndexed> bulk = new BulkImport <TIndexed>(PartitionSize); if (!SkipToEnd) { try { foreach (var block in blockFetcher) { ThrowIfException(); if (blockFetcher.NeedSave) { if (SaveProgression) { EnqueueTasks(tasks, bulk, true, scheduler); Save(tasks, blockFetcher, bulk); } } ProcessBlock(block, bulk); if (bulk.HasFullPartition) { EnqueueTasks(tasks, bulk, false, scheduler); } } EnqueueTasks(tasks, bulk, true, scheduler); } catch (OperationCanceledException ex) { if (ex.CancellationToken != blockFetcher.CancellationToken) { throw; } } } else { blockFetcher.SkipToEnd(); } if (SaveProgression) { Save(tasks, blockFetcher, bulk); } WaitFinished(tasks); ThrowIfException(); } catch (AggregateException aex) { ExceptionDispatchInfo.Capture(aex.InnerException).Throw(); throw; } }
protected abstract void ProcessBlock(BlockInfo block, BulkImport <TIndexed> bulk);
private void Save(ConcurrentDictionary <Task, Task> tasks, BlockFetcher fetcher, BulkImport <TIndexed> bulk) { WaitFinished(tasks); ThrowIfException(); fetcher.SaveCheckpoint(); }