private void EnqueueTasks(ConcurrentDictionary <Task, Task> tasks, BulkImport <TIndexed> bulk, bool uncompletePartitions, TaskScheduler scheduler)
{
if (!uncompletePartitions && !bulk.HasFullPartition)
{
return;
}
if (uncompletePartitions)
{
bulk.FlushUncompletePartitions();
}
while (bulk._ReadyPartitions.Count != 0)
{
var item = bulk._ReadyPartitions.Dequeue();
var task = retry.Do(() => IndexCore(item.Item1, item.Item2), scheduler);
tasks.TryAdd(task, task);
task.ContinueWith(prev =>
{
_Exception = prev.Exception ?? _Exception;
tasks.TryRemove(prev, out prev);
});
if (tasks.Count > MaxQueued)
{
WaitFinished(tasks, MaxQueued / 2);
}
}
}
protected override void ProcessBlock(BlockInfo block, BulkImport <TransactionEntry.Entity> bulk)
{
foreach (var transaction in block.Block.Transactions)
{
var indexed = new TransactionEntry.Entity(null, transaction, block.BlockId);
bulk.Add(indexed.PartitionKey, indexed);
}
}
protected override void ProcessBlock(BlockInfo block, BulkImport <OrderedBalanceChange> bulk)
{
foreach (var tx in block.Block.Transactions)
{
var txId = tx.GetHash();
var entries = extract(txId, tx, block.BlockId, block.Block.Header, block.Height);
foreach (var entry in entries)
{
bulk.Add(entry.PartitionKey, entry);
}
}
}
protected override void ProcessBlock(BlockInfo block, BulkImport <BlockInfo> bulk)
{
bulk.Add("o", block);
}
protected override void ProcessBlock(BlockInfo block, BulkImport <ITableEntity> bulk)
{
throw new NotSupportedException();
}
public void Index(BlockFetcher blockFetcher, TaskScheduler scheduler)
{
ConcurrentDictionary <Task, Task> tasks = new ConcurrentDictionary <Task, Task>();
try
{
SetThrottling();
if (EnsureIsSetup)
{
EnsureSetup().Wait();
}
BulkImport <TIndexed> bulk = new BulkImport <TIndexed>(PartitionSize);
if (!SkipToEnd)
{
try
{
foreach (var block in blockFetcher)
{
ThrowIfException();
if (blockFetcher.NeedSave)
{
if (SaveProgression)
{
EnqueueTasks(tasks, bulk, true, scheduler);
Save(tasks, blockFetcher, bulk);
}
}
ProcessBlock(block, bulk);
if (bulk.HasFullPartition)
{
EnqueueTasks(tasks, bulk, false, scheduler);
}
}
EnqueueTasks(tasks, bulk, true, scheduler);
}
catch (OperationCanceledException ex)
{
if (ex.CancellationToken != blockFetcher.CancellationToken)
{
throw;
}
}
}
else
{
blockFetcher.SkipToEnd();
}
if (SaveProgression)
{
Save(tasks, blockFetcher, bulk);
}
WaitFinished(tasks);
ThrowIfException();
}
catch (AggregateException aex)
{
ExceptionDispatchInfo.Capture(aex.InnerException).Throw();
throw;
}
}
protected abstract void ProcessBlock(BlockInfo block, BulkImport <TIndexed> bulk);
private void Save(ConcurrentDictionary <Task, Task> tasks, BlockFetcher fetcher, BulkImport <TIndexed> bulk)
{
WaitFinished(tasks);
ThrowIfException();
fetcher.SaveCheckpoint();
}