private void IndexBalances(ChainBase chain, string checkpointName, Func <uint256, Transaction, uint256, BlockHeader, int, IEnumerable <OrderedBalanceChange> > extract) { SetThrottling(); BlockingCollection <OrderedBalanceChange[]> indexedEntries = new BlockingCollection <OrderedBalanceChange[]>(100); var tasks = CreateTaskPool(indexedEntries, (entries) => Index(entries.Select(e => e.ToEntity()), this.Configuration.GetBalanceTable()), 30); using (IndexerTrace.NewCorrelation("Import balances " + checkpointName + " to azure started").Open()) { this.Configuration.GetBalanceTable().CreateIfNotExists(); var buckets = new MultiValueDictionary <string, OrderedBalanceChange>(); using (var storedBlocks = Enumerate(checkpointName, chain)) { foreach (var block in storedBlocks) { foreach (var tx in block.Block.Transactions) { var txId = tx.GetHash(); try { var entries = extract(txId, tx, block.BlockId, block.Block.Header, block.Height); foreach (var entry in entries) { buckets.Add(entry.PartitionKey, entry); var bucket = buckets[entry.PartitionKey]; if (bucket.Count == 100) { indexedEntries.Add(bucket.ToArray()); buckets.Remove(entry.PartitionKey); } } if (storedBlocks.NeedSave) { foreach (var kv in buckets.AsLookup().ToArray()) { indexedEntries.Add(kv.ToArray()); } buckets.Clear(); tasks.Stop(); storedBlocks.SaveCheckpoint(); tasks.Start(); } } catch (Exception ex) { IndexerTrace.ErrorWhileImportingBalancesToAzure(ex, txId); throw; } } } foreach (var kv in buckets.AsLookup().ToArray()) { indexedEntries.Add(kv.ToArray()); } tasks.Stop(); storedBlocks.SaveCheckpoint(); } } }