public long IndexBlocks(ChainBase chain = null)
{
long blkCount = 0;
SetThrottling();
BlockingCollection <Block> blocks = new BlockingCollection <Block>(20);
var tasks = CreateTaskPool(blocks, Index, 15);
using (IndexerTrace.NewCorrelation("Import blocks to azure started").Open())
{
Configuration.GetBlocksContainer().CreateIfNotExists();
using (var storedBlocks = Enumerate("blocks", chain))
{
foreach (var block in storedBlocks)
{
blkCount++;
blocks.Add(block.Block);
if (storedBlocks.NeedSave)
{
tasks.Stop();
storedBlocks.SaveCheckpoint();
tasks.Start();
}
}
tasks.Stop();
storedBlocks.SaveCheckpoint();
}
}
return(blkCount);
}
public long IndexTransactions(ChainBase chain = null)
{
long txCount = 0;
SetThrottling();
BlockingCollection <TransactionEntry.Entity[]> transactions = new BlockingCollection <TransactionEntry.Entity[]>(20);
var tasks = CreateTaskPool(transactions, (txs) => Index(txs), 30);
using (IndexerTrace.NewCorrelation("Import transactions to azure started").Open())
{
Configuration.GetTransactionTable().CreateIfNotExists();
var buckets = new MultiValueDictionary <string, TransactionEntry.Entity>();
using (var storedBlocks = Enumerate("transactions", chain))
{
foreach (var block in storedBlocks)
{
foreach (var transaction in block.Block.Transactions)
{
txCount++;
var indexed = new TransactionEntry.Entity(null, transaction, block.BlockId);
buckets.Add(indexed.PartitionKey, indexed);
var collection = buckets[indexed.PartitionKey];
if (collection.Count == 100)
{
PushTransactions(buckets, collection, transactions);
}
if (storedBlocks.NeedSave)
{
foreach (var kv in buckets.AsLookup().ToArray())
{
PushTransactions(buckets, kv, transactions);
}
tasks.Stop();
storedBlocks.SaveCheckpoint();
tasks.Start();
}
}
}
foreach (var kv in buckets.AsLookup().ToArray())
{
PushTransactions(buckets, kv, transactions);
}
tasks.Stop();
storedBlocks.SaveCheckpoint();
}
}
return(txCount);
}
public int IndexWalletBalances(ChainBase chain)
{
using (IndexerTrace.NewCorrelation("Import wallet balances to azure started"))
{
using (var node = Configuration.ConnectToNode(false))
{
node.VersionHandshake();
var task = new IndexBalanceTask(Configuration, Configuration.CreateIndexerClient().GetAllWalletRules());
task.SaveProgression = !IgnoreCheckpoints;
task.Index(GetBlockFetcher(GetCheckpointInternal(IndexerCheckpoints.Wallets), node, chain), TaskScheduler);
return(task.IndexedEntities);
}
}
}
public long IndexBlocks(ChainBase chain = null)
{
using (IndexerTrace.NewCorrelation("Import blocks to azure started"))
{
using (var node = Configuration.ConnectToNode(false))
{
node.VersionHandshake();
var task = new IndexBlocksTask(Configuration);
task.SaveProgression = !IgnoreCheckpoints;
task.Index(GetBlockFetcher(GetCheckpointInternal(IndexerCheckpoints.Blocks), node, chain), TaskScheduler);
return(task.IndexedBlocks);
}
}
}
public int IndexOrderedBalances(ChainBase chain)
{
using (IndexerTrace.NewCorrelation("Import balances to azure started").Open())
{
using (var node = Configuration.ConnectToNode(false))
{
node.VersionHandshake();
var task = new IndexBalanceTask(Configuration, null);
task.SaveProgression = !IgnoreCheckpoints;
task.Index(GetBlockFetcher(GetCheckpointInternal(IndexerCheckpoints.Balances), node, chain), TaskScheduler);
return(task.IndexedEntities);
}
}
}
public void IndexChain(ChainBase chain)
{
if (chain == null)
{
throw new ArgumentNullException("chain");
}
SetThrottling();
using (IndexerTrace.NewCorrelation("Index main chain to azure started"))
{
Configuration.GetChainTable().CreateIfNotExistsAsync().GetAwaiter().GetResult();
IndexerTrace.InputChainTip(chain.Tip);
var client = Configuration.CreateIndexerClient();
var changes = client.GetChainChangesUntilFork(chain.Tip, true).ToList();
var height = 0;
if (changes.Count != 0)
{
IndexerTrace.IndexedChainTip(changes[0].BlockId, changes[0].Height);
if (changes[0].Height > chain.Tip.Height)
{
IndexerTrace.InputChainIsLate();
return;
}
height = changes[changes.Count - 1].Height + 1;
if (height > chain.Height)
{
IndexerTrace.IndexedChainIsUpToDate(chain.Tip);
return;
}
}
else
{
IndexerTrace.NoForkFoundWithStored();
}
IndexerTrace.IndexingChain(chain.GetBlock(height), chain.Tip);
Index(chain, height);
}
}
private void IndexBalances(ChainBase chain, string checkpointName, Func <uint256, Transaction, uint256, BlockHeader, int, IEnumerable <OrderedBalanceChange> > extract)
{
SetThrottling();
BlockingCollection <OrderedBalanceChange[]> indexedEntries = new BlockingCollection <OrderedBalanceChange[]>(100);
var tasks = CreateTaskPool(indexedEntries, (entries) => Index(entries.Select(e => e.ToEntity()), this.Configuration.GetBalanceTable()), 30);
using (IndexerTrace.NewCorrelation("Import balances " + checkpointName + " to azure started").Open())
{
this.Configuration.GetBalanceTable().CreateIfNotExists();
var buckets = new MultiValueDictionary <string, OrderedBalanceChange>();
using (var storedBlocks = Enumerate(checkpointName, chain))
{
foreach (var block in storedBlocks)
{
foreach (var tx in block.Block.Transactions)
{
var txId = tx.GetHash();
try
{
var entries = extract(txId, tx, block.BlockId, block.Block.Header, block.Height);
foreach (var entry in entries)
{
buckets.Add(entry.PartitionKey, entry);
var bucket = buckets[entry.PartitionKey];
if (bucket.Count == 100)
{
indexedEntries.Add(bucket.ToArray());
buckets.Remove(entry.PartitionKey);
}
}
if (storedBlocks.NeedSave)
{
foreach (var kv in buckets.AsLookup().ToArray())
{
indexedEntries.Add(kv.ToArray());
}
buckets.Clear();
tasks.Stop();
storedBlocks.SaveCheckpoint();
tasks.Start();
}
}
catch (Exception ex)
{
IndexerTrace.ErrorWhileImportingBalancesToAzure(ex, txId);
throw;
}
}
}
foreach (var kv in buckets.AsLookup().ToArray())
{
indexedEntries.Add(kv.ToArray());
}
tasks.Stop();
storedBlocks.SaveCheckpoint();
}
}
}
public void Index(Block block)
{
var hash = block.GetHash().ToString();
using (IndexerTrace.NewCorrelation("Upload of " + hash).Open())
{
Stopwatch watch = new Stopwatch();
watch.Start();
bool failedBefore = false;
while (true)
{
try
{
var container = Configuration.GetBlocksContainer();
var client = container.ServiceClient;
client.DefaultRequestOptions.SingleBlobUploadThresholdInBytes = 32 * 1024 * 1024;
var blob = container.GetPageBlobReference(hash);
MemoryStream ms = new MemoryStream();
block.ReadWrite(ms, true);
var blockBytes = ms.GetBuffer();
long length = 512 - (ms.Length % 512);
if (length == 512)
{
length = 0;
}
Array.Resize(ref blockBytes, (int)(ms.Length + length));
try
{
blob.UploadFromByteArray(blockBytes, 0, blockBytes.Length, new AccessCondition()
{
//Will throw if already exist, save 1 call
IfNotModifiedSinceTime = failedBefore ? (DateTimeOffset?)null : DateTimeOffset.MinValue
}, new BlobRequestOptions()
{
MaximumExecutionTime = _Timeout,
ServerTimeout = _Timeout
});
watch.Stop();
IndexerTrace.BlockUploaded(watch.Elapsed, blockBytes.Length);
break;
}
catch (StorageException ex)
{
var alreadyExist = ex.RequestInformation != null && ex.RequestInformation.HttpStatusCode == 412;
if (!alreadyExist)
{
throw;
}
watch.Stop();
IndexerTrace.BlockAlreadyUploaded();
break;
}
}
catch (Exception ex)
{
IndexerTrace.ErrorWhileImportingBlockToAzure(new uint256(hash), ex);
failedBefore = true;
Thread.Sleep(5000);
}
}
}
}
