public long IndexBlocks(ChainBase chain = null) { long blkCount = 0; SetThrottling(); BlockingCollection <Block> blocks = new BlockingCollection <Block>(20); var tasks = CreateTaskPool(blocks, Index, 15); using (IndexerTrace.NewCorrelation("Import blocks to azure started").Open()) { Configuration.GetBlocksContainer().CreateIfNotExists(); using (var storedBlocks = Enumerate("blocks", chain)) { foreach (var block in storedBlocks) { blkCount++; blocks.Add(block.Block); if (storedBlocks.NeedSave) { tasks.Stop(); storedBlocks.SaveCheckpoint(); tasks.Start(); } } tasks.Stop(); storedBlocks.SaveCheckpoint(); } } return(blkCount); }
public long IndexTransactions(ChainBase chain = null) { long txCount = 0; SetThrottling(); BlockingCollection <TransactionEntry.Entity[]> transactions = new BlockingCollection <TransactionEntry.Entity[]>(20); var tasks = CreateTaskPool(transactions, (txs) => Index(txs), 30); using (IndexerTrace.NewCorrelation("Import transactions to azure started").Open()) { Configuration.GetTransactionTable().CreateIfNotExists(); var buckets = new MultiValueDictionary <string, TransactionEntry.Entity>(); using (var storedBlocks = Enumerate("transactions", chain)) { foreach (var block in storedBlocks) { foreach (var transaction in block.Block.Transactions) { txCount++; var indexed = new TransactionEntry.Entity(null, transaction, block.BlockId); buckets.Add(indexed.PartitionKey, indexed); var collection = buckets[indexed.PartitionKey]; if (collection.Count == 100) { PushTransactions(buckets, collection, transactions); } if (storedBlocks.NeedSave) { foreach (var kv in buckets.AsLookup().ToArray()) { PushTransactions(buckets, kv, transactions); } tasks.Stop(); storedBlocks.SaveCheckpoint(); tasks.Start(); } } } foreach (var kv in buckets.AsLookup().ToArray()) { PushTransactions(buckets, kv, transactions); } tasks.Stop(); storedBlocks.SaveCheckpoint(); } } return(txCount); }
public int IndexWalletBalances(ChainBase chain) { using (IndexerTrace.NewCorrelation("Import wallet balances to azure started")) { using (var node = Configuration.ConnectToNode(false)) { node.VersionHandshake(); var task = new IndexBalanceTask(Configuration, Configuration.CreateIndexerClient().GetAllWalletRules()); task.SaveProgression = !IgnoreCheckpoints; task.Index(GetBlockFetcher(GetCheckpointInternal(IndexerCheckpoints.Wallets), node, chain), TaskScheduler); return(task.IndexedEntities); } } }
public long IndexBlocks(ChainBase chain = null) { using (IndexerTrace.NewCorrelation("Import blocks to azure started")) { using (var node = Configuration.ConnectToNode(false)) { node.VersionHandshake(); var task = new IndexBlocksTask(Configuration); task.SaveProgression = !IgnoreCheckpoints; task.Index(GetBlockFetcher(GetCheckpointInternal(IndexerCheckpoints.Blocks), node, chain), TaskScheduler); return(task.IndexedBlocks); } } }
public int IndexOrderedBalances(ChainBase chain) { using (IndexerTrace.NewCorrelation("Import balances to azure started").Open()) { using (var node = Configuration.ConnectToNode(false)) { node.VersionHandshake(); var task = new IndexBalanceTask(Configuration, null); task.SaveProgression = !IgnoreCheckpoints; task.Index(GetBlockFetcher(GetCheckpointInternal(IndexerCheckpoints.Balances), node, chain), TaskScheduler); return(task.IndexedEntities); } } }
public void IndexChain(ChainBase chain) { if (chain == null) { throw new ArgumentNullException("chain"); } SetThrottling(); using (IndexerTrace.NewCorrelation("Index main chain to azure started")) { Configuration.GetChainTable().CreateIfNotExistsAsync().GetAwaiter().GetResult(); IndexerTrace.InputChainTip(chain.Tip); var client = Configuration.CreateIndexerClient(); var changes = client.GetChainChangesUntilFork(chain.Tip, true).ToList(); var height = 0; if (changes.Count != 0) { IndexerTrace.IndexedChainTip(changes[0].BlockId, changes[0].Height); if (changes[0].Height > chain.Tip.Height) { IndexerTrace.InputChainIsLate(); return; } height = changes[changes.Count - 1].Height + 1; if (height > chain.Height) { IndexerTrace.IndexedChainIsUpToDate(chain.Tip); return; } } else { IndexerTrace.NoForkFoundWithStored(); } IndexerTrace.IndexingChain(chain.GetBlock(height), chain.Tip); Index(chain, height); } }
private void IndexBalances(ChainBase chain, string checkpointName, Func <uint256, Transaction, uint256, BlockHeader, int, IEnumerable <OrderedBalanceChange> > extract) { SetThrottling(); BlockingCollection <OrderedBalanceChange[]> indexedEntries = new BlockingCollection <OrderedBalanceChange[]>(100); var tasks = CreateTaskPool(indexedEntries, (entries) => Index(entries.Select(e => e.ToEntity()), this.Configuration.GetBalanceTable()), 30); using (IndexerTrace.NewCorrelation("Import balances " + checkpointName + " to azure started").Open()) { this.Configuration.GetBalanceTable().CreateIfNotExists(); var buckets = new MultiValueDictionary <string, OrderedBalanceChange>(); using (var storedBlocks = Enumerate(checkpointName, chain)) { foreach (var block in storedBlocks) { foreach (var tx in block.Block.Transactions) { var txId = tx.GetHash(); try { var entries = extract(txId, tx, block.BlockId, block.Block.Header, block.Height); foreach (var entry in entries) { buckets.Add(entry.PartitionKey, entry); var bucket = buckets[entry.PartitionKey]; if (bucket.Count == 100) { indexedEntries.Add(bucket.ToArray()); buckets.Remove(entry.PartitionKey); } } if (storedBlocks.NeedSave) { foreach (var kv in buckets.AsLookup().ToArray()) { indexedEntries.Add(kv.ToArray()); } buckets.Clear(); tasks.Stop(); storedBlocks.SaveCheckpoint(); tasks.Start(); } } catch (Exception ex) { IndexerTrace.ErrorWhileImportingBalancesToAzure(ex, txId); throw; } } } foreach (var kv in buckets.AsLookup().ToArray()) { indexedEntries.Add(kv.ToArray()); } tasks.Stop(); storedBlocks.SaveCheckpoint(); } } }
public void Index(Block block) { var hash = block.GetHash().ToString(); using (IndexerTrace.NewCorrelation("Upload of " + hash).Open()) { Stopwatch watch = new Stopwatch(); watch.Start(); bool failedBefore = false; while (true) { try { var container = Configuration.GetBlocksContainer(); var client = container.ServiceClient; client.DefaultRequestOptions.SingleBlobUploadThresholdInBytes = 32 * 1024 * 1024; var blob = container.GetPageBlobReference(hash); MemoryStream ms = new MemoryStream(); block.ReadWrite(ms, true); var blockBytes = ms.GetBuffer(); long length = 512 - (ms.Length % 512); if (length == 512) { length = 0; } Array.Resize(ref blockBytes, (int)(ms.Length + length)); try { blob.UploadFromByteArray(blockBytes, 0, blockBytes.Length, new AccessCondition() { //Will throw if already exist, save 1 call IfNotModifiedSinceTime = failedBefore ? (DateTimeOffset?)null : DateTimeOffset.MinValue }, new BlobRequestOptions() { MaximumExecutionTime = _Timeout, ServerTimeout = _Timeout }); watch.Stop(); IndexerTrace.BlockUploaded(watch.Elapsed, blockBytes.Length); break; } catch (StorageException ex) { var alreadyExist = ex.RequestInformation != null && ex.RequestInformation.HttpStatusCode == 412; if (!alreadyExist) { throw; } watch.Stop(); IndexerTrace.BlockAlreadyUploaded(); break; } } catch (Exception ex) { IndexerTrace.ErrorWhileImportingBlockToAzure(new uint256(hash), ex); failedBefore = true; Thread.Sleep(5000); } } } }