/// <summary> /// Performs "Chain" indexing into Azure storage. /// </summary> /// <param name="cancellationToken">The token used for cancellation.</param> /// <returns>A task for asynchronous completion.</returns> private async Task IndexChainAsync(CancellationToken cancellationToken) { this.logger.LogTrace("()"); while (!cancellationToken.IsCancellationRequested) { try { this.AzureIndexer.IndexChain(this.Chain, cancellationToken); await Task.Delay(TimeSpan.FromMinutes(1), cancellationToken).ContinueWith(t => { }).ConfigureAwait(false); } catch (OperationCanceledException) { break; } catch (Exception ex) { // If something goes wrong then try again 1 minute later IndexerTrace.ErrorWhileImportingBlockToAzure(this.StoreTip.HashBlock, ex); await Task.Delay(TimeSpan.FromMinutes(1), cancellationToken).ContinueWith(t => { }).ConfigureAwait(false); } } this.logger.LogTrace("(-)"); }
private void Index(List <ChainPartEntry> chainParts, CancellationToken cancellationToken = default(CancellationToken)) { this.logger.LogTrace("()"); CloudTable table = this.Configuration.GetChainTable(); TableBatchOperation batch = new TableBatchOperation(); var last = chainParts[chainParts.Count - 1]; foreach (var entry in chainParts) { batch.Add(TableOperation.InsertOrReplace(entry.ToEntity())); if (batch.Count == 100) { table.ExecuteBatchAsync(batch).GetAwaiter().GetResult(); batch = new TableBatchOperation(); } IndexerTrace.RemainingBlockChain(entry.ChainOffset, last.ChainOffset + last.BlockHeaders.Count - 1); } if (batch.Count > 0) { this.logger.LogTrace("Batch count: {0}", batch.Count); table.ExecuteBatchAsync(batch, null, null, cancellationToken).GetAwaiter().GetResult(); } this.logger.LogTrace("(-)"); }
internal void SkipToEnd() { var height = Math.Min(ToHeight, _BlockHeaders.Tip.Height); _LastProcessed = _BlockHeaders.GetBlock(height); IndexerTrace.Information("Skipped to the end at height " + height); }
public ChainBase GetNodeChain(Node node) { var chain = new ConcurrentChain(this.Configuration.Network); IndexerTrace.Information("Synchronizing with local node"); node.SynchronizeChain(chain); IndexerTrace.Information("Chain loaded with height " + chain.Height); return(chain); }
public void SaveCheckpoint() { if (_LastProcessed != null) { _Checkpoint.SaveProgress(_LastProcessed); IndexerTrace.CheckpointSaved(_LastProcessed, _Checkpoint.CheckpointName); } _LastSaved = DateTime.UtcNow; }
public ChainBase GetNodeChain() { IndexerTrace.Information("Connecting to node " + this.Configuration.Node); using (var node = this.Configuration.ConnectToNode(false)) { IndexerTrace.Information("Handshaking"); node.VersionHandshake(); return(this.GetNodeChain(node)); } }
public IEnumerator <BlockInfo> GetEnumerator() { Queue <DateTime> lastLogs = new Queue <DateTime>(); Queue <int> lastHeights = new Queue <int>(); var fork = _BlockHeaders.FindFork(_Checkpoint.BlockLocator); var headers = _BlockHeaders.EnumerateAfter(fork); headers = headers.Where(h => h.Height <= ToHeight); var first = headers.FirstOrDefault(); if (first == null) { yield break; } var height = first.Height; if (first.Height == 1) { headers = new[] { fork }.Concat(headers); height = 0; } foreach (var block in _BlocksRepository.GetBlocks(headers.Select(b => b.HashBlock), CancellationToken)) { var header = _BlockHeaders.GetBlock(height); if (block == null) { var storeTip = _BlocksRepository.GetStoreTip(); if (storeTip != null) { // Store is caught up with Chain but the block is missing from the store. if (header.Header.BlockTime <= storeTip.Header.BlockTime) { throw new InvalidOperationException($"Chained block not found in store (height = { height }). Re-create the block store."); } } // Allow Store to catch up with Chain. break; } _LastProcessed = header; yield return(new BlockInfo() { Block = block, BlockId = header.HashBlock, Height = header.Height }); IndexerTrace.Processed(height, Math.Min(ToHeight, _BlockHeaders.Tip.Height), lastLogs, lastHeights); height++; } }
public void IndexChain(ChainBase chain, CancellationToken cancellationToken = default(CancellationToken)) { this.logger.LogTrace("()"); if (chain == null) { throw new ArgumentNullException("chain"); } this.SetThrottling(); using (IndexerTrace.NewCorrelation("Index main chain to azure started")) { this.Configuration.GetChainTable().CreateIfNotExistsAsync().GetAwaiter().GetResult(); IndexerTrace.InputChainTip(chain.Tip); var client = this.Configuration.CreateIndexerClient(); var changes = client.GetChainChangesUntilFork(chain.Tip, true, cancellationToken).ToList(); var height = 0; if (changes.Count != 0) { this.logger.LogTrace("Changes count: {0}", changes.Count); IndexerTrace.IndexedChainTip(changes[0].BlockId, changes[0].Height); if (changes[0].Height > chain.Tip.Height) { IndexerTrace.InputChainIsLate(); this.logger.LogTrace("(-):LATE"); return; } height = changes[changes.Count - 1].Height + 1; if (height > chain.Height) { IndexerTrace.IndexedChainIsUpToDate(chain.Tip); this.logger.LogTrace("(-):UP_TO_DATE"); return; } } else { this.logger.LogTrace("No work found"); IndexerTrace.NoForkFoundWithStored(); } IndexerTrace.IndexingChain(chain.GetBlock(height), chain.Tip); this.Index(chain, height, cancellationToken); } this.logger.LogTrace("(-)"); }
public int IndexWalletBalances(ChainBase chain) { using (IndexerTrace.NewCorrelation("Import wallet balances to azure started")) { using (var node = this.Configuration.ConnectToNode(false)) { node.VersionHandshake(); var task = new IndexBalanceTask(this.Configuration, this.Configuration.CreateIndexerClient().GetAllWalletRules()); task.SaveProgression = !this.IgnoreCheckpoints; task.Index(this.GetBlockFetcher(this.GetCheckpointInternal(IndexerCheckpoints.Wallets), node, chain), this.TaskScheduler); return(task.IndexedEntities); } } }
public long IndexBlocks(ChainBase chain = null) { using (IndexerTrace.NewCorrelation("Import blocks to azure started")) { using (var node = this.Configuration.ConnectToNode(false)) { node.VersionHandshake(); var task = new IndexBlocksTask(this.Configuration); task.SaveProgression = !this.IgnoreCheckpoints; task.Index(this.GetBlockFetcher(this.GetCheckpointInternal(IndexerCheckpoints.Blocks), node, chain), this.TaskScheduler); return(task.IndexedBlocks); } } }
public void SaveCheckpoint() { this.logger.LogTrace("()"); if (_LastProcessed != null) { this.logger.LogTrace("Saving checkpoints"); _Checkpoint.SaveProgress(_LastProcessed); IndexerTrace.CheckpointSaved(_LastProcessed, _Checkpoint.CheckpointName); } _LastSaved = DateTime.UtcNow; this.logger.LogTrace("(-)"); }
/// <summary> /// Performs indexing into Azure storage. /// </summary> /// <param name="cancellationToken">The token used for cancellation.</param> /// <returns>A task for asynchronous completion.</returns> private async Task IndexAsync(CancellationToken cancellationToken) { this.logger.LogTrace("()"); while (this.StoreTip.Height < indexerSettings.To && !cancellationToken.IsCancellationRequested) { try { // All indexes will progress more or less in step int fromHeight = this.StoreTip.Height + 1; int toHeight = Math.Min(this.StoreTip.Height + IndexBatchSize, this.indexerSettings.To); this.logger.LogTrace("Starting indexing from {0}, to {1}", fromHeight, toHeight); // Index a batch of blocks PerformIndexing(IndexerCheckpoints.Blocks, fromHeight, toHeight); // Index a batch of transactions PerformIndexing(IndexerCheckpoints.Transactions, fromHeight, toHeight); // Index a batch of balances PerformIndexing(IndexerCheckpoints.Balances, fromHeight, toHeight); // Index a batch of wallets PerformIndexing(IndexerCheckpoints.Wallets, fromHeight, toHeight); // Update the StoreTip UpdateStoreTip(); this.logger.LogTrace("Indexing iteration finished"); } catch (OperationCanceledException) { break; } catch (Exception ex) { // If something goes wrong then try again 1 minute later this.logger.LogError("Exception occurred: {0}", ex.ToString()); IndexerTrace.ErrorWhileImportingBlockToAzure(this.StoreTip.HashBlock, ex); await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken).ContinueWith(t => { }).ConfigureAwait(false); } } this.logger.LogTrace("(-)"); }
/// <summary> /// Get a block fetcher of the specified chain from the specified checkpoint /// </summary> /// <param name="checkpoint">The checkpoint to load from</param> /// <param name="chain">The chain to fetcher (default: the Node's main chain)</param> /// <returns>A BlockFetcher for enumerating blocks and saving progression</returns> public BlockFetcher GetBlockFetcher(Checkpoint checkpoint, Node node, ChainBase chain = null) { if (checkpoint == null) { throw new ArgumentNullException("checkpoint"); } if (node == null) { throw new ArgumentNullException("node"); } chain = chain ?? this.GetNodeChain(node); IndexerTrace.CheckpointLoaded(chain.FindFork(checkpoint.BlockLocator), checkpoint.CheckpointName); return(new BlockFetcher(checkpoint, new NodeBlocksRepository(node), chain) { NeedSaveInterval = this.CheckpointInterval, FromHeight = this.FromHeight, ToHeight = this.ToHeight }); }
public IEnumerator <BlockInfo> GetEnumerator() { Queue <DateTime> lastLogs = new Queue <DateTime>(); Queue <int> lastHeights = new Queue <int>(); var fork = _BlockHeaders.FindFork(_Checkpoint.BlockLocator); _LastProcessed = fork; var headers = _BlockHeaders.EnumerateAfter(fork); headers = headers.Where(h => h.Height >= FromHeight && h.Height <= ToHeight); var first = headers.FirstOrDefault(); if (first == null) { yield break; } var height = first.Height; if (first.Height == 1) { headers = new[] { fork }.Concat(headers); height = 0; } foreach (var block in _BlocksRepository.GetBlocks(headers.Select(b => b.HashBlock), CancellationToken).TakeWhile(b => b != null)) { var header = _BlockHeaders.GetBlock(height); _LastProcessed = header; yield return(new BlockInfo() { Block = block, BlockId = header.HashBlock, Height = header.Height }); IndexerTrace.Processed(height, Math.Min(ToHeight, _BlockHeaders.Tip.Height), lastLogs, lastHeights); height++; } }
/// <summary> /// Performs indexing into Azure storage. /// </summary> /// <param name="cancellationToken">The token used for cancellation.</param> /// <returns>A task for asynchronous completion.</returns> private async Task IndexAsync(CancellationToken cancellationToken) { this.logger.LogTrace("()"); while (this.StoreTip.Height < indexerSettings.To && !cancellationToken.IsCancellationRequested) { try { // All indexes will progress more or less in step // Use 'minHeight' to track the current indexed height int minHeight = int.MaxValue; // Index a batch of blocks if (!cancellationToken.IsCancellationRequested) { var task = new IndexBlocksTask(this.IndexerConfig); task.SaveProgression = !this.indexerSettings.IgnoreCheckpoints; var fetcher = this.GetBlockFetcher(IndexerCheckpoints.Blocks, cancellationToken); task.Index(fetcher, this.AzureIndexer.TaskScheduler); minHeight = Math.Min(minHeight, fetcher._LastProcessed.Height); } // Index a batch of transactions if (!cancellationToken.IsCancellationRequested) { var task = new IndexTransactionsTask(this.IndexerConfig); task.SaveProgression = !this.indexerSettings.IgnoreCheckpoints; var fetcher = this.GetBlockFetcher(IndexerCheckpoints.Transactions, cancellationToken); task.Index(fetcher, this.AzureIndexer.TaskScheduler); minHeight = Math.Min(minHeight, fetcher._LastProcessed.Height); } // Index a batch of balances if (!cancellationToken.IsCancellationRequested) { var task = new IndexBalanceTask(this.IndexerConfig, null); task.SaveProgression = !this.indexerSettings.IgnoreCheckpoints; var fetcher = this.GetBlockFetcher(IndexerCheckpoints.Balances, cancellationToken); task.Index(fetcher, this.AzureIndexer.TaskScheduler); minHeight = Math.Min(minHeight, fetcher._LastProcessed.Height); } // Index a batch of wallets if (!cancellationToken.IsCancellationRequested) { var task = new IndexBalanceTask(this.IndexerConfig, this.IndexerConfig.CreateIndexerClient().GetAllWalletRules()); task.SaveProgression = !this.indexerSettings.IgnoreCheckpoints; var fetcher = this.GetBlockFetcher(IndexerCheckpoints.Wallets, cancellationToken); task.Index(fetcher, this.AzureIndexer.TaskScheduler); minHeight = Math.Min(minHeight, fetcher._LastProcessed.Height); } // Update the StoreTip value from the minHeight this.SetStoreTip(this.Chain.GetBlock(Math.Min(minHeight, this.indexerSettings.To))); } catch (OperationCanceledException) { break; } catch (Exception ex) { // If something goes wrong then try again 1 minute later IndexerTrace.ErrorWhileImportingBlockToAzure(this.StoreTip.HashBlock, ex); await Task.Delay(TimeSpan.FromMinutes(1), cancellationToken).ContinueWith(t => { }).ConfigureAwait(false); } } this.logger.LogTrace("(-)"); }
public async Task <TransactionEntry> GetTransactionAsync(bool loadPreviousOutput, bool fetchColor, uint256 txId) { if (txId == null) { return(null); } TransactionEntry result = null; var table = Configuration.GetTransactionTable(); var searchedEntity = new TransactionEntry.Entity(txId); var query = new TableQuery() .Where( TableQuery.CombineFilters( TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, searchedEntity.PartitionKey), TableOperators.And, TableQuery.CombineFilters( TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThan, txId.ToString() + "-"), TableOperators.And, TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThan, txId.ToString() + "|") ) )); query.TakeCount = 10; //Should not have more List <TransactionEntry.Entity> entities = new List <TransactionEntry.Entity>(); foreach (var e in await table.ExecuteQuerySegmentedAsync(query, null).ConfigureAwait(false)) { if (e.IsFat()) { entities.Add(new TransactionEntry.Entity(await FetchFatEntity(e).ConfigureAwait(false))); } else { entities.Add(new TransactionEntry.Entity(e)); } } if (entities.Count == 0) { result = null; } else { result = new TransactionEntry(entities.ToArray()); if (result.Transaction == null) { foreach (var block in result.BlockIds.Select(id => GetBlock(id)).Where(b => b != null)) { result.Transaction = block.Transactions.FirstOrDefault(t => t.GetHash() == txId); entities[0].Transaction = result.Transaction; if (entities[0].Transaction != null) { await UpdateEntity(table, entities[0].CreateTableEntity()).ConfigureAwait(false); } break; } } if (fetchColor && result.ColoredTransaction == null) { result.ColoredTransaction = await ColoredTransaction.FetchColorsAsync(txId, result.Transaction, new CachedColoredTransactionRepository(new IndexerColoredTransactionRepository(Configuration))).ConfigureAwait(false); entities[0].ColoredTransaction = result.ColoredTransaction; if (entities[0].ColoredTransaction != null) { await UpdateEntity(table, entities[0].CreateTableEntity()).ConfigureAwait(false); } } var needTxOut = result.SpentCoins == null && loadPreviousOutput && result.Transaction != null; if (needTxOut) { var inputs = result.Transaction.Inputs.Select(o => o.PrevOut).ToArray(); var parents = await GetTransactionsAsync(false, false, inputs .Select(i => i.Hash) .ToArray()).ConfigureAwait(false); for (int i = 0; i < parents.Length; i++) { if (parents[i] == null) { IndexerTrace.MissingTransactionFromDatabase(result.Transaction.Inputs[i].PrevOut.Hash); return(null); } } var outputs = parents.Select((p, i) => p.Transaction.Outputs[inputs[i].N]).ToArray(); result.SpentCoins = Enumerable .Range(0, inputs.Length) .Select(i => new Spendable(inputs[i], outputs[i])) .ToList(); entities[0].PreviousTxOuts.Clear(); entities[0].PreviousTxOuts.AddRange(outputs); if (entities[0].IsLoaded) { await UpdateEntity(table, entities[0].CreateTableEntity()).ConfigureAwait(false); } } } return(result != null && result.Transaction != null ? result : null); }
/// <summary> /// Performs indexing into Azure storage. /// </summary> /// <param name="cancellationToken">The token used for cancellation.</param> /// <returns>A task for asynchronous completion.</returns> private async Task IndexAsync(CancellationToken cancellationToken) { this.logger.LogTrace("()"); while (this.StoreTip.Height < indexerSettings.To && !cancellationToken.IsCancellationRequested) { try { // All indexes will progress more or less in step int fromHeight = this.StoreTip.Height + 1; int toHeight = Math.Min(this.StoreTip.Height + IndexBatchSize, this.indexerSettings.To); // Index a batch of blocks if (!cancellationToken.IsCancellationRequested && toHeight > this.BlocksFetcher._LastProcessed.Height) { this.BlocksFetcher.FromHeight = Math.Max(this.BlocksFetcher._LastProcessed.Height + 1, fromHeight); this.BlocksFetcher.ToHeight = toHeight; var task = new IndexBlocksTask(this.IndexerConfig); task.SaveProgression = !this.indexerSettings.IgnoreCheckpoints; task.Index(this.BlocksFetcher, this.AzureIndexer.TaskScheduler); } // Index a batch of transactions if (!cancellationToken.IsCancellationRequested && toHeight > this.TransactionsFetcher._LastProcessed.Height) { this.TransactionsFetcher.FromHeight = Math.Max(this.TransactionsFetcher._LastProcessed.Height + 1, fromHeight); this.TransactionsFetcher.ToHeight = toHeight; var task = new IndexTransactionsTask(this.IndexerConfig); task.SaveProgression = !this.indexerSettings.IgnoreCheckpoints; task.Index(this.TransactionsFetcher, this.AzureIndexer.TaskScheduler); } // Index a batch of balances if (!cancellationToken.IsCancellationRequested && toHeight > this.BalancesFetcher._LastProcessed.Height) { this.BalancesFetcher.FromHeight = Math.Max(this.BalancesFetcher._LastProcessed.Height + 1, fromHeight); this.BalancesFetcher.ToHeight = toHeight; var task = new IndexBalanceTask(this.IndexerConfig, null); task.SaveProgression = !this.indexerSettings.IgnoreCheckpoints; task.Index(this.BalancesFetcher, this.AzureIndexer.TaskScheduler); } // Index a batch of wallets if (!cancellationToken.IsCancellationRequested && toHeight > this.WalletsFetcher._LastProcessed.Height) { this.WalletsFetcher.FromHeight = Math.Max(this.WalletsFetcher._LastProcessed.Height + 1, fromHeight); this.WalletsFetcher.ToHeight = toHeight; var task = new IndexBalanceTask(this.IndexerConfig, this.IndexerConfig.CreateIndexerClient().GetAllWalletRules()); task.SaveProgression = !this.indexerSettings.IgnoreCheckpoints; task.Index(this.WalletsFetcher, this.AzureIndexer.TaskScheduler); } // Update the StoreTip value from the minHeight int minHeight = this.BlocksFetcher._LastProcessed.Height; minHeight = Math.Min(minHeight, this.BalancesFetcher._LastProcessed.Height); minHeight = Math.Min(minHeight, this.TransactionsFetcher._LastProcessed.Height); minHeight = Math.Min(minHeight, this.WalletsFetcher._LastProcessed.Height); this.SetStoreTip(this.Chain.GetBlock(minHeight)); } catch (OperationCanceledException) { break; } catch (Exception ex) { // If something goes wrong then try again 1 minute later this.logger.LogError(ex.Message); IndexerTrace.ErrorWhileImportingBlockToAzure(this.StoreTip.HashBlock, ex); await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken).ContinueWith(t => { }).ConfigureAwait(false); } } this.logger.LogTrace("(-)"); }