/// <summary>
/// Performs "Chain" indexing into Azure storage.
/// </summary>
/// <param name="cancellationToken">The token used for cancellation.</param>
/// <returns>A task for asynchronous completion.</returns>
private async Task IndexChainAsync(CancellationToken cancellationToken)
{
this.logger.LogTrace("()");
while (!cancellationToken.IsCancellationRequested)
{
try
{
this.AzureIndexer.IndexChain(this.Chain, cancellationToken);
await Task.Delay(TimeSpan.FromMinutes(1), cancellationToken).ContinueWith(t => { }).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
break;
}
catch (Exception ex)
{
// If something goes wrong then try again 1 minute later
IndexerTrace.ErrorWhileImportingBlockToAzure(this.StoreTip.HashBlock, ex);
await Task.Delay(TimeSpan.FromMinutes(1), cancellationToken).ContinueWith(t => { }).ConfigureAwait(false);
}
}
this.logger.LogTrace("(-)");
}
private void Index(List <ChainPartEntry> chainParts, CancellationToken cancellationToken = default(CancellationToken))
{
this.logger.LogTrace("()");
CloudTable table = this.Configuration.GetChainTable();
TableBatchOperation batch = new TableBatchOperation();
var last = chainParts[chainParts.Count - 1];
foreach (var entry in chainParts)
{
batch.Add(TableOperation.InsertOrReplace(entry.ToEntity()));
if (batch.Count == 100)
{
table.ExecuteBatchAsync(batch).GetAwaiter().GetResult();
batch = new TableBatchOperation();
}
IndexerTrace.RemainingBlockChain(entry.ChainOffset, last.ChainOffset + last.BlockHeaders.Count - 1);
}
if (batch.Count > 0)
{
this.logger.LogTrace("Batch count: {0}", batch.Count);
table.ExecuteBatchAsync(batch, null, null, cancellationToken).GetAwaiter().GetResult();
}
this.logger.LogTrace("(-)");
}
internal void SkipToEnd()
{
var height = Math.Min(ToHeight, _BlockHeaders.Tip.Height);
_LastProcessed = _BlockHeaders.GetBlock(height);
IndexerTrace.Information("Skipped to the end at height " + height);
}
public ChainBase GetNodeChain(Node node)
{
var chain = new ConcurrentChain(this.Configuration.Network);
IndexerTrace.Information("Synchronizing with local node");
node.SynchronizeChain(chain);
IndexerTrace.Information("Chain loaded with height " + chain.Height);
return(chain);
}
public void SaveCheckpoint()
{
if (_LastProcessed != null)
{
_Checkpoint.SaveProgress(_LastProcessed);
IndexerTrace.CheckpointSaved(_LastProcessed, _Checkpoint.CheckpointName);
}
_LastSaved = DateTime.UtcNow;
}
public ChainBase GetNodeChain()
{
IndexerTrace.Information("Connecting to node " + this.Configuration.Node);
using (var node = this.Configuration.ConnectToNode(false))
{
IndexerTrace.Information("Handshaking");
node.VersionHandshake();
return(this.GetNodeChain(node));
}
}
public IEnumerator <BlockInfo> GetEnumerator()
{
Queue <DateTime> lastLogs = new Queue <DateTime>();
Queue <int> lastHeights = new Queue <int>();
var fork = _BlockHeaders.FindFork(_Checkpoint.BlockLocator);
var headers = _BlockHeaders.EnumerateAfter(fork);
headers = headers.Where(h => h.Height <= ToHeight);
var first = headers.FirstOrDefault();
if (first == null)
{
yield break;
}
var height = first.Height;
if (first.Height == 1)
{
headers = new[] { fork }.Concat(headers);
height = 0;
}
foreach (var block in _BlocksRepository.GetBlocks(headers.Select(b => b.HashBlock), CancellationToken))
{
var header = _BlockHeaders.GetBlock(height);
if (block == null)
{
var storeTip = _BlocksRepository.GetStoreTip();
if (storeTip != null)
{
// Store is caught up with Chain but the block is missing from the store.
if (header.Header.BlockTime <= storeTip.Header.BlockTime)
{
throw new InvalidOperationException($"Chained block not found in store (height = { height }). Re-create the block store.");
}
}
// Allow Store to catch up with Chain.
break;
}
_LastProcessed = header;
yield return(new BlockInfo()
{
Block = block,
BlockId = header.HashBlock,
Height = header.Height
});
IndexerTrace.Processed(height, Math.Min(ToHeight, _BlockHeaders.Tip.Height), lastLogs, lastHeights);
height++;
}
}
public void IndexChain(ChainBase chain, CancellationToken cancellationToken = default(CancellationToken))
{
this.logger.LogTrace("()");
if (chain == null)
{
throw new ArgumentNullException("chain");
}
this.SetThrottling();
using (IndexerTrace.NewCorrelation("Index main chain to azure started"))
{
this.Configuration.GetChainTable().CreateIfNotExistsAsync().GetAwaiter().GetResult();
IndexerTrace.InputChainTip(chain.Tip);
var client = this.Configuration.CreateIndexerClient();
var changes = client.GetChainChangesUntilFork(chain.Tip, true, cancellationToken).ToList();
var height = 0;
if (changes.Count != 0)
{
this.logger.LogTrace("Changes count: {0}", changes.Count);
IndexerTrace.IndexedChainTip(changes[0].BlockId, changes[0].Height);
if (changes[0].Height > chain.Tip.Height)
{
IndexerTrace.InputChainIsLate();
this.logger.LogTrace("(-):LATE");
return;
}
height = changes[changes.Count - 1].Height + 1;
if (height > chain.Height)
{
IndexerTrace.IndexedChainIsUpToDate(chain.Tip);
this.logger.LogTrace("(-):UP_TO_DATE");
return;
}
}
else
{
this.logger.LogTrace("No work found");
IndexerTrace.NoForkFoundWithStored();
}
IndexerTrace.IndexingChain(chain.GetBlock(height), chain.Tip);
this.Index(chain, height, cancellationToken);
}
this.logger.LogTrace("(-)");
}
public int IndexWalletBalances(ChainBase chain)
{
using (IndexerTrace.NewCorrelation("Import wallet balances to azure started"))
{
using (var node = this.Configuration.ConnectToNode(false))
{
node.VersionHandshake();
var task = new IndexBalanceTask(this.Configuration, this.Configuration.CreateIndexerClient().GetAllWalletRules());
task.SaveProgression = !this.IgnoreCheckpoints;
task.Index(this.GetBlockFetcher(this.GetCheckpointInternal(IndexerCheckpoints.Wallets), node, chain), this.TaskScheduler);
return(task.IndexedEntities);
}
}
}
public long IndexBlocks(ChainBase chain = null)
{
using (IndexerTrace.NewCorrelation("Import blocks to azure started"))
{
using (var node = this.Configuration.ConnectToNode(false))
{
node.VersionHandshake();
var task = new IndexBlocksTask(this.Configuration);
task.SaveProgression = !this.IgnoreCheckpoints;
task.Index(this.GetBlockFetcher(this.GetCheckpointInternal(IndexerCheckpoints.Blocks), node, chain), this.TaskScheduler);
return(task.IndexedBlocks);
}
}
}
public void SaveCheckpoint()
{
this.logger.LogTrace("()");
if (_LastProcessed != null)
{
this.logger.LogTrace("Saving checkpoints");
_Checkpoint.SaveProgress(_LastProcessed);
IndexerTrace.CheckpointSaved(_LastProcessed, _Checkpoint.CheckpointName);
}
_LastSaved = DateTime.UtcNow;
this.logger.LogTrace("(-)");
}
/// <summary>
/// Performs indexing into Azure storage.
/// </summary>
/// <param name="cancellationToken">The token used for cancellation.</param>
/// <returns>A task for asynchronous completion.</returns>
private async Task IndexAsync(CancellationToken cancellationToken)
{
this.logger.LogTrace("()");
while (this.StoreTip.Height < indexerSettings.To && !cancellationToken.IsCancellationRequested)
{
try
{
// All indexes will progress more or less in step
int fromHeight = this.StoreTip.Height + 1;
int toHeight = Math.Min(this.StoreTip.Height + IndexBatchSize, this.indexerSettings.To);
this.logger.LogTrace("Starting indexing from {0}, to {1}", fromHeight, toHeight);
// Index a batch of blocks
PerformIndexing(IndexerCheckpoints.Blocks, fromHeight, toHeight);
// Index a batch of transactions
PerformIndexing(IndexerCheckpoints.Transactions, fromHeight, toHeight);
// Index a batch of balances
PerformIndexing(IndexerCheckpoints.Balances, fromHeight, toHeight);
// Index a batch of wallets
PerformIndexing(IndexerCheckpoints.Wallets, fromHeight, toHeight);
// Update the StoreTip
UpdateStoreTip();
this.logger.LogTrace("Indexing iteration finished");
}
catch (OperationCanceledException)
{
break;
}
catch (Exception ex)
{
// If something goes wrong then try again 1 minute later
this.logger.LogError("Exception occurred: {0}", ex.ToString());
IndexerTrace.ErrorWhileImportingBlockToAzure(this.StoreTip.HashBlock, ex);
await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken).ContinueWith(t => { }).ConfigureAwait(false);
}
}
this.logger.LogTrace("(-)");
}
/// <summary>
/// Get a block fetcher of the specified chain from the specified checkpoint
/// </summary>
/// <param name="checkpoint">The checkpoint to load from</param>
/// <param name="chain">The chain to fetcher (default: the Node's main chain)</param>
/// <returns>A BlockFetcher for enumerating blocks and saving progression</returns>
public BlockFetcher GetBlockFetcher(Checkpoint checkpoint, Node node, ChainBase chain = null)
{
if (checkpoint == null)
{
throw new ArgumentNullException("checkpoint");
}
if (node == null)
{
throw new ArgumentNullException("node");
}
chain = chain ?? this.GetNodeChain(node);
IndexerTrace.CheckpointLoaded(chain.FindFork(checkpoint.BlockLocator), checkpoint.CheckpointName);
return(new BlockFetcher(checkpoint, new NodeBlocksRepository(node), chain)
{
NeedSaveInterval = this.CheckpointInterval,
FromHeight = this.FromHeight,
ToHeight = this.ToHeight
});
}
public IEnumerator <BlockInfo> GetEnumerator()
{
Queue <DateTime> lastLogs = new Queue <DateTime>();
Queue <int> lastHeights = new Queue <int>();
var fork = _BlockHeaders.FindFork(_Checkpoint.BlockLocator);
_LastProcessed = fork;
var headers = _BlockHeaders.EnumerateAfter(fork);
headers = headers.Where(h => h.Height >= FromHeight && h.Height <= ToHeight);
var first = headers.FirstOrDefault();
if (first == null)
{
yield break;
}
var height = first.Height;
if (first.Height == 1)
{
headers = new[] { fork }.Concat(headers);
height = 0;
}
foreach (var block in _BlocksRepository.GetBlocks(headers.Select(b => b.HashBlock), CancellationToken).TakeWhile(b => b != null))
{
var header = _BlockHeaders.GetBlock(height);
_LastProcessed = header;
yield return(new BlockInfo()
{
Block = block,
BlockId = header.HashBlock,
Height = header.Height
});
IndexerTrace.Processed(height, Math.Min(ToHeight, _BlockHeaders.Tip.Height), lastLogs, lastHeights);
height++;
}
}
/// <summary>
/// Performs indexing into Azure storage.
/// </summary>
/// <param name="cancellationToken">The token used for cancellation.</param>
/// <returns>A task for asynchronous completion.</returns>
private async Task IndexAsync(CancellationToken cancellationToken)
{
this.logger.LogTrace("()");
while (this.StoreTip.Height < indexerSettings.To && !cancellationToken.IsCancellationRequested)
{
try
{
// All indexes will progress more or less in step
// Use 'minHeight' to track the current indexed height
int minHeight = int.MaxValue;
// Index a batch of blocks
if (!cancellationToken.IsCancellationRequested)
{
var task = new IndexBlocksTask(this.IndexerConfig);
task.SaveProgression = !this.indexerSettings.IgnoreCheckpoints;
var fetcher = this.GetBlockFetcher(IndexerCheckpoints.Blocks, cancellationToken);
task.Index(fetcher, this.AzureIndexer.TaskScheduler);
minHeight = Math.Min(minHeight, fetcher._LastProcessed.Height);
}
// Index a batch of transactions
if (!cancellationToken.IsCancellationRequested)
{
var task = new IndexTransactionsTask(this.IndexerConfig);
task.SaveProgression = !this.indexerSettings.IgnoreCheckpoints;
var fetcher = this.GetBlockFetcher(IndexerCheckpoints.Transactions, cancellationToken);
task.Index(fetcher, this.AzureIndexer.TaskScheduler);
minHeight = Math.Min(minHeight, fetcher._LastProcessed.Height);
}
// Index a batch of balances
if (!cancellationToken.IsCancellationRequested)
{
var task = new IndexBalanceTask(this.IndexerConfig, null);
task.SaveProgression = !this.indexerSettings.IgnoreCheckpoints;
var fetcher = this.GetBlockFetcher(IndexerCheckpoints.Balances, cancellationToken);
task.Index(fetcher, this.AzureIndexer.TaskScheduler);
minHeight = Math.Min(minHeight, fetcher._LastProcessed.Height);
}
// Index a batch of wallets
if (!cancellationToken.IsCancellationRequested)
{
var task = new IndexBalanceTask(this.IndexerConfig, this.IndexerConfig.CreateIndexerClient().GetAllWalletRules());
task.SaveProgression = !this.indexerSettings.IgnoreCheckpoints;
var fetcher = this.GetBlockFetcher(IndexerCheckpoints.Wallets, cancellationToken);
task.Index(fetcher, this.AzureIndexer.TaskScheduler);
minHeight = Math.Min(minHeight, fetcher._LastProcessed.Height);
}
// Update the StoreTip value from the minHeight
this.SetStoreTip(this.Chain.GetBlock(Math.Min(minHeight, this.indexerSettings.To)));
}
catch (OperationCanceledException)
{
break;
}
catch (Exception ex)
{
// If something goes wrong then try again 1 minute later
IndexerTrace.ErrorWhileImportingBlockToAzure(this.StoreTip.HashBlock, ex);
await Task.Delay(TimeSpan.FromMinutes(1), cancellationToken).ContinueWith(t => { }).ConfigureAwait(false);
}
}
this.logger.LogTrace("(-)");
}
public async Task <TransactionEntry> GetTransactionAsync(bool loadPreviousOutput, bool fetchColor, uint256 txId)
{
if (txId == null)
{
return(null);
}
TransactionEntry result = null;
var table = Configuration.GetTransactionTable();
var searchedEntity = new TransactionEntry.Entity(txId);
var query = new TableQuery()
.Where(
TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, searchedEntity.PartitionKey),
TableOperators.And,
TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThan, txId.ToString() + "-"),
TableOperators.And,
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThan, txId.ToString() + "|")
)
));
query.TakeCount = 10; //Should not have more
List <TransactionEntry.Entity> entities = new List <TransactionEntry.Entity>();
foreach (var e in await table.ExecuteQuerySegmentedAsync(query, null).ConfigureAwait(false))
{
if (e.IsFat())
{
entities.Add(new TransactionEntry.Entity(await FetchFatEntity(e).ConfigureAwait(false)));
}
else
{
entities.Add(new TransactionEntry.Entity(e));
}
}
if (entities.Count == 0)
{
result = null;
}
else
{
result = new TransactionEntry(entities.ToArray());
if (result.Transaction == null)
{
foreach (var block in result.BlockIds.Select(id => GetBlock(id)).Where(b => b != null))
{
result.Transaction = block.Transactions.FirstOrDefault(t => t.GetHash() == txId);
entities[0].Transaction = result.Transaction;
if (entities[0].Transaction != null)
{
await UpdateEntity(table, entities[0].CreateTableEntity()).ConfigureAwait(false);
}
break;
}
}
if (fetchColor && result.ColoredTransaction == null)
{
result.ColoredTransaction = await ColoredTransaction.FetchColorsAsync(txId, result.Transaction, new CachedColoredTransactionRepository(new IndexerColoredTransactionRepository(Configuration))).ConfigureAwait(false);
entities[0].ColoredTransaction = result.ColoredTransaction;
if (entities[0].ColoredTransaction != null)
{
await UpdateEntity(table, entities[0].CreateTableEntity()).ConfigureAwait(false);
}
}
var needTxOut = result.SpentCoins == null && loadPreviousOutput && result.Transaction != null;
if (needTxOut)
{
var inputs = result.Transaction.Inputs.Select(o => o.PrevOut).ToArray();
var parents = await
GetTransactionsAsync(false, false, inputs
.Select(i => i.Hash)
.ToArray()).ConfigureAwait(false);
for (int i = 0; i < parents.Length; i++)
{
if (parents[i] == null)
{
IndexerTrace.MissingTransactionFromDatabase(result.Transaction.Inputs[i].PrevOut.Hash);
return(null);
}
}
var outputs = parents.Select((p, i) => p.Transaction.Outputs[inputs[i].N]).ToArray();
result.SpentCoins = Enumerable
.Range(0, inputs.Length)
.Select(i => new Spendable(inputs[i], outputs[i]))
.ToList();
entities[0].PreviousTxOuts.Clear();
entities[0].PreviousTxOuts.AddRange(outputs);
if (entities[0].IsLoaded)
{
await UpdateEntity(table, entities[0].CreateTableEntity()).ConfigureAwait(false);
}
}
}
return(result != null && result.Transaction != null ? result : null);
}
/// <summary>
/// Performs indexing into Azure storage.
/// </summary>
/// <param name="cancellationToken">The token used for cancellation.</param>
/// <returns>A task for asynchronous completion.</returns>
private async Task IndexAsync(CancellationToken cancellationToken)
{
this.logger.LogTrace("()");
while (this.StoreTip.Height < indexerSettings.To && !cancellationToken.IsCancellationRequested)
{
try
{
// All indexes will progress more or less in step
int fromHeight = this.StoreTip.Height + 1;
int toHeight = Math.Min(this.StoreTip.Height + IndexBatchSize, this.indexerSettings.To);
// Index a batch of blocks
if (!cancellationToken.IsCancellationRequested && toHeight > this.BlocksFetcher._LastProcessed.Height)
{
this.BlocksFetcher.FromHeight = Math.Max(this.BlocksFetcher._LastProcessed.Height + 1, fromHeight);
this.BlocksFetcher.ToHeight = toHeight;
var task = new IndexBlocksTask(this.IndexerConfig);
task.SaveProgression = !this.indexerSettings.IgnoreCheckpoints;
task.Index(this.BlocksFetcher, this.AzureIndexer.TaskScheduler);
}
// Index a batch of transactions
if (!cancellationToken.IsCancellationRequested && toHeight > this.TransactionsFetcher._LastProcessed.Height)
{
this.TransactionsFetcher.FromHeight = Math.Max(this.TransactionsFetcher._LastProcessed.Height + 1, fromHeight);
this.TransactionsFetcher.ToHeight = toHeight;
var task = new IndexTransactionsTask(this.IndexerConfig);
task.SaveProgression = !this.indexerSettings.IgnoreCheckpoints;
task.Index(this.TransactionsFetcher, this.AzureIndexer.TaskScheduler);
}
// Index a batch of balances
if (!cancellationToken.IsCancellationRequested && toHeight > this.BalancesFetcher._LastProcessed.Height)
{
this.BalancesFetcher.FromHeight = Math.Max(this.BalancesFetcher._LastProcessed.Height + 1, fromHeight);
this.BalancesFetcher.ToHeight = toHeight;
var task = new IndexBalanceTask(this.IndexerConfig, null);
task.SaveProgression = !this.indexerSettings.IgnoreCheckpoints;
task.Index(this.BalancesFetcher, this.AzureIndexer.TaskScheduler);
}
// Index a batch of wallets
if (!cancellationToken.IsCancellationRequested && toHeight > this.WalletsFetcher._LastProcessed.Height)
{
this.WalletsFetcher.FromHeight = Math.Max(this.WalletsFetcher._LastProcessed.Height + 1, fromHeight);
this.WalletsFetcher.ToHeight = toHeight;
var task = new IndexBalanceTask(this.IndexerConfig, this.IndexerConfig.CreateIndexerClient().GetAllWalletRules());
task.SaveProgression = !this.indexerSettings.IgnoreCheckpoints;
task.Index(this.WalletsFetcher, this.AzureIndexer.TaskScheduler);
}
// Update the StoreTip value from the minHeight
int minHeight = this.BlocksFetcher._LastProcessed.Height;
minHeight = Math.Min(minHeight, this.BalancesFetcher._LastProcessed.Height);
minHeight = Math.Min(minHeight, this.TransactionsFetcher._LastProcessed.Height);
minHeight = Math.Min(minHeight, this.WalletsFetcher._LastProcessed.Height);
this.SetStoreTip(this.Chain.GetBlock(minHeight));
}
catch (OperationCanceledException)
{
break;
}
catch (Exception ex)
{
// If something goes wrong then try again 1 minute later
this.logger.LogError(ex.Message);
IndexerTrace.ErrorWhileImportingBlockToAzure(this.StoreTip.HashBlock, ex);
await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken).ContinueWith(t => { }).ConfigureAwait(false);
}
}
this.logger.LogTrace("(-)");
}
