public long IndexBlocks(ChainBase chain = null)
{
long blkCount = 0;
SetThrottling();
BlockingCollection <Block> blocks = new BlockingCollection <Block>(20);
var tasks = CreateTaskPool(blocks, Index, 15);
using (IndexerTrace.NewCorrelation("Import blocks to azure started").Open())
{
Configuration.GetBlocksContainer().CreateIfNotExists();
using (var storedBlocks = Enumerate("blocks", chain))
{
foreach (var block in storedBlocks)
{
blkCount++;
blocks.Add(block.Block);
if (storedBlocks.NeedSave)
{
tasks.Stop();
storedBlocks.SaveCheckpoint();
tasks.Start();
}
}
tasks.Stop();
storedBlocks.SaveCheckpoint();
}
}
return(blkCount);
}
internal void SkipToEnd()
{
var height = Math.Min(ToHeight, _BlockHeaders.Tip.Height);
_LastProcessed = _BlockHeaders.GetBlock(height);
IndexerTrace.Information("Skipped to the end at height " + height);
}
public void DeleteCheckpoints()
{
foreach (var checkpoint in _Checkpoints)
{
var file = Configuration.GetFilePath(checkpoint);
File.Delete(file);
IndexerTrace.Information(file + " Deleted");
}
}
public ChainBase GetNodeChain(Node node)
{
var chain = new ConcurrentChain(Configuration.Network);
IndexerTrace.Information("Synchronizing with local node");
node.SynchronizeChain(chain);
IndexerTrace.Information("Chain loaded with height " + chain.Height);
return(chain);
}
public void SaveCheckpoint()
{
if (_LastProcessed != null)
{
_Checkpoint.SaveProgress(_LastProcessed);
IndexerTrace.CheckpointSaved(_LastProcessed, _Checkpoint.CheckpointName);
}
_LastSaved = DateTime.UtcNow;
}
public ChainBase GetNodeChain()
{
IndexerTrace.Information("Connecting to node " + Configuration.Node);
using (var node = Configuration.ConnectToNode(false))
{
IndexerTrace.Information("Handshaking");
node.VersionHandshake();
return(GetNodeChain(node));
}
}
public TaskPool <TItem> CreateTaskPool <TItem>(BlockingCollection <TItem> collection, Action <TItem> action, int defaultTaskCount)
{
var pool = new TaskPool <TItem>(collection, action, defaultTaskCount)
{
TaskCount = TaskCount
};
pool.Start();
IndexerTrace.TaskCount(pool.Tasks.Length);
return(pool);
}
private void Index(IEnumerable <ITableEntity> entities, CloudTable table)
{
int exceptionCount = 0;
while (true)
{
try
{
var options = new TableRequestOptions()
{
PayloadFormat = TablePayloadFormat.Json,
MaximumExecutionTime = _Timeout,
ServerTimeout = _Timeout,
};
var batch = new TableBatchOperation();
int count = 0;
foreach (var entity in entities)
{
batch.Add(TableOperation.InsertOrReplace(entity));
count++;
}
if (count > 1)
{
table.ExecuteBatch(batch, options);
}
else
{
if (count == 1)
{
table.Execute(batch[0], options);
}
}
if (exceptionCount != 0)
{
IndexerTrace.RetryWorked();
}
break;
}
catch (Exception ex)
{
IndexerTrace.ErrorWhileImportingEntitiesToAzure(entities.ToArray(), ex);
exceptionCount++;
if (exceptionCount > 5)
{
throw;
}
Thread.Sleep(exceptionCount * 1000);
}
}
}
public long IndexTransactions(ChainBase chain = null)
{
long txCount = 0;
SetThrottling();
BlockingCollection <TransactionEntry.Entity[]> transactions = new BlockingCollection <TransactionEntry.Entity[]>(20);
var tasks = CreateTaskPool(transactions, (txs) => Index(txs), 30);
using (IndexerTrace.NewCorrelation("Import transactions to azure started").Open())
{
Configuration.GetTransactionTable().CreateIfNotExists();
var buckets = new MultiValueDictionary <string, TransactionEntry.Entity>();
using (var storedBlocks = Enumerate("transactions", chain))
{
foreach (var block in storedBlocks)
{
foreach (var transaction in block.Block.Transactions)
{
txCount++;
var indexed = new TransactionEntry.Entity(null, transaction, block.BlockId);
buckets.Add(indexed.PartitionKey, indexed);
var collection = buckets[indexed.PartitionKey];
if (collection.Count == 100)
{
PushTransactions(buckets, collection, transactions);
}
if (storedBlocks.NeedSave)
{
foreach (var kv in buckets.AsLookup().ToArray())
{
PushTransactions(buckets, kv, transactions);
}
tasks.Stop();
storedBlocks.SaveCheckpoint();
tasks.Start();
}
}
}
foreach (var kv in buckets.AsLookup().ToArray())
{
PushTransactions(buckets, kv, transactions);
}
tasks.Stop();
storedBlocks.SaveCheckpoint();
}
}
return(txCount);
}
public int IndexWalletBalances(ChainBase chain)
{
using (IndexerTrace.NewCorrelation("Import wallet balances to azure started"))
{
using (var node = Configuration.ConnectToNode(false))
{
node.VersionHandshake();
var task = new IndexBalanceTask(Configuration, Configuration.CreateIndexerClient().GetAllWalletRules());
task.SaveProgression = !IgnoreCheckpoints;
task.Index(GetBlockFetcher(GetCheckpointInternal(IndexerCheckpoints.Wallets), node, chain), TaskScheduler);
return(task.IndexedEntities);
}
}
}
public long IndexBlocks(ChainBase chain = null)
{
using (IndexerTrace.NewCorrelation("Import blocks to azure started"))
{
using (var node = Configuration.ConnectToNode(false))
{
node.VersionHandshake();
var task = new IndexBlocksTask(Configuration);
task.SaveProgression = !IgnoreCheckpoints;
task.Index(GetBlockFetcher(GetCheckpointInternal(IndexerCheckpoints.Blocks), node, chain), TaskScheduler);
return(task.IndexedBlocks);
}
}
}
public ChainBase GetNodeChain()
{
IndexerTrace.Information("Connecting to node " + Configuration.Node);
using (var node = Configuration.ConnectToNode(false))
{
IndexerTrace.Information("Handshaking");
node.VersionHandshake();
var chain = new ConcurrentChain(Configuration.Network);
IndexerTrace.Information("Synchronizing with local node");
node.SynchronizeChain(chain);
IndexerTrace.Information("Chain loaded with height " + chain.Height);
return(chain);
}
}
public int IndexOrderedBalances(ChainBase chain)
{
using (IndexerTrace.NewCorrelation("Import balances to azure started").Open())
{
using (var node = Configuration.ConnectToNode(false))
{
node.VersionHandshake();
var task = new IndexBalanceTask(Configuration, null);
task.SaveProgression = !IgnoreCheckpoints;
task.Index(GetBlockFetcher(GetCheckpointInternal(IndexerCheckpoints.Balances), node, chain), TaskScheduler);
return(task.IndexedEntities);
}
}
}
public void SaveCheckpoint()
{
if (DisableSaving || _LastProcessed == null)
{
return;
}
_Checkpoint.SaveProgress(_LastProcessed);
IndexerTrace.CheckpointSaved(_LastProcessed, _Checkpoint.FileName);
if (NeedSave)
{
_LastSaved = DateTime.UtcNow;
}
}
public IEnumerator <BlockInfo> GetEnumerator()
{
Queue <DateTime> lastLogs = new Queue <DateTime>();
Queue <int> lastHeights = new Queue <int>();
var locator = DisableSaving ? new BlockLocator(new List <uint256>()
{
_Checkpoint.Genesis
}) : _Checkpoint.BlockLocator;
var fork = _BlockHeaders.FindFork(locator);
var headers = _BlockHeaders.EnumerateAfter(fork);
headers = headers.Where(h => h.Height >= FromHeight && h.Height <= ToHeight);
var first = headers.FirstOrDefault();
if (first == null)
{
yield break;
}
var height = first.Height;
if (first.Height == 1)
{
headers = new[] { fork }.Concat(headers);
height = 0;
}
foreach (var block in _Node.GetBlocks(headers.Select(b => b.HashBlock)))
{
var header = _BlockHeaders.GetBlock(height);
_LastProcessed = header;
yield return(new BlockInfo()
{
Block = block,
BlockId = header.HashBlock,
Height = header.Height
});
IndexerTrace.Processed(height, Math.Min(ToHeight, _BlockHeaders.Tip.Height), lastLogs, lastHeights);
height++;
}
}
/// <summary>
/// Get a block fetcher of the specified chain from the specified checkpoint
/// </summary>
/// <param name="checkpoint">The checkpoint to load from</param>
/// <param name="chain">The chain to fetcher (default: the Node's main chain)</param>
/// <returns>A BlockFetcher for enumerating blocks and saving progression</returns>
public BlockFetcher GetBlockFetcher(Checkpoint checkpoint, Node node, ChainBase chain = null)
{
if (checkpoint == null)
{
throw new ArgumentNullException("checkpoint");
}
if (node == null)
{
throw new ArgumentNullException("node");
}
chain = chain ?? GetNodeChain(node);
IndexerTrace.CheckpointLoaded(chain.FindFork(checkpoint.BlockLocator), checkpoint.CheckpointName);
return(new BlockFetcher(checkpoint, new NodeBlocksRepository(node), chain)
{
NeedSaveInterval = CheckpointInterval,
FromHeight = FromHeight,
ToHeight = ToHeight
});
}
private void Index(List <ChainPartEntry> chainParts)
{
CloudTable table = Configuration.GetChainTable();
TableBatchOperation batch = new TableBatchOperation();
var last = chainParts[chainParts.Count - 1];
foreach (var entry in chainParts)
{
batch.Add(TableOperation.InsertOrReplace(entry.ToEntity()));
if (batch.Count == 100)
{
table.ExecuteBatchAsync(batch).GetAwaiter().GetResult();
batch = new TableBatchOperation();
}
IndexerTrace.RemainingBlockChain(entry.ChainOffset, last.ChainOffset + last.BlockHeaders.Count - 1);
}
if (batch.Count > 0)
{
table.ExecuteBatchAsync(batch).GetAwaiter().GetResult();
}
}
public void IndexChain(ChainBase chain)
{
if (chain == null)
{
throw new ArgumentNullException("chain");
}
SetThrottling();
using (IndexerTrace.NewCorrelation("Index main chain to azure started"))
{
Configuration.GetChainTable().CreateIfNotExistsAsync().GetAwaiter().GetResult();
IndexerTrace.InputChainTip(chain.Tip);
var client = Configuration.CreateIndexerClient();
var changes = client.GetChainChangesUntilFork(chain.Tip, true).ToList();
var height = 0;
if (changes.Count != 0)
{
IndexerTrace.IndexedChainTip(changes[0].BlockId, changes[0].Height);
if (changes[0].Height > chain.Tip.Height)
{
IndexerTrace.InputChainIsLate();
return;
}
height = changes[changes.Count - 1].Height + 1;
if (height > chain.Height)
{
IndexerTrace.IndexedChainIsUpToDate(chain.Tip);
return;
}
}
else
{
IndexerTrace.NoForkFoundWithStored();
}
IndexerTrace.IndexingChain(chain.GetBlock(height), chain.Tip);
Index(chain, height);
}
}
public IEnumerator <BlockInfo> GetEnumerator()
{
Queue <DateTime> lastLogs = new Queue <DateTime>();
Queue <int> lastHeights = new Queue <int>();
var fork = _BlockHeaders.FindFork(_Checkpoint.BlockLocator);
var headers = _BlockHeaders.EnumerateAfter(fork);
headers = headers.Where(h => h.Height >= FromHeight && h.Height <= ToHeight);
var first = headers.FirstOrDefault();
if (first == null)
{
yield break;
}
var height = first.Height;
if (first.Height == 1)
{
headers = new[] { fork }.Concat(headers);
height = 0;
}
foreach (var block in _BlocksRepository.GetBlocks(headers.Select(b => b.HashBlock), CancellationToken).TakeWhile(b => b != null))
{
var header = _BlockHeaders.GetBlock(height);
_LastProcessed = header;
yield return(new BlockInfo()
{
Block = block,
BlockId = header.HashBlock,
Height = header.Height
});
IndexerTrace.Processed(height, Math.Min(ToHeight, _BlockHeaders.Tip.Height), lastLogs, lastHeights);
height++;
}
}
public async Task <TransactionEntry> GetTransactionAsync(bool loadPreviousOutput, bool fetchColor, uint256 txId)
{
if (txId == null)
{
return(null);
}
TransactionEntry result = null;
var table = Configuration.GetTransactionTable();
var searchedEntity = new TransactionEntry.Entity(txId);
var query = new TableQuery <DynamicTableEntity>()
.Where(
TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, searchedEntity.PartitionKey),
TableOperators.And,
TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThan, txId.ToString() + "-"),
TableOperators.And,
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThan, txId.ToString() + "|")
)
));
query.TakeCount = 10; //Should not have more
List <TransactionEntry.Entity> entities = new List <TransactionEntry.Entity>();
foreach (var e in await table.ExecuteQuerySegmentedAsync(query, null).ConfigureAwait(false))
{
if (e.IsFat())
{
entities.Add(new TransactionEntry.Entity(await FetchFatEntity(e).ConfigureAwait(false), ConsensuFactory));
}
else
{
entities.Add(new TransactionEntry.Entity(e, ConsensuFactory));
}
}
if (entities.Count == 0)
{
result = null;
}
else
{
result = new TransactionEntry(entities.ToArray());
if (result.Transaction == null)
{
foreach (var block in result.BlockIds.Select(id => GetBlock(id)).Where(b => b != null))
{
result.Transaction = block.Transactions.FirstOrDefault(t => t.GetHash() == txId);
entities[0].Transaction = result.Transaction;
if (entities[0].Transaction != null)
{
await UpdateEntity(table, entities[0].CreateTableEntity()).ConfigureAwait(false);
}
break;
}
}
if (fetchColor && result.ColoredTransaction == null)
{
result.ColoredTransaction = await ColoredTransaction.FetchColorsAsync(txId, result.Transaction, new CachedColoredTransactionRepository(new IndexerColoredTransactionRepository(Configuration))).ConfigureAwait(false);
entities[0].ColoredTransaction = result.ColoredTransaction;
if (entities[0].ColoredTransaction != null)
{
await UpdateEntity(table, entities[0].CreateTableEntity()).ConfigureAwait(false);
}
}
var needTxOut = result.SpentCoins == null && loadPreviousOutput && result.Transaction != null;
if (needTxOut)
{
var inputs = result.Transaction.Inputs.Select(o => o.PrevOut).ToArray();
var parents = await
GetTransactionsAsync(false, false, inputs
.Select(i => i.Hash)
.ToArray()).ConfigureAwait(false);
for (int i = 0; i < parents.Length; i++)
{
if (parents[i] == null)
{
IndexerTrace.MissingTransactionFromDatabase(result.Transaction.Inputs[i].PrevOut.Hash);
return(null);
}
}
var outputs = parents.Select((p, i) => p.Transaction.Outputs[inputs[i].N]).ToArray();
result.SpentCoins = Enumerable
.Range(0, inputs.Length)
.Select(i => new Spendable(inputs[i], outputs[i]))
.ToList();
entities[0].PreviousTxOuts.Clear();
entities[0].PreviousTxOuts.AddRange(outputs);
if (entities[0].IsLoaded)
{
await UpdateEntity(table, entities[0].CreateTableEntity()).ConfigureAwait(false);
}
}
}
return(result != null && result.Transaction != null ? result : null);
}
private void IndexBalances(ChainBase chain, string checkpointName, Func <uint256, Transaction, uint256, BlockHeader, int, IEnumerable <OrderedBalanceChange> > extract)
{
SetThrottling();
BlockingCollection <OrderedBalanceChange[]> indexedEntries = new BlockingCollection <OrderedBalanceChange[]>(100);
var tasks = CreateTaskPool(indexedEntries, (entries) => Index(entries.Select(e => e.ToEntity()), this.Configuration.GetBalanceTable()), 30);
using (IndexerTrace.NewCorrelation("Import balances " + checkpointName + " to azure started").Open())
{
this.Configuration.GetBalanceTable().CreateIfNotExists();
var buckets = new MultiValueDictionary <string, OrderedBalanceChange>();
using (var storedBlocks = Enumerate(checkpointName, chain))
{
foreach (var block in storedBlocks)
{
foreach (var tx in block.Block.Transactions)
{
var txId = tx.GetHash();
try
{
var entries = extract(txId, tx, block.BlockId, block.Block.Header, block.Height);
foreach (var entry in entries)
{
buckets.Add(entry.PartitionKey, entry);
var bucket = buckets[entry.PartitionKey];
if (bucket.Count == 100)
{
indexedEntries.Add(bucket.ToArray());
buckets.Remove(entry.PartitionKey);
}
}
if (storedBlocks.NeedSave)
{
foreach (var kv in buckets.AsLookup().ToArray())
{
indexedEntries.Add(kv.ToArray());
}
buckets.Clear();
tasks.Stop();
storedBlocks.SaveCheckpoint();
tasks.Start();
}
}
catch (Exception ex)
{
IndexerTrace.ErrorWhileImportingBalancesToAzure(ex, txId);
throw;
}
}
}
foreach (var kv in buckets.AsLookup().ToArray())
{
indexedEntries.Add(kv.ToArray());
}
tasks.Stop();
storedBlocks.SaveCheckpoint();
}
}
}
public async Task <TransactionEntry> GetTransactionAsync(bool lazyLoadPreviousOutput, bool fetchColor, uint256 txId)
{
if (txId == null)
{
return(null);
}
TransactionEntry result = null;
var table = Configuration.GetTransactionTable();
var searchedEntity = new TransactionEntry.Entity(txId);
var query = new TableQuery()
.Where(
TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, searchedEntity.PartitionKey),
TableOperators.And,
TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThan, txId.ToString() + "-"),
TableOperators.And,
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThan, txId.ToString() + "|")
)
));
query.TakeCount = 10; //Should not have more
var entities = (await table.ExecuteQuerySegmentedAsync(query, null).ConfigureAwait(false))
.Select(e => new TransactionEntry.Entity(e)).ToArray();
if (entities.Length == 0)
{
result = null;
}
else
{
result = new TransactionEntry(entities);
if (result.Transaction == null)
{
foreach (var block in result.BlockIds.Select(id => GetBlock(id)).Where(b => b != null))
{
result.Transaction = block.Transactions.FirstOrDefault(t => t.GetHash() == txId);
entities[0].Transaction = result.Transaction;
if (entities[0].Transaction != null)
{
await table.ExecuteAsync(TableOperation.Merge(entities[0].CreateTableEntity())).ConfigureAwait(false);
}
break;
}
}
if (fetchColor && result.ColoredTransaction == null)
{
result.ColoredTransaction = ColoredTransaction.FetchColors(txId, result.Transaction, new IndexerColoredTransactionRepository(Configuration));
entities[0].ColoredTransaction = result.ColoredTransaction;
if (entities[0].ColoredTransaction != null)
{
await table.ExecuteAsync(TableOperation.Merge(entities[0].CreateTableEntity())).ConfigureAwait(false);
}
}
var needTxOut = result.SpentCoins == null && lazyLoadPreviousOutput && result.Transaction != null;
if (needTxOut)
{
var tasks =
result.Transaction
.Inputs
.Select(async txin =>
{
var parentTx = await GetTransactionAsync(false, false, txin.PrevOut.Hash).ConfigureAwait(false);
if (parentTx == null)
{
IndexerTrace.MissingTransactionFromDatabase(txin.PrevOut.Hash);
return(null);
}
return(parentTx.Transaction.Outputs[(int)txin.PrevOut.N]);
})
.ToArray();
await Task.WhenAll(tasks).ConfigureAwait(false);
if (tasks.All(t => t.Result != null))
{
var outputs = tasks.Select(t => t.Result).ToArray();
result.SpentCoins = outputs.Select((o, n) => new Spendable(result.Transaction.Inputs[n].PrevOut, o)).ToList();
entities[0].PreviousTxOuts.Clear();
entities[0].PreviousTxOuts.AddRange(outputs);
if (entities[0].IsLoaded)
{
await table.ExecuteAsync(TableOperation.Merge(entities[0].CreateTableEntity())).ConfigureAwait(false);
}
}
}
if (result.Transaction == null)
{
result = null;
}
}
return(result);
}
public void Index(Block block)
{
var hash = block.GetHash().ToString();
using (IndexerTrace.NewCorrelation("Upload of " + hash).Open())
{
Stopwatch watch = new Stopwatch();
watch.Start();
bool failedBefore = false;
while (true)
{
try
{
var container = Configuration.GetBlocksContainer();
var client = container.ServiceClient;
client.DefaultRequestOptions.SingleBlobUploadThresholdInBytes = 32 * 1024 * 1024;
var blob = container.GetPageBlobReference(hash);
MemoryStream ms = new MemoryStream();
block.ReadWrite(ms, true);
var blockBytes = ms.GetBuffer();
long length = 512 - (ms.Length % 512);
if (length == 512)
{
length = 0;
}
Array.Resize(ref blockBytes, (int)(ms.Length + length));
try
{
blob.UploadFromByteArray(blockBytes, 0, blockBytes.Length, new AccessCondition()
{
//Will throw if already exist, save 1 call
IfNotModifiedSinceTime = failedBefore ? (DateTimeOffset?)null : DateTimeOffset.MinValue
}, new BlobRequestOptions()
{
MaximumExecutionTime = _Timeout,
ServerTimeout = _Timeout
});
watch.Stop();
IndexerTrace.BlockUploaded(watch.Elapsed, blockBytes.Length);
break;
}
catch (StorageException ex)
{
var alreadyExist = ex.RequestInformation != null && ex.RequestInformation.HttpStatusCode == 412;
if (!alreadyExist)
{
throw;
}
watch.Stop();
IndexerTrace.BlockAlreadyUploaded();
break;
}
}
catch (Exception ex)
{
IndexerTrace.ErrorWhileImportingBlockToAzure(new uint256(hash), ex);
failedBefore = true;
Thread.Sleep(5000);
}
}
}
}
