public long IndexTransactions(ChainBase chain = null)
{
long txCount = 0;
SetThrottling();
BlockingCollection <TransactionEntry.Entity[]> transactions = new BlockingCollection <TransactionEntry.Entity[]>(20);
var tasks = CreateTaskPool(transactions, (txs) => Index(txs), 30);
using (IndexerTrace.NewCorrelation("Import transactions to azure started").Open())
{
Configuration.GetTransactionTable().CreateIfNotExists();
var buckets = new MultiValueDictionary <string, TransactionEntry.Entity>();
using (var storedBlocks = Enumerate("transactions", chain))
{
foreach (var block in storedBlocks)
{
foreach (var transaction in block.Block.Transactions)
{
txCount++;
var indexed = new TransactionEntry.Entity(null, transaction, block.BlockId);
buckets.Add(indexed.PartitionKey, indexed);
var collection = buckets[indexed.PartitionKey];
if (collection.Count == 100)
{
PushTransactions(buckets, collection, transactions);
}
if (storedBlocks.NeedSave)
{
foreach (var kv in buckets.AsLookup().ToArray())
{
PushTransactions(buckets, kv, transactions);
}
tasks.Stop();
storedBlocks.SaveCheckpoint();
tasks.Start();
}
}
}
foreach (var kv in buckets.AsLookup().ToArray())
{
PushTransactions(buckets, kv, transactions);
}
tasks.Stop();
storedBlocks.SaveCheckpoint();
}
}
return(txCount);
}
public async Task<TransactionEntry> GetTransactionAsync(bool loadPreviousOutput, bool fetchColor, uint256 txId)
{
if(txId == null)
return null;
TransactionEntry result = null;
var table = Configuration.GetTransactionTable();
var searchedEntity = new TransactionEntry.Entity(txId);
var query = new TableQuery()
.Where(
TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, searchedEntity.PartitionKey),
TableOperators.And,
TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThan, txId.ToString() + "-"),
TableOperators.And,
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThan, txId.ToString() + "|")
)
));
query.TakeCount = 10; //Should not have more
List<TransactionEntry.Entity> entities = new List<TransactionEntry.Entity>();
foreach(var e in await table.ExecuteQuerySegmentedAsync(query, null).ConfigureAwait(false))
{
if(e.IsFat())
entities.Add(new TransactionEntry.Entity(await FetchFatEntity(e).ConfigureAwait(false)));
else
entities.Add(new TransactionEntry.Entity(e));
}
if(entities.Count == 0)
result = null;
else
{
result = new TransactionEntry(entities.ToArray());
if(result.Transaction == null)
{
foreach(var block in result.BlockIds.Select(id => GetBlock(id)).Where(b => b != null))
{
result.Transaction = block.Transactions.FirstOrDefault(t => t.GetHash() == txId);
entities[0].Transaction = result.Transaction;
if(entities[0].Transaction != null)
{
await UpdateEntity(table, entities[0].CreateTableEntity()).ConfigureAwait(false);
}
break;
}
}
if(fetchColor && result.ColoredTransaction == null)
{
result.ColoredTransaction = await ColoredTransaction.FetchColorsAsync(txId, result.Transaction, new CachedColoredTransactionRepository(new IndexerColoredTransactionRepository(Configuration))).ConfigureAwait(false);
entities[0].ColoredTransaction = result.ColoredTransaction;
if(entities[0].ColoredTransaction != null)
{
await UpdateEntity(table, entities[0].CreateTableEntity()).ConfigureAwait(false);
}
}
var needTxOut = result.SpentCoins == null && loadPreviousOutput && result.Transaction != null;
if(needTxOut)
{
var inputs = result.Transaction.Inputs.Select(o => o.PrevOut).ToArray();
var parents = await
GetTransactionsAsync(false, false, inputs
.Select(i => i.Hash)
.ToArray()).ConfigureAwait(false);
for(int i = 0; i < parents.Length; i++)
{
if(parents[i] == null)
{
IndexerTrace.MissingTransactionFromDatabase(result.Transaction.Inputs[i].PrevOut.Hash);
return null;
}
}
var outputs = parents.Select((p, i) => p.Transaction.Outputs[inputs[i].N]).ToArray();
result.SpentCoins = Enumerable
.Range(0, inputs.Length)
.Select(i => new Spendable(inputs[i], outputs[i]))
.ToList();
entities[0].PreviousTxOuts.Clear();
entities[0].PreviousTxOuts.AddRange(outputs);
if(entities[0].IsLoaded)
{
await UpdateEntity(table, entities[0].CreateTableEntity()).ConfigureAwait(false);
}
}
}
return result != null && result.Transaction != null ? result : null;
}
public async Task <TransactionEntry> GetTransactionAsync(bool loadPreviousOutput, bool fetchColor, uint256 txId)
{
if (txId == null)
{
return(null);
}
TransactionEntry result = null;
var table = Configuration.GetTransactionTable();
var searchedEntity = new TransactionEntry.Entity(txId);
var query = new TableQuery <DynamicTableEntity>()
.Where(
TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, searchedEntity.PartitionKey),
TableOperators.And,
TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThan, txId.ToString() + "-"),
TableOperators.And,
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThan, txId.ToString() + "|")
)
));
query.TakeCount = 10; //Should not have more
List <TransactionEntry.Entity> entities = new List <TransactionEntry.Entity>();
foreach (var e in await table.ExecuteQuerySegmentedAsync(query, null).ConfigureAwait(false))
{
if (e.IsFat())
{
entities.Add(new TransactionEntry.Entity(await FetchFatEntity(e).ConfigureAwait(false), ConsensuFactory));
}
else
{
entities.Add(new TransactionEntry.Entity(e, ConsensuFactory));
}
}
if (entities.Count == 0)
{
result = null;
}
else
{
result = new TransactionEntry(entities.ToArray());
if (result.Transaction == null)
{
foreach (var block in result.BlockIds.Select(id => GetBlock(id)).Where(b => b != null))
{
result.Transaction = block.Transactions.FirstOrDefault(t => t.GetHash() == txId);
entities[0].Transaction = result.Transaction;
if (entities[0].Transaction != null)
{
await UpdateEntity(table, entities[0].CreateTableEntity()).ConfigureAwait(false);
}
break;
}
}
if (fetchColor && result.ColoredTransaction == null)
{
result.ColoredTransaction = await ColoredTransaction.FetchColorsAsync(txId, result.Transaction, new CachedColoredTransactionRepository(new IndexerColoredTransactionRepository(Configuration))).ConfigureAwait(false);
entities[0].ColoredTransaction = result.ColoredTransaction;
if (entities[0].ColoredTransaction != null)
{
await UpdateEntity(table, entities[0].CreateTableEntity()).ConfigureAwait(false);
}
}
var needTxOut = result.SpentCoins == null && loadPreviousOutput && result.Transaction != null;
if (needTxOut)
{
var inputs = result.Transaction.Inputs.Select(o => o.PrevOut).ToArray();
var parents = await
GetTransactionsAsync(false, false, inputs
.Select(i => i.Hash)
.ToArray()).ConfigureAwait(false);
for (int i = 0; i < parents.Length; i++)
{
if (parents[i] == null)
{
IndexerTrace.MissingTransactionFromDatabase(result.Transaction.Inputs[i].PrevOut.Hash);
return(null);
}
}
var outputs = parents.Select((p, i) => p.Transaction.Outputs[inputs[i].N]).ToArray();
result.SpentCoins = Enumerable
.Range(0, inputs.Length)
.Select(i => new Spendable(inputs[i], outputs[i]))
.ToList();
entities[0].PreviousTxOuts.Clear();
entities[0].PreviousTxOuts.AddRange(outputs);
if (entities[0].IsLoaded)
{
await UpdateEntity(table, entities[0].CreateTableEntity()).ConfigureAwait(false);
}
}
}
return(result != null && result.Transaction != null ? result : null);
}
public async Task <TransactionEntry> GetTransactionAsync(bool lazyLoadPreviousOutput, bool fetchColor, uint256 txId)
{
if (txId == null)
{
return(null);
}
TransactionEntry result = null;
var table = Configuration.GetTransactionTable();
var searchedEntity = new TransactionEntry.Entity(txId);
var query = new TableQuery()
.Where(
TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, searchedEntity.PartitionKey),
TableOperators.And,
TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThan, txId.ToString() + "-"),
TableOperators.And,
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThan, txId.ToString() + "|")
)
));
query.TakeCount = 10; //Should not have more
var entities = (await table.ExecuteQuerySegmentedAsync(query, null).ConfigureAwait(false))
.Select(e => new TransactionEntry.Entity(e)).ToArray();
if (entities.Length == 0)
{
result = null;
}
else
{
result = new TransactionEntry(entities);
if (result.Transaction == null)
{
foreach (var block in result.BlockIds.Select(id => GetBlock(id)).Where(b => b != null))
{
result.Transaction = block.Transactions.FirstOrDefault(t => t.GetHash() == txId);
entities[0].Transaction = result.Transaction;
if (entities[0].Transaction != null)
{
await table.ExecuteAsync(TableOperation.Merge(entities[0].CreateTableEntity())).ConfigureAwait(false);
}
break;
}
}
if (fetchColor && result.ColoredTransaction == null)
{
result.ColoredTransaction = ColoredTransaction.FetchColors(txId, result.Transaction, new IndexerColoredTransactionRepository(Configuration));
entities[0].ColoredTransaction = result.ColoredTransaction;
if (entities[0].ColoredTransaction != null)
{
await table.ExecuteAsync(TableOperation.Merge(entities[0].CreateTableEntity())).ConfigureAwait(false);
}
}
var needTxOut = result.SpentCoins == null && lazyLoadPreviousOutput && result.Transaction != null;
if (needTxOut)
{
var tasks =
result.Transaction
.Inputs
.Select(async txin =>
{
var parentTx = await GetTransactionAsync(false, false, txin.PrevOut.Hash).ConfigureAwait(false);
if (parentTx == null)
{
IndexerTrace.MissingTransactionFromDatabase(txin.PrevOut.Hash);
return(null);
}
return(parentTx.Transaction.Outputs[(int)txin.PrevOut.N]);
})
.ToArray();
await Task.WhenAll(tasks).ConfigureAwait(false);
if (tasks.All(t => t.Result != null))
{
var outputs = tasks.Select(t => t.Result).ToArray();
result.SpentCoins = outputs.Select((o, n) => new Spendable(result.Transaction.Inputs[n].PrevOut, o)).ToList();
entities[0].PreviousTxOuts.Clear();
entities[0].PreviousTxOuts.AddRange(outputs);
if (entities[0].IsLoaded)
{
await table.ExecuteAsync(TableOperation.Merge(entities[0].CreateTableEntity())).ConfigureAwait(false);
}
}
}
if (result.Transaction == null)
{
result = null;
}
}
return(result);
}