public long IndexTransactions(ChainBase chain = null) { long txCount = 0; SetThrottling(); BlockingCollection <TransactionEntry.Entity[]> transactions = new BlockingCollection <TransactionEntry.Entity[]>(20); var tasks = CreateTaskPool(transactions, (txs) => Index(txs), 30); using (IndexerTrace.NewCorrelation("Import transactions to azure started").Open()) { Configuration.GetTransactionTable().CreateIfNotExists(); var buckets = new MultiValueDictionary <string, TransactionEntry.Entity>(); using (var storedBlocks = Enumerate("transactions", chain)) { foreach (var block in storedBlocks) { foreach (var transaction in block.Block.Transactions) { txCount++; var indexed = new TransactionEntry.Entity(null, transaction, block.BlockId); buckets.Add(indexed.PartitionKey, indexed); var collection = buckets[indexed.PartitionKey]; if (collection.Count == 100) { PushTransactions(buckets, collection, transactions); } if (storedBlocks.NeedSave) { foreach (var kv in buckets.AsLookup().ToArray()) { PushTransactions(buckets, kv, transactions); } tasks.Stop(); storedBlocks.SaveCheckpoint(); tasks.Start(); } } } foreach (var kv in buckets.AsLookup().ToArray()) { PushTransactions(buckets, kv, transactions); } tasks.Stop(); storedBlocks.SaveCheckpoint(); } } return(txCount); }
public async Task<TransactionEntry> GetTransactionAsync(bool loadPreviousOutput, bool fetchColor, uint256 txId) { if(txId == null) return null; TransactionEntry result = null; var table = Configuration.GetTransactionTable(); var searchedEntity = new TransactionEntry.Entity(txId); var query = new TableQuery() .Where( TableQuery.CombineFilters( TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, searchedEntity.PartitionKey), TableOperators.And, TableQuery.CombineFilters( TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThan, txId.ToString() + "-"), TableOperators.And, TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThan, txId.ToString() + "|") ) )); query.TakeCount = 10; //Should not have more List<TransactionEntry.Entity> entities = new List<TransactionEntry.Entity>(); foreach(var e in await table.ExecuteQuerySegmentedAsync(query, null).ConfigureAwait(false)) { if(e.IsFat()) entities.Add(new TransactionEntry.Entity(await FetchFatEntity(e).ConfigureAwait(false))); else entities.Add(new TransactionEntry.Entity(e)); } if(entities.Count == 0) result = null; else { result = new TransactionEntry(entities.ToArray()); if(result.Transaction == null) { foreach(var block in result.BlockIds.Select(id => GetBlock(id)).Where(b => b != null)) { result.Transaction = block.Transactions.FirstOrDefault(t => t.GetHash() == txId); entities[0].Transaction = result.Transaction; if(entities[0].Transaction != null) { await UpdateEntity(table, entities[0].CreateTableEntity()).ConfigureAwait(false); } break; } } if(fetchColor && result.ColoredTransaction == null) { result.ColoredTransaction = await ColoredTransaction.FetchColorsAsync(txId, result.Transaction, new CachedColoredTransactionRepository(new IndexerColoredTransactionRepository(Configuration))).ConfigureAwait(false); entities[0].ColoredTransaction = result.ColoredTransaction; if(entities[0].ColoredTransaction != null) { await UpdateEntity(table, entities[0].CreateTableEntity()).ConfigureAwait(false); } } var needTxOut = result.SpentCoins == null && loadPreviousOutput && result.Transaction != null; if(needTxOut) { var inputs = result.Transaction.Inputs.Select(o => o.PrevOut).ToArray(); var parents = await GetTransactionsAsync(false, false, inputs .Select(i => i.Hash) .ToArray()).ConfigureAwait(false); for(int i = 0; i < parents.Length; i++) { if(parents[i] == null) { IndexerTrace.MissingTransactionFromDatabase(result.Transaction.Inputs[i].PrevOut.Hash); return null; } } var outputs = parents.Select((p, i) => p.Transaction.Outputs[inputs[i].N]).ToArray(); result.SpentCoins = Enumerable .Range(0, inputs.Length) .Select(i => new Spendable(inputs[i], outputs[i])) .ToList(); entities[0].PreviousTxOuts.Clear(); entities[0].PreviousTxOuts.AddRange(outputs); if(entities[0].IsLoaded) { await UpdateEntity(table, entities[0].CreateTableEntity()).ConfigureAwait(false); } } } return result != null && result.Transaction != null ? result : null; }
public async Task <TransactionEntry> GetTransactionAsync(bool loadPreviousOutput, bool fetchColor, uint256 txId) { if (txId == null) { return(null); } TransactionEntry result = null; var table = Configuration.GetTransactionTable(); var searchedEntity = new TransactionEntry.Entity(txId); var query = new TableQuery <DynamicTableEntity>() .Where( TableQuery.CombineFilters( TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, searchedEntity.PartitionKey), TableOperators.And, TableQuery.CombineFilters( TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThan, txId.ToString() + "-"), TableOperators.And, TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThan, txId.ToString() + "|") ) )); query.TakeCount = 10; //Should not have more List <TransactionEntry.Entity> entities = new List <TransactionEntry.Entity>(); foreach (var e in await table.ExecuteQuerySegmentedAsync(query, null).ConfigureAwait(false)) { if (e.IsFat()) { entities.Add(new TransactionEntry.Entity(await FetchFatEntity(e).ConfigureAwait(false), ConsensuFactory)); } else { entities.Add(new TransactionEntry.Entity(e, ConsensuFactory)); } } if (entities.Count == 0) { result = null; } else { result = new TransactionEntry(entities.ToArray()); if (result.Transaction == null) { foreach (var block in result.BlockIds.Select(id => GetBlock(id)).Where(b => b != null)) { result.Transaction = block.Transactions.FirstOrDefault(t => t.GetHash() == txId); entities[0].Transaction = result.Transaction; if (entities[0].Transaction != null) { await UpdateEntity(table, entities[0].CreateTableEntity()).ConfigureAwait(false); } break; } } if (fetchColor && result.ColoredTransaction == null) { result.ColoredTransaction = await ColoredTransaction.FetchColorsAsync(txId, result.Transaction, new CachedColoredTransactionRepository(new IndexerColoredTransactionRepository(Configuration))).ConfigureAwait(false); entities[0].ColoredTransaction = result.ColoredTransaction; if (entities[0].ColoredTransaction != null) { await UpdateEntity(table, entities[0].CreateTableEntity()).ConfigureAwait(false); } } var needTxOut = result.SpentCoins == null && loadPreviousOutput && result.Transaction != null; if (needTxOut) { var inputs = result.Transaction.Inputs.Select(o => o.PrevOut).ToArray(); var parents = await GetTransactionsAsync(false, false, inputs .Select(i => i.Hash) .ToArray()).ConfigureAwait(false); for (int i = 0; i < parents.Length; i++) { if (parents[i] == null) { IndexerTrace.MissingTransactionFromDatabase(result.Transaction.Inputs[i].PrevOut.Hash); return(null); } } var outputs = parents.Select((p, i) => p.Transaction.Outputs[inputs[i].N]).ToArray(); result.SpentCoins = Enumerable .Range(0, inputs.Length) .Select(i => new Spendable(inputs[i], outputs[i])) .ToList(); entities[0].PreviousTxOuts.Clear(); entities[0].PreviousTxOuts.AddRange(outputs); if (entities[0].IsLoaded) { await UpdateEntity(table, entities[0].CreateTableEntity()).ConfigureAwait(false); } } } return(result != null && result.Transaction != null ? result : null); }
public async Task <TransactionEntry> GetTransactionAsync(bool lazyLoadPreviousOutput, bool fetchColor, uint256 txId) { if (txId == null) { return(null); } TransactionEntry result = null; var table = Configuration.GetTransactionTable(); var searchedEntity = new TransactionEntry.Entity(txId); var query = new TableQuery() .Where( TableQuery.CombineFilters( TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, searchedEntity.PartitionKey), TableOperators.And, TableQuery.CombineFilters( TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThan, txId.ToString() + "-"), TableOperators.And, TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThan, txId.ToString() + "|") ) )); query.TakeCount = 10; //Should not have more var entities = (await table.ExecuteQuerySegmentedAsync(query, null).ConfigureAwait(false)) .Select(e => new TransactionEntry.Entity(e)).ToArray(); if (entities.Length == 0) { result = null; } else { result = new TransactionEntry(entities); if (result.Transaction == null) { foreach (var block in result.BlockIds.Select(id => GetBlock(id)).Where(b => b != null)) { result.Transaction = block.Transactions.FirstOrDefault(t => t.GetHash() == txId); entities[0].Transaction = result.Transaction; if (entities[0].Transaction != null) { await table.ExecuteAsync(TableOperation.Merge(entities[0].CreateTableEntity())).ConfigureAwait(false); } break; } } if (fetchColor && result.ColoredTransaction == null) { result.ColoredTransaction = ColoredTransaction.FetchColors(txId, result.Transaction, new IndexerColoredTransactionRepository(Configuration)); entities[0].ColoredTransaction = result.ColoredTransaction; if (entities[0].ColoredTransaction != null) { await table.ExecuteAsync(TableOperation.Merge(entities[0].CreateTableEntity())).ConfigureAwait(false); } } var needTxOut = result.SpentCoins == null && lazyLoadPreviousOutput && result.Transaction != null; if (needTxOut) { var tasks = result.Transaction .Inputs .Select(async txin => { var parentTx = await GetTransactionAsync(false, false, txin.PrevOut.Hash).ConfigureAwait(false); if (parentTx == null) { IndexerTrace.MissingTransactionFromDatabase(txin.PrevOut.Hash); return(null); } return(parentTx.Transaction.Outputs[(int)txin.PrevOut.N]); }) .ToArray(); await Task.WhenAll(tasks).ConfigureAwait(false); if (tasks.All(t => t.Result != null)) { var outputs = tasks.Select(t => t.Result).ToArray(); result.SpentCoins = outputs.Select((o, n) => new Spendable(result.Transaction.Inputs[n].PrevOut, o)).ToList(); entities[0].PreviousTxOuts.Clear(); entities[0].PreviousTxOuts.AddRange(outputs); if (entities[0].IsLoaded) { await table.ExecuteAsync(TableOperation.Merge(entities[0].CreateTableEntity())).ConfigureAwait(false); } } } if (result.Transaction == null) { result = null; } } return(result); }