private void EnqueueRange(ChainBase chain, ChainedBlock startCumul, int blockCount)
{
ListenerTrace.Info("Enqueing from " + startCumul.Height + " " + blockCount + " blocks");
if (blockCount == 0)
{
return;
}
var tasks = _IndexTasks
.Where(t => chain.FindFork(t.Value.Item1.BlockLocator).Height <= startCumul.Height + blockCount)
.Select(t => new BlockRange()
{
From = startCumul.Height,
Count = blockCount,
Target = t.Key
})
.Select(t => _Conf.Topics.InitialIndexing.AddAsync(t))
.ToArray();
try
{
Task.WaitAll(tasks);
}
catch (AggregateException aex)
{
ExceptionDispatchInfo.Capture(aex.InnerException).Throw();
throw;
}
}
/// <summary>
/// Returns the first common chained block header between two chains.
/// </summary>
/// <param name="chainSrc">The source chain.</param>
/// <param name="otherChain">The other chain.</param>
/// <returns>First common chained block header or <c>null</c>.</returns>
private ChainedBlock FindFork(ChainBase chainSrc, ChainBase otherChain)
{
if (otherChain == null)
{
throw new ArgumentNullException("otherChain");
}
return(chainSrc.FindFork(otherChain.Tip.EnumerateToGenesis().Select(o => o.HashBlock)));
}
public IEnumerator <BlockInfo> GetEnumerator()
{
Queue <DateTime> lastLogs = new Queue <DateTime>();
Queue <int> lastHeights = new Queue <int>();
var fork = _BlockHeaders.FindFork(_Checkpoint.BlockLocator);
var headers = _BlockHeaders.EnumerateAfter(fork);
headers = headers.Where(h => h.Height <= ToHeight);
var first = headers.FirstOrDefault();
if (first == null)
{
yield break;
}
var height = first.Height;
if (first.Height == 1)
{
headers = new[] { fork }.Concat(headers);
height = 0;
}
foreach (var block in _BlocksRepository.GetBlocks(headers.Select(b => b.HashBlock), CancellationToken))
{
var header = _BlockHeaders.GetBlock(height);
if (block == null)
{
var storeTip = _BlocksRepository.GetStoreTip();
if (storeTip != null)
{
// Store is caught up with Chain but the block is missing from the store.
if (header.Header.BlockTime <= storeTip.Header.BlockTime)
{
throw new InvalidOperationException($"Chained block not found in store (height = { height }). Re-create the block store.");
}
}
// Allow Store to catch up with Chain.
break;
}
_LastProcessed = header;
yield return(new BlockInfo()
{
Block = block,
BlockId = header.HashBlock,
Height = header.Height
});
IndexerTrace.Processed(height, Math.Min(ToHeight, _BlockHeaders.Tip.Height), lastLogs, lastHeights);
height++;
}
}
public IEnumerator <BlockInfo> GetEnumerator()
{
Queue <DateTime> lastLogs = new Queue <DateTime>();
Queue <int> lastHeights = new Queue <int>();
var locator = DisableSaving ? new BlockLocator(new List <uint256>()
{
_Checkpoint.Genesis
}) : _Checkpoint.BlockLocator;
var fork = _BlockHeaders.FindFork(locator);
var headers = _BlockHeaders.EnumerateAfter(fork);
headers = headers.Where(h => h.Height >= FromHeight && h.Height <= ToHeight);
var first = headers.FirstOrDefault();
if (first == null)
{
yield break;
}
var height = first.Height;
if (first.Height == 1)
{
headers = new[] { fork }.Concat(headers);
height = 0;
}
foreach (var block in _Node.GetBlocks(headers.Select(b => b.HashBlock)))
{
var header = _BlockHeaders.GetBlock(height);
_LastProcessed = header;
yield return(new BlockInfo()
{
Block = block,
BlockId = header.HashBlock,
Height = header.Height
});
IndexerTrace.Processed(height, Math.Min(ToHeight, _BlockHeaders.Tip.Height), lastLogs, lastHeights);
height++;
}
}
/// <summary>
/// Get a block fetcher of the specified chain from the specified checkpoint
/// </summary>
/// <param name="checkpoint">The checkpoint to load from</param>
/// <param name="chain">The chain to fetcher (default: the Node's main chain)</param>
/// <returns>A BlockFetcher for enumerating blocks and saving progression</returns>
public BlockFetcher GetBlockFetcher(Checkpoint checkpoint, Node node, ChainBase chain = null)
{
if (checkpoint == null)
{
throw new ArgumentNullException("checkpoint");
}
if (node == null)
{
throw new ArgumentNullException("node");
}
chain = chain ?? this.GetNodeChain(node);
IndexerTrace.CheckpointLoaded(chain.FindFork(checkpoint.BlockLocator), checkpoint.CheckpointName);
return(new BlockFetcher(checkpoint, new NodeBlocksRepository(node), chain)
{
NeedSaveInterval = this.CheckpointInterval,
FromHeight = this.FromHeight,
ToHeight = this.ToHeight
});
}
public IEnumerator <BlockInfo> GetEnumerator()
{
Queue <DateTime> lastLogs = new Queue <DateTime>();
Queue <int> lastHeights = new Queue <int>();
var fork = _BlockHeaders.FindFork(_Checkpoint.BlockLocator);
_LastProcessed = fork;
var headers = _BlockHeaders.EnumerateAfter(fork);
headers = headers.Where(h => h.Height >= FromHeight && h.Height <= ToHeight);
var first = headers.FirstOrDefault();
if (first == null)
{
yield break;
}
var height = first.Height;
if (first.Height == 1)
{
headers = new[] { fork }.Concat(headers);
height = 0;
}
foreach (var block in _BlocksRepository.GetBlocks(headers.Select(b => b.HashBlock), CancellationToken).TakeWhile(b => b != null))
{
var header = _BlockHeaders.GetBlock(height);
_LastProcessed = header;
yield return(new BlockInfo()
{
Block = block,
BlockId = header.HashBlock,
Height = header.Height
});
IndexerTrace.Processed(height, Math.Min(ToHeight, _BlockHeaders.Tip.Height), lastLogs, lastHeights);
height++;
}
}
public bool Process(ChainBase mainChain, IBlockProvider blockProvider)
{
var chainCopy = Chain.Clone();
var chainPosition = chainCopy.Changes.Position;
var accountCopy = Account.Clone();
var accountPosition = accountCopy.Entries.Position;
bool newChain = false;
if (!chainCopy.Initialized)
{
newChain = true;
var firstBlock = mainChain.GetBlock(StartHeight);
chainCopy.Initialize(firstBlock.Header, StartHeight);
}
var forkBlock = mainChain.FindFork(chainCopy);
if (forkBlock.HashBlock != chainCopy.Tip.HashBlock)
{
var subChain = chainCopy.CreateSubChain(forkBlock, false, chainCopy.Tip, true);
chainCopy.SetTip(chainCopy.GetBlock(forkBlock.Height));
foreach (var e in accountCopy.GetInChain(subChain, true)
.Where(c => c.Reason != AccountEntryReason.Lock && c.Reason != AccountEntryReason.Unlock)
.Reverse())
{
var neutralized = e.Neutralize();
accountCopy.PushAccountEntry(neutralized);
}
}
var unprocessedBlocks = mainChain.ToEnumerable(true)
.TakeWhile(block => block != forkBlock)
.Concat(newChain ? new ChainedBlock[] { forkBlock } : new ChainedBlock[0])
.Reverse().ToArray();
foreach (var block in unprocessedBlocks)
{
List <byte[]> searchedData = new List <byte[]>();
Scanner.GetScannedPushData(searchedData);
foreach (var unspent in accountCopy.Unspent)
{
searchedData.Add(unspent.OutPoint.ToBytes());
}
var fullBlock = blockProvider.GetBlock(block.HashBlock, searchedData);
if (fullBlock == null)
{
continue;
}
List <Tuple <OutPoint, AccountEntry> > spents = new List <Tuple <OutPoint, AccountEntry> >();
foreach (var spent in FindSpent(fullBlock, accountCopy.Unspent))
{
var entry = new AccountEntry(AccountEntryReason.Outcome,
block.HashBlock,
spent.Spendable, -spent.Spendable.TxOut.Value, spent.TxId);
spents.Add(Tuple.Create(entry.Spendable.OutPoint, entry));
}
if (CheckDoubleSpend)
{
var spentsDico = spents.ToDictionary(t => t.Item1, t => t.Item2);
foreach (var spent in Scanner.FindSpent(fullBlock))
{
if (!spentsDico.ContainsKey(spent.PrevOut))
{
return(false);
}
}
}
foreach (var spent in spents)
{
if (accountCopy.PushAccountEntry(spent.Item2) == null)
{
return(false);
}
}
foreach (var coins in Scanner.ScanCoins(fullBlock, (int)block.Height))
{
int i = 0;
foreach (var output in coins.Coins.Outputs)
{
if (!output.IsNull)
{
var entry = new AccountEntry(AccountEntryReason.Income, block.HashBlock,
new Spendable(new OutPoint(coins.TxId, i), output), output.Value, null);
if (accountCopy.PushAccountEntry(entry) == null)
{
return(false);
}
}
i++;
}
}
chainCopy.SetTip(block);
}
accountCopy.Entries.GoTo(accountPosition);
Account.PushAccountEntries(accountCopy.Entries);
chainCopy.Changes.GoTo(chainPosition);
Chain.PushChanges(chainCopy.Changes);
return(true);
}
/// <summary>
/// Get a block fetcher of the specified chain from the specified checkpoint
/// </summary>
/// <param name="checkpoint">The checkpoint to load from</param>
/// <param name="chain">The chain to fetcher (default: the Node's main chain)</param>
/// <returns>A BlockFetcher for enumerating blocks and saving progression</returns>
public BlockFetcher GetBlockFetcher(Checkpoint checkpoint, Node node, ChainBase chain = null)
{
if(checkpoint == null)
throw new ArgumentNullException("checkpoint");
if(node == null)
throw new ArgumentNullException("node");
chain = chain ?? GetNodeChain(node);
IndexerTrace.CheckpointLoaded(chain.FindFork(checkpoint.BlockLocator), checkpoint.CheckpointName);
return new BlockFetcher(checkpoint, new NodeBlocksRepository(node), chain)
{
NeedSaveInterval = CheckpointInterval,
FromHeight = FromHeight,
ToHeight = ToHeight
};
}
static void Main(string[] args)
{
try
{
var options = new IndexerOptions();
if (args.Length == 0)
{
System.Console.WriteLine(options.GetUsage());
}
if (Parser.Default.ParseArguments(args, options))
{
System.Console.WriteLine("NBitcoin.Indexer " + typeof(AzureIndexer).Assembly.GetName().Version);
if (options.All)
{
options.IndexAddresses = true;
options.IndexBlocks = true;
options.IndexWallets = true;
options.IndexChain = true;
options.IndexTransactions = true;
}
var indexer = AzureIndexer.CreateIndexer();
indexer.Configuration.EnsureSetup();
indexer.TaskScheduler = new CustomThreadPoolTaskScheduler(30, 100);
indexer.CheckpointInterval = TimeSpan.Parse(options.CheckpointInterval);
indexer.IgnoreCheckpoints = options.IgnoreCheckpoints;
indexer.FromHeight = options.From;
indexer.ToHeight = options.To;
ChainBase chain = null;
var checkpointRepository = indexer.GetCheckpointRepository();
checkpointRepository.CheckpointSet = null;
if (options.ListCheckpoints)
{
foreach (var checkpoint in checkpointRepository.GetCheckpointsAsync().Result)
{
chain = chain ?? indexer.GetNodeChain();
var fork = chain.FindFork(checkpoint.BlockLocator);
System.Console.WriteLine("Name : " + checkpoint.CheckpointName);
if (fork != null)
{
System.Console.WriteLine("Height : " + fork.Height);
System.Console.WriteLine("Hash : " + fork.HashBlock);
}
System.Console.WriteLine();
}
}
if (options.DeleteCheckpoint != null)
{
checkpointRepository.GetCheckpoint(options.DeleteCheckpoint).DeleteAsync().Wait();
System.Console.WriteLine("Checkpoint " + options.DeleteCheckpoint + " deleted");
}
if (options.AddCheckpoint != null)
{
chain = chain ?? indexer.GetNodeChain();
var split = options.AddCheckpoint.Split(':');
var name = split[0];
var height = int.Parse(split[1]);
var b = chain.GetBlock(height);
var checkpoint = checkpointRepository.GetCheckpoint(name);
checkpoint.SaveProgress(b.GetLocator());
System.Console.WriteLine("Checkpoint " + options.AddCheckpoint + " saved to height " + b.Height);
}
if (ConfigurationManager.AppSettings["MainDirectory"] != null)
{
System.Console.WriteLine("Warning : obsolete appsetting detected, MainDirectory");
string[] oldCheckpoints = new string[] { "transactions", "blocks", "wallets", "balances" };
foreach (var chk in oldCheckpoints)
{
var path = GetFilePath(indexer.Configuration, chk);
if (File.Exists(path))
{
var onlineCheckpoint = checkpointRepository.GetCheckpointsAsync().Result.FirstOrDefault(r => r.CheckpointName.ToLowerInvariant() == chk);
if (onlineCheckpoint == null)
{
onlineCheckpoint = checkpointRepository.GetCheckpoint(indexer.Configuration.CheckpointSetName + "/" + chk);
BlockLocator offlineLocator = new BlockLocator();
offlineLocator.FromBytes(File.ReadAllBytes(path));
onlineCheckpoint.SaveProgress(offlineLocator);
System.Console.WriteLine("Local checkpoint " + chk + " saved in azure");
}
File.Delete(path);
System.Console.WriteLine("Checkpoint File deleted " + path);
}
}
}
if (options.IndexBlocks)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexBlocks(chain);
}
if (options.IndexTransactions)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexTransactions(chain);
}
if (options.IndexAddresses)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexOrderedBalances(chain);
}
if (options.IndexWallets)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexWalletBalances(chain);
}
if (options.IndexChain)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexChain(chain);
}
}
}
catch (ConfigurationErrorsException ex)
{
System.Console.WriteLine("LocalSettings.config missing settings : " + ex.Message);
}
}