private async Task SaveMatches(Block block) { _LastBlockDownloaded = DateTimeOffset.UtcNow; block.Header.PrecomputeHash(false, false); foreach (var tx in block.Transactions) { tx.PrecomputeHash(false, true); } var blockHash = block.GetHash(); var delay = TimeSpan.FromSeconds(1); retry: try { DateTimeOffset now = DateTimeOffset.UtcNow; var matches = (await Repository.GetMatches(block.Transactions, blockHash, now, true)) .ToArray(); await SaveMatches(matches, blockHash, now, true); var slimBlockHeader = Chain.GetBlock(blockHash); if (slimBlockHeader != null) { var blockEvent = new Models.NewBlockEvent() { CryptoCode = _Repository.Network.CryptoCode, Hash = blockHash, Height = slimBlockHeader.Height, PreviousBlockHash = slimBlockHeader.Previous }; var saving = Repository.SaveEvent(blockEvent); _EventAggregator.Publish(blockEvent); await saving; } } catch (ObjectDisposedException) { throw; } catch (Exception ex) { Logs.Explorer.LogWarning(ex, $"{Network.CryptoCode}: Error while saving block in database, retrying in {delay.TotalSeconds} seconds ({ex.Message})"); await Task.Delay(delay, _Cts.Token); delay = delay * 2; var maxDelay = TimeSpan.FromSeconds(60); if (delay > maxDelay) { delay = maxDelay; } goto retry; } if (_InFlights.TryRemove(blockHash, out var unused) && _InFlights.IsEmpty) { var highestInFlight = _HighestInFlight; _HighestInFlight = null; if (highestInFlight != null) { CurrentLocation = highestInFlight; await Repository.SetIndexProgress(highestInFlight); } AskBlocks(); } }
private void TransferRecentStates(GetRecentStates getRecentStates) { BlockLocator baseLocator = getRecentStates.BaseLocator; HashDigest <SHA256>? @base = BlockChain.FindBranchPoint(baseLocator); HashDigest <SHA256> target = getRecentStates.TargetBlockHash; IImmutableDictionary <HashDigest <SHA256>, IImmutableDictionary <string, IValue> > blockStates = null; IImmutableDictionary <string, IImmutableList <HashDigest <SHA256> > > stateRefs = null; long nextOffset = -1; int iteration = 0; if (BlockChain.StateStore is IBlockStatesStore blockStatesStore && BlockChain.ContainsBlock(target)) { ReaderWriterLockSlim rwlock = BlockChain._rwlock; rwlock.EnterReadLock(); try { Guid chainId = BlockChain.Id; _logger.Debug( "Getting state references from {Offset}", getRecentStates.Offset); long baseIndex = (@base is HashDigest <SHA256> bbh && _store.GetBlockIndex(bbh) is long bbIdx) ? bbIdx : 0; long lowestIndex = baseIndex + getRecentStates.Offset; long targetIndex = (target is HashDigest <SHA256> tgt && _store.GetBlockIndex(tgt) is long tgtIdx) ? tgtIdx : long.MaxValue; iteration = (int)Math.Ceiling( (double)(targetIndex - baseIndex + 1) / FindNextStatesChunkSize); long highestIndex = lowestIndex + FindNextStatesChunkSize - 1 > targetIndex ? targetIndex : lowestIndex + FindNextStatesChunkSize - 1; nextOffset = highestIndex == targetIndex ? -1 : getRecentStates.Offset + FindNextStatesChunkSize; stateRefs = blockStatesStore.ListAllStateReferences( chainId, lowestIndex: lowestIndex, highestIndex: highestIndex ); if (_logger.IsEnabled(LogEventLevel.Verbose)) { _logger.Verbose( "List state references from {From} to {To}:\n{StateReferences}", lowestIndex, highestIndex, string.Join( "\n", stateRefs.Select(kv => $"{kv.Key}: {string.Join(", ", kv.Value)}") ) ); } // GetBlockStates may return null since swarm may not have deep states. blockStates = stateRefs.Values .Select(refs => refs.Last()) .ToImmutableHashSet() .Select(bh => (bh, blockStatesStore.GetBlockStates(bh))) .Where(pair => !(pair.Item2 is null)) .ToImmutableDictionary( pair => pair.Item1, pair => (IImmutableDictionary <string, IValue>)pair.Item2 .ToImmutableDictionary(kv => kv.Key, kv => kv.Value) ); } finally { rwlock.ExitReadLock(); } } if (_logger.IsEnabled(LogEventLevel.Verbose)) { if (BlockChain.ContainsBlock(target)) { var baseString = @base is HashDigest <SHA256> h ? $"{BlockChain[h].Index}:{h}" : null; var targetString = $"{BlockChain[target].Index}:{target}"; _logger.Verbose( "State references to send (preload): {StateReferences} ({Base}-{Target})", stateRefs.Select(kv => ( kv.Key, string.Join(", ", kv.Value.Select(v => v.ToString())) ) ).ToArray(), baseString, targetString ); _logger.Verbose( "Block states to send (preload): {BlockStates} ({Base}-{Target})", blockStates.Select(kv => (kv.Key, kv.Value)).ToArray(), baseString, targetString ); } else { _logger.Verbose("Nothing to reply because {TargetHash} doesn't exist.", target); } } var reply = new RecentStates( target, nextOffset, iteration, blockStates, stateRefs?.ToImmutableDictionary()) { Identity = getRecentStates.Identity, }; Transport.ReplyMessage(reply); }
public BlockLocator GetCurrentChainLocator() { List<StoredBlock> headers = new List<StoredBlock>(); using (BlockchainRepository repo = new BlockchainRepository(conn)) { StoredBlock lastBlock = repo.GetLastBlockHeader(); if (lastBlock != null) { headers = repo.ReadHeadersWithHeight(BlockLocator.GetRequiredBlockHeights(lastBlock.Height)); } } BlockLocator locator = new BlockLocator(); foreach (StoredBlock header in headers) { locator.AddHash(header.Height, header.Hash); } return locator; }
public GetHeadersPayload(BlockLocator locator) { this.BlockLocator = locator; }
/// <summary> /// Start a scan, if a scan is already running, will change only if the parameters are anterior /// </summary> /// <param name="locator"></param> public void Scan(BlockLocator locator, DateTimeOffset skipBefore) { lock(cs) { if(_SkipBefore == default(DateTimeOffset) || skipBefore < _SkipBefore) _SkipBefore = skipBefore; if(_CurrentProgress == null || EarlierThanCurrentProgress(locator)) _CurrentProgress = locator; } }
/// <inheritdoc /> public Task <ChainedHeader> LoadAsync(ChainedHeader genesisHeader) { Task <ChainedHeader> task = Task.Run(() => { ChainedHeader tip = null; ChainData data = this.chainStore.GetChainData(0); if (data == null) { genesisHeader.SetChainStore(this.chainStore); return(genesisHeader); } Guard.Assert(data.Hash == genesisHeader.HashBlock); // can't swap networks int height = 0; while (true) { data = this.chainStore.GetChainData(height); if (data == null) { break; } tip = new ChainedHeader(data.Hash, data.Work, tip); if (tip.Height == 0) { tip.SetChainStore(this.chainStore); } if (height % 50_000 == 0) { if (this.signals != null) { this.signals.Publish(new FullNodeEvent() { Message = $"Loading chain at height {height}.", State = FullNodeState.Initializing.ToString() }); } this.logger.LogInformation($"Loading chain at height {height}."); } height++; } if (tip == null) { genesisHeader.SetChainStore(this.chainStore); tip = genesisHeader; } else { // Confirm that the chain tip exists in the headers table. this.chainStore.GetHeader(tip, tip.HashBlock); } this.locator = tip.GetLocator(); return(tip); }); return(task); }
public int Run(ChainBase chain = null) { ListenerTrace.Info("Start initial indexing"); int totalProcessed = 0; using (var node = _Conf.Indexer.ConnectToNode(false)) { ListenerTrace.Info("Handshaking..."); node.VersionHandshake(); ListenerTrace.Info("Handshaked"); chain = chain ?? node.GetChain(); ListenerTrace.Info("Current chain at height " + chain.Height); var blockRepository = new NodeBlocksRepository(node); var blobLock = GetInitBlob(); string lease = null; try { blobLock.UploadText("Enqueuing"); lease = blobLock.AcquireLease(null, null); } catch (StorageException) { } if (lease != null) { ListenerTrace.Info("Queueing index jobs"); EnqueueJobs(blockRepository, chain, blobLock, lease); } ListenerTrace.Info("Dequeuing index jobs"); while (true) { var msg = _Conf.Topics .InitialIndexing .ReceiveAsync(TimeSpan.FromMilliseconds(1000)) .Result; var ns = _Conf.Topics.InitialIndexing.GetNamespace(); var description = ns.GetQueue(_Conf.Topics.InitialIndexing.Queue); Console.WriteLine("Work remaining in the queue : " + description.MessageCountDetails.ActiveMessageCount); if (msg == null) { var state = blobLock.DownloadText(); if (state == "Enqueuing" || description.MessageCountDetails.ActiveMessageCount != 0) { ListenerTrace.Info("Additional work will be enqueued..."); continue; } else { var locator = new BlockLocator(); locator.FromBytes(Encoders.Hex.DecodeData(state)); UpdateCheckpoints(locator); break; } } using (msg.Message) { var range = msg.Body; using (var sched = new CustomThreadPoolTaskScheduler(50, 100, range.ToString())) { ListenerTrace.Info("Processing " + range.ToString()); totalProcessed++; var task = _IndexTasks[range.Target]; BlockFetcher fetcher = new BlockFetcher(task.Item1, blockRepository, chain) { FromHeight = range.From, ToHeight = range.From + range.Count - 1 }; try { task.Item2.SaveProgression = false; task.Item2.EnsureIsSetup = totalProcessed == 0; var index = Task.Factory.StartNew(() => { task.Item2.Index(fetcher, sched); }, TaskCreationOptions.LongRunning); while (!index.Wait(TimeSpan.FromMinutes(4))) { msg.Message.RenewLock(); ListenerTrace.Info("Lock renewed"); } } catch (AggregateException aex) { ExceptionDispatchInfo.Capture(aex.InnerException).Throw(); throw; } range.Processed = true; msg.Message.Complete(); } } } } ListenerTrace.Info("Initial indexing terminated"); return(totalProcessed); }
public GetHeadersPayload(BlockLocator locator) { BlockLocators = locator; }
private void TryUpdateLocation() { var group = _Group; if(group != null) { var progress = group.ConnectedNodes .Select(f => f.Behaviors.Find<TrackerBehavior>().CurrentProgress) .Where(p => p != null) .Select(l => new { Locator = l, Block = Chain.FindFork(l) }) .OrderByDescending(o => o.Block.Height) .Select(o => o.Block) .FirstOrDefault(); if(progress != null) { progress = progress.EnumerateToGenesis().Skip(5).FirstOrDefault() ?? progress; //Step down 5 blocks, it does not cost a lot to rescan them in case we missed something _ScanLocation = progress.GetLocator(); } } }
public GetBlocksPayload(BlockLocator locator) { BlockLocators = locator; }
public GetBlocks( BlockLocator locator, HashDigest <SHA256>?stop) { Locator = locator; Stop = stop; }
/// <summary> /// Attempts to connect to a full BitCoin node and request any missing block headers. /// </summary> private static async Task <Node> ConnectNodeAndSyncHeaders(ConcurrentChain chain, CancellationToken ct) { logger.DebugFormat("Connecting to full node..."); ct.ThrowIfCancellationRequested(); ManualResetEventSlim headersSyncedSignal = new ManualResetEventSlim(); var parameters = new NodeConnectionParameters(); parameters.IsRelay = false; var scanLocation = new BlockLocator(); scanLocation.Blocks.Add(chain.Tip != null ? chain.Tip.HashBlock : _network.GetGenesis().GetHash()); var node = Node.ConnectToLocal(_network, parameters); logger.DebugFormat("Connected to node " + node.RemoteSocketEndpoint + "."); node.MessageReceived += (node1, message) => { ct.ThrowIfCancellationRequested(); switch (message.Message.Payload) { case HeadersPayload hdr: if (hdr.Headers != null && hdr.Headers.Count > 0) { logger.DebugFormat("Received {0} blocks start {1} to {2} height {3}.", hdr.Headers.Count, hdr.Headers.First().BlockTime, hdr.Headers.Last().BlockTime, chain.Height); scanLocation.Blocks.Clear(); scanLocation.Blocks.Add(hdr.Headers.Last().GetHash()); if (hdr != null) { var tip = chain.Tip; foreach (var header in hdr.Headers) { var prev = tip.FindAncestorOrSelf(header.HashPrevBlock); if (prev == null) { break; } tip = new ChainedBlock(header, header.GetHash(), prev); chain.SetTip(tip); ct.ThrowIfCancellationRequested(); } } var getHeadersPayload = new GetHeadersPayload(scanLocation); node.SendMessageAsync(getHeadersPayload); } else { // Headers synchronised. logger.DebugFormat("Block headers synchronised."); headersSyncedSignal.Set(); } break; case InvPayload inv: logger.DebugFormat("Inventory items {0}, first type {1}.", inv.Count(), inv.First().Type); if (inv.Any(x => x.Type == InventoryType.MSG_BLOCK)) { // New block available. var getHeadersPayload = new GetHeadersPayload(scanLocation); node.SendMessage(getHeadersPayload); } break; case MerkleBlockPayload merkleBlk: break; case TxPayload tx: break; default: logger.DebugFormat(message.Message.Command); break; } }; node.Disconnected += n => { logger.DebugFormat("Node disconnected, chain height " + chain.Height + "."); }; node.VersionHandshake(ct); node.PingPong(ct); logger.DebugFormat("Requesting block headers greater than height {0}.", chain.Height); node.SendMessage(new GetHeadersPayload(scanLocation)); logger.DebugFormat("Bitcoin node connected."); await Task.Run(() => { headersSyncedSignal.Wait(ct); }); return(node); }
#pragma warning disable MEN003 private async Task <BlockChain <T> > FillBlocksAsync( BoundPeer peer, BlockChain <T> blockChain, BlockHash?stop, IProgress <BlockDownloadState> progress, long totalBlockCount, long receivedBlockCount, bool evaluateActions, TimeSpan timeout, int logSessionId, CancellationToken cancellationToken ) { var sessionRandom = new Random(); const string fname = nameof(FillBlocksAsync); BlockChain <T> workspace = blockChain; var scope = new List <Guid>(); bool renderActions = evaluateActions; bool renderBlocks = true; try { while (!cancellationToken.IsCancellationRequested) { int subSessionId = sessionRandom.Next(); Block <T> tip = workspace?.Tip; _logger.Debug( "{SessionId}/{SubSessionId}: Trying to find branchpoint...", logSessionId, subSessionId ); BlockLocator locator = workspace.GetBlockLocator(); _logger.Debug( "{SessionId}/{SubSessionId}: Locator's length: {LocatorLength}", logSessionId, subSessionId, locator.Count() ); IAsyncEnumerable <Tuple <long, BlockHash> > hashesAsync = GetBlockHashes( peer: peer, locator: locator, stop: stop, timeout: timeout, logSessionIds: (logSessionId, subSessionId), cancellationToken: cancellationToken ); IEnumerable <Tuple <long, BlockHash> > hashes = await hashesAsync.ToArrayAsync(); if (!hashes.Any()) { _logger.Debug( "{SessionId}/{SubSessionId}: Peer {0} returned no hashes; ignored.", logSessionId, subSessionId, peer.Address.ToHex() ); return(workspace); } hashes.First().Deconstruct( out long branchIndex, out BlockHash branchpoint ); _logger.Debug( "{SessionId}/{SubSessionId}: Branchpoint is #{BranchIndex} {BranchHash}.", logSessionId, subSessionId, branchIndex, branchpoint ); if (tip is null || branchpoint.Equals(tip.Hash)) { _logger.Debug( "{SessionId}/{SubSessionId}: It doesn't need to fork.", logSessionId, subSessionId ); } else if (!workspace.ContainsBlock(branchpoint)) { // FIXME: This behavior can unexpectedly terminate the swarm (and the game // app) if it encounters a peer having a different blockchain, and therefore // can be exploited to remotely shut down other nodes as well. // Since the intention of this behavior is to prevent mistakes to try to // connect incorrect seeds (by a user), this behavior should be limited for // only seed peers. var msg = $"Since the genesis block is fixed to {BlockChain.Genesis} " + "protocol-wise, the blockchain which does not share " + "any mutual block is not acceptable."; throw new InvalidGenesisBlockException( branchpoint, workspace.Genesis.Hash, msg); } else { _logger.Debug( "{SessionId}/{SubSessionId}: Needs to fork; trying to fork...", logSessionId, subSessionId ); workspace = workspace.Fork(branchpoint); Guid workChainId = workspace.Id; scope.Add(workChainId); renderActions = false; renderBlocks = false; _logger.Debug( "{SessionId}/{SubSessionId}: Fork finished.", logSessionId, subSessionId ); } if (!(workspace.Tip is null)) { hashes = hashes.Skip(1); } _logger.Debug( "{SessionId}/{SubSessionId}: Trying to fill up previous blocks...", logSessionId, subSessionId ); var hashesAsArray = hashes as Tuple <long, BlockHash>[] ?? hashes.ToArray(); if (!hashesAsArray.Any()) { break; } int hashCount = hashesAsArray.Count(); _logger.Debug( "{SessionId}/{SubSessionId}: Required {Hashes} hashes " + "(tip: #{TipIndex} {TipHash}).", logSessionId, subSessionId, hashCount, workspace.Tip?.Index, workspace.Tip?.Hash ); totalBlockCount = Math.Max(totalBlockCount, receivedBlockCount + hashCount); IAsyncEnumerable <Block <T> > blocks = GetBlocksAsync( peer, hashesAsArray.Select(pair => pair.Item2), cancellationToken ); var receivedBlockCountCurrentLoop = 0; await foreach (Block <T> block in blocks) { const string startMsg = "{SessionId}/{SubSessionId}: Try to append a block " + "#{BlockIndex} {BlockHash}..."; _logger.Debug( startMsg, logSessionId, subSessionId, block.Index, block.Hash ); cancellationToken.ThrowIfCancellationRequested(); workspace.Append( block, DateTimeOffset.UtcNow, evaluateActions: evaluateActions, renderBlocks: renderBlocks, renderActions: renderActions ); receivedBlockCountCurrentLoop++; progress?.Report(new BlockDownloadState { TotalBlockCount = totalBlockCount, ReceivedBlockCount = receivedBlockCount + receivedBlockCountCurrentLoop, ReceivedBlockHash = block.Hash, SourcePeer = peer, }); const string endMsg = "{SessionId}/{SubSessionId}: Block #{BlockIndex} {BlockHash} " + "was appended."; _logger.Debug(endMsg, logSessionId, subSessionId, block.Index, block.Hash); } receivedBlockCount += receivedBlockCountCurrentLoop; var isEndedFirstTime = receivedBlockCount == receivedBlockCountCurrentLoop && receivedBlockCount < FindNextHashesChunkSize - 1; if (receivedBlockCountCurrentLoop < FindNextHashesChunkSize && isEndedFirstTime) { _logger.Debug( "{SessionId}/{SubSessionId}: Got {Blocks} blocks from Peer {Peer} " + "(tip: #{TipIndex} {TipHash})", logSessionId, subSessionId, receivedBlockCountCurrentLoop, peer.Address.ToHex(), workspace.Tip?.Index, workspace.Tip?.Hash ); break; } }
public GetRecentStates(BlockLocator baseLocator, HashDigest <SHA256> target) { BaseLocator = baseLocator; TargetBlockHash = target; }
static void Main(string[] args) { try { var options = new IndexerOptions(); if (args.Length == 0) System.Console.WriteLine(options.GetUsage()); if (Parser.Default.ParseArguments(args, options)) { System.Console.WriteLine("NBitcoin.Indexer " + typeof(AzureIndexer).Assembly.GetName().Version); if (options.All) { options.IndexAddresses = true; options.IndexBlocks = true; options.IndexWallets = true; options.IndexChain = true; options.IndexTransactions = true; } var indexer = AzureIndexer.CreateIndexer(); indexer.Configuration.EnsureSetup(); indexer.TaskScheduler = new CustomThreadPoolTaskScheduler(30, 100); indexer.CheckpointInterval = TimeSpan.Parse(options.CheckpointInterval); indexer.IgnoreCheckpoints = options.IgnoreCheckpoints; indexer.FromHeight = options.From; indexer.ToHeight = options.To; ChainBase chain = null; var checkpointRepository = indexer.GetCheckpointRepository(); checkpointRepository.CheckpointSet = null; if (options.ListCheckpoints) { foreach (var checkpoint in checkpointRepository.GetCheckpointsAsync().Result) { chain = chain ?? indexer.GetNodeChain(); var fork = chain.FindFork(checkpoint.BlockLocator); System.Console.WriteLine("Name : " + checkpoint.CheckpointName); if (fork != null) { System.Console.WriteLine("Height : " + fork.Height); System.Console.WriteLine("Hash : " + fork.HashBlock); } System.Console.WriteLine(); } } if (options.DeleteCheckpoint != null) { checkpointRepository.GetCheckpoint(options.DeleteCheckpoint).DeleteAsync().Wait(); System.Console.WriteLine("Checkpoint " + options.DeleteCheckpoint + " deleted"); } if (options.AddCheckpoint != null) { chain = chain ?? indexer.GetNodeChain(); var split = options.AddCheckpoint.Split(':'); var name = split[0]; var height = int.Parse(split[1]); var b = chain.GetBlock(height); var checkpoint = checkpointRepository.GetCheckpoint(name); checkpoint.SaveProgress(b.GetLocator()); System.Console.WriteLine("Checkpoint " + options.AddCheckpoint + " saved to height " + b.Height); } if (ConfigurationManager.AppSettings["MainDirectory"] != null) { System.Console.WriteLine("Warning : obsolete appsetting detected, MainDirectory"); string[] oldCheckpoints = new string[] { "transactions", "blocks", "wallets", "balances" }; foreach (var chk in oldCheckpoints) { var path = GetFilePath(indexer.Configuration, chk); if (File.Exists(path)) { var onlineCheckpoint = checkpointRepository.GetCheckpointsAsync().Result.FirstOrDefault(r => r.CheckpointName.ToLowerInvariant() == chk); if (onlineCheckpoint == null) { onlineCheckpoint = checkpointRepository.GetCheckpoint(indexer.Configuration.CheckpointSetName + "/" + chk); BlockLocator offlineLocator = new BlockLocator(); offlineLocator.FromBytes(File.ReadAllBytes(path)); onlineCheckpoint.SaveProgress(offlineLocator); System.Console.WriteLine("Local checkpoint " + chk + " saved in azure"); } File.Delete(path); System.Console.WriteLine("Checkpoint File deleted " + path); } } } if (options.IndexBlocks) { chain = chain ?? indexer.GetNodeChain(); indexer.IndexBlocks(chain); } if (options.IndexTransactions) { chain = chain ?? indexer.GetNodeChain(); indexer.IndexTransactions(chain); } if (options.IndexAddresses) { chain = chain ?? indexer.GetNodeChain(); indexer.IndexOrderedBalances(chain); } if (options.IndexWallets) { chain = chain ?? indexer.GetNodeChain(); indexer.IndexWalletBalances(chain); } if (options.IndexChain) { chain = chain ?? indexer.GetNodeChain(); indexer.IndexChain(chain); } } } catch (ConfigurationErrorsException ex) { System.Console.WriteLine("LocalSettings.config missing settings : " + ex.Message); } }
void LoadCore(Stream stream) { JObject obj = JObject.Load(new JsonTextReader(new StreamReader(stream)) { DateParseHandling = DateParseHandling.DateTimeOffset }); _CurrentIndex = (int)(long)obj["CurrentIndex"]; _KeyPoolSize = (int)(long)obj["KeyPoolSize"]; _LoadedKeys = (int)(long)obj["LoadedKeys"]; Created = (DateTimeOffset)obj["Created"]; _ScanLocation = new BlockLocator(); _ScanLocation.FromBytes(Encoders.Hex.DecodeData((string)obj["Location"])); _Parameters = WalletCreation.FromJson((JObject)obj["Parameters"]); _KnownScripts.Clear(); var knownScripts = (JArray)obj["KnownScripts"]; foreach(var known in knownScripts.OfType<JObject>()) { Script script = Script.FromBytesUnsafe(Encoders.Hex.DecodeData((string)known["ScriptPubKey"])); KeyPath keypath = KeyPath.Parse((string)known["KeyPath"]); _KnownScripts.Add(script, keypath); } }
public GetBlockHashes(BlockLocator locator, BlockHash?stop) { Locator = locator; Stop = stop; }
public GetHeadersPayload(BlockLocator locator) { BlockLocators = locator; }
private void AttachedNode_MessageReceived(Node node, IncomingMessage message) { message.Message.IfPayloadIs <InvPayload>(invs => { var data = new GetDataPayload(); foreach (var inv in invs.Inventory) { inv.Type = node.AddSupportedOptions(inv.Type); if (inv.Type.HasFlag(InventoryType.MSG_TX)) { data.Inventory.Add(inv); } } if (data.Inventory.Count != 0) { node.SendMessageAsync(data); } }); message.Message.IfPayloadIs <HeadersPayload>(headers => { if (headers.Headers.Count == 0) { return; } AskBlocks(); }); message.Message.IfPayloadIs <BlockPayload>(block => { block.Object.Header.PrecomputeHash(false, false); Download o; if (_InFlights.ContainsKey(block.Object.GetHash())) { var blockHeader = Chain.GetBlock(block.Object.GetHash()); if (blockHeader == null) { return; } var currentLocation = blockHeader.GetLocator(); _CurrentLocation = currentLocation; if (_InFlights.TryRemove(block.Object.GetHash(), out o)) { foreach (var tx in block.Object.Transactions) { tx.PrecomputeHash(false, true); } var matches = block.Object.Transactions .SelectMany(tx => Repository.GetMatches(tx)) .ToArray(); var blockHash = block.Object.GetHash(); SaveMatches(matches, blockHash); //Save index progress everytimes if not synching, or once every 100 blocks otherwise if (!IsSynching() || blockHash.GetLow32() % 100 == 0) { Repository.SetIndexProgress(currentLocation); } _EventAggregator.Publish(new Events.NewBlockEvent(this._Repository.Network.CryptoCode, blockHash)); } if (_InFlights.Count == 0) { AskBlocks(); } } }); message.Message.IfPayloadIs <TxPayload>(txPayload => { var matches = Repository.GetMatches(txPayload.Object).ToArray(); SaveMatches(matches, null); }); }
private void AttachedNode_MessageReceived(Node node, IncomingMessage message) { message.Message.IfPayloadIs <InvPayload>(invs => { var data = new GetDataPayload(); foreach (var inv in invs.Inventory) { inv.Type = node.AddSupportedOptions(inv.Type); if (inv.Type.HasFlag(InventoryType.MSG_TX)) { data.Inventory.Add(inv); } } if (data.Inventory.Count != 0) { node.SendMessageAsync(data); } }); message.Message.IfPayloadIs <HeadersPayload>(headers => { if (headers.Headers.Count == 0) { return; } AskBlocks(); }); message.Message.IfPayloadIs <BlockPayload>(block => { block.Object.Header.CacheHashes(); Download o; if (_InFlights.ContainsKey(block.Object.GetHash())) { var blockHeader = Chain.GetBlock(block.Object.GetHash()); if (blockHeader == null) { return; } var currentLocation = blockHeader.GetLocator(); _CurrentLocation = currentLocation; if (_InFlights.TryRemove(block.Object.GetHash(), out o)) { if (!IsSynching()) { var unused = _Callbacks.SendCallbacks(block.Object.GetHash()); } foreach (var tx in block.Object.Transactions) { tx.CacheHashes(); } var matches = block.Object.Transactions .SelectMany(tx => GetMatches(tx)) .ToArray(); var blockHash = block.Object.GetHash(); SaveMatches(matches, blockHash); if (!IsSynching()) { var unused = _Callbacks.SendCallbacks(matches); } //Save index progress everytimes if not synching, or once every 100 blocks otherwise if (!IsSynching() || blockHash.GetLow32() % 100 == 0) { Repository.SetIndexProgress(currentLocation); } Logs.Explorer.LogInformation($"Processed block {blockHash}"); } if (_InFlights.Count == 0) { AskBlocks(); } } }); message.Message.IfPayloadIs <TxPayload>(txPayload => { var matches = GetMatches(txPayload.Object).ToArray(); SaveMatches(matches, null); }); }
public GetBlocksPayload(BlockLocator locator) { this.BlockLocators = locator; }
public void SaveProgress(BlockLocator locator) { _BlockLocator = locator; File.WriteAllBytes(_FileName, _BlockLocator.ToBytes()); }
private bool EarlierThanCurrentProgress(BlockLocator locator) { return(_Chain.FindFork(locator).Height < _Chain.FindFork(_CurrentProgress).Height); }
/// <summary> /// Start a scan, if a scan is already running, will change only if the parameters are anterior /// </summary> /// <param name="locator"></param> public void Scan(BlockLocator locator, DateTimeOffset skipBefore) { lock(cs) { if(_SkipBefore == default(DateTimeOffset) || skipBefore < _SkipBefore) _SkipBefore = skipBefore; if(_CurrentProgress == null || _Chain.FindFork(locator).Height < _Chain.FindFork(_CurrentProgress).Height) _CurrentProgress = locator; } }
private bool EarlierThanCurrentProgress(BlockLocator locator) { return _Chain.FindFork(locator).Height < _Chain.FindFork(_CurrentProgress).Height; }
/// <summary> /// Create a new wallet /// </summary> /// <param name="creation">Creation parameters</param> /// <param name="keyPoolSize">The number of keys which will be pre-created</param> public Wallet(WalletCreation creation, int keyPoolSize = 500) { if(creation == null) throw new ArgumentNullException("creation"); _Parameters = creation; _ScanLocation = new BlockLocator(); _ScanLocation.Blocks.Add(creation.Network.GetGenesis().GetHash()); _KeyPoolSize = keyPoolSize; Created = DateTimeOffset.UtcNow; }
public IEnumerable <ChainedHeader> GetHeadersFromFork(INetworkPeer peer, ChainedHeader currentTip, uint256 hashStop = null, CancellationToken cancellationToken = default(CancellationToken)) { this.AssertStateAsync(peer, NetworkPeerState.HandShaked, cancellationToken).GetAwaiter().GetResult(); using (var listener = new NetworkPeerListener(peer, this.GetOrCreateAsyncProvider())) { int acceptMaxReorgDepth = 0; while (true) { // Get before last so, at the end, we should only receive 1 header equals to this one (so we will not have race problems with concurrent GetChains). BlockLocator awaited = currentTip.Previous == null?currentTip.GetLocator() : currentTip.Previous.GetLocator(); peer.SendMessageAsync(new GetHeadersPayload() { BlockLocator = awaited, HashStop = hashStop }, cancellationToken).GetAwaiter().GetResult(); while (true) { bool isOurs = false; HeadersPayload headers = null; using (CancellationTokenSource headersCancel = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) { headersCancel.CancelAfter(TimeSpan.FromMinutes(1.0)); try { headers = listener.ReceivePayloadAsync <HeadersPayload>(headersCancel.Token).GetAwaiter().GetResult(); } catch (OperationCanceledException) { acceptMaxReorgDepth += 6; if (cancellationToken.IsCancellationRequested) { throw; } // Send a new GetHeaders. break; } } // In the special case where the remote node is at height 0 as well as us, then the headers count will be 0. if ((headers.Headers.Count == 0) && (peer.PeerVersion.StartHeight == 0) && (currentTip.HashBlock == peer.Network.GenesisHash)) { yield break; } if ((headers.Headers.Count == 1) && (headers.Headers[0].GetHash() == currentTip.HashBlock)) { yield break; } foreach (BlockHeader header in headers.Headers) { uint256 hash = header.GetHash(); if (hash == currentTip.HashBlock) { continue; } // The previous headers request timeout, this can arrive in case of big reorg. if (header.HashPrevBlock != currentTip.HashBlock) { int reorgDepth = 0; ChainedHeader tempCurrentTip = currentTip; while (reorgDepth != acceptMaxReorgDepth && tempCurrentTip != null && header.HashPrevBlock != tempCurrentTip.HashBlock) { reorgDepth++; tempCurrentTip = tempCurrentTip.Previous; } if (reorgDepth != acceptMaxReorgDepth && tempCurrentTip != null) { currentTip = tempCurrentTip; } } if (header.HashPrevBlock == currentTip.HashBlock) { isOurs = true; currentTip = new ChainedHeader(header, hash, currentTip); yield return(currentTip); if (currentTip.HashBlock == hashStop) { yield break; } } else { break; // Not our headers, continue receive. } } if (isOurs) { break; //Go ask for next header. } } } } }
private void TryUpdateLocation(IEnumerable<Node> nodes) { if(nodes != null) { var current = Chain.FindFork(_ScanLocation); if(current == null) return; var progress = nodes .Select(f => f.Behaviors.Find<TrackerBehavior>()) .Where(f => f != null) .Select(f => f.CurrentProgress) .Where(p => p != null) .Select(l => new { Locator = l, Block = Chain.FindFork(l) }) .Where(o => o.Block.Height > current.Height) .OrderByDescending(o => o.Block.Height) .Select(o => o.Block) .FirstOrDefault(); if(progress != null) { _ScanLocation = progress.GetLocator(); } } }
public GetProvenHeadersPayload(BlockLocator locator) { this.BlockLocator = locator; this.HashStop = uint256.Zero; }
void LoadCore(Stream stream) { JObject obj = JObject.Load(new JsonTextReader(new StreamReader(stream)) { DateParseHandling = DateParseHandling.DateTimeOffset }); _Parameters = WalletCreation.FromJson((JObject)obj["Parameters"]); _PathStates = new Dictionary<KeyPath, PathState>(); if(obj.Property("CurrentIndex") != null) //legacy { var idx = (int)(long)obj["CurrentIndex"]; var loadedKeys = (int)(long)obj["LoadedKeys"]; _PathStates.Add(_Parameters.DerivationPath.Derive(0), new PathState() { Next = idx, Loaded = loadedKeys }); _PathStates.Add(_Parameters.DerivationPath.Derive(1), new PathState() { Next = idx, Loaded = loadedKeys }); } var indices = obj["Indices"] as JArray; if(indices != null) { foreach(var indice in indices.OfType<JObject>()) { _PathStates.Add(KeyPath.Parse((string)indice["KeyPath"]), new PathState() { Next = (int)(long)indice["Next"], Loaded = (int)(long)indice["Loaded"] }); } } _KeyPoolSize = (int)(long)obj["KeyPoolSize"]; Created = (DateTimeOffset)obj["Created"]; _ScanLocation = new BlockLocator(); _ScanLocation.FromBytes(Encoders.Hex.DecodeData((string)obj["Location"])); _KnownScripts.Clear(); var knownScripts = (JArray)obj["KnownScripts"]; foreach(var known in knownScripts.OfType<JObject>()) { Script script = Script.FromBytesUnsafe(Encoders.Hex.DecodeData((string)known["ScriptPubKey"])); if(known["KeyPath"] != null) //Legacy data { KeyPath keypath = KeyPath.Parse((string)known["KeyPath"]); _KnownScripts.Add(script, _Parameters.DerivationPath.Derive(keypath)); } if(known["AbsoluteKeyPath"] != null) { KeyPath keypath = KeyPath.Parse((string)known["AbsoluteKeyPath"]); _KnownScripts.Add(script, keypath); } } }
/// <inheritdoc /> public void Start() { this.signals.OnTransactionReceived.Attach(this.ProcessTransaction); this.signals.OnBlockConnected.Attach(this.OnBlockConnected); // if there is no wallet created yet, the wallet tip is the chain tip. if (!this.walletManager.ContainsWallets) { this.walletTip = this.chain.Tip; } else { this.walletTip = this.chain.GetBlock(this.walletManager.WalletTipHash); if (this.walletTip == null && this.chain.Height > 0) { // the wallet tip was not found in the main chain. // this can happen if the node crashes unexpectedly. // to recover we need to find the first common fork // with the best chain, as the wallet does not have a // list of chain headers we use a BlockLocator and persist // that in the wallet. the block locator will help finding // a common fork and bringing the wallet back to a good // state (behind the best chain) ICollection <uint256> locators = this.walletManager.GetFirstWalletBlockLocator(); var blockLocator = new BlockLocator { Blocks = locators.ToList() }; ChainedHeader fork = this.chain.FindFork(blockLocator); this.walletManager.RemoveBlocks(fork); this.walletManager.WalletTipHash = fork.HashBlock; this.walletTip = fork; this.logger.LogWarning($"Wallet tip was out of sync, wallet tip reverted back to Height = {this.walletTip.Height} hash = {this.walletTip.HashBlock}."); } // we're looking from where to start syncing the wallets. // we start by looking at the heights of the wallets and we start syncing from the oldest one (the smallest height). // if for some reason we can't find a height, we look at the creation date of the wallets and we start syncing from the earliest date. int?earliestWalletHeight = this.walletManager.GetEarliestWalletHeight(); if (earliestWalletHeight == null) { DateTimeOffset oldestWalletDate = this.walletManager.GetOldestWalletCreationTime(); if (oldestWalletDate > this.walletTip.Header.BlockTime) { oldestWalletDate = this.walletTip.Header.BlockTime; } this.SyncFromDate(oldestWalletDate.LocalDateTime); } else { // If we reorged and the fork point is before the earliest wallet height start to // sync from the fork point. // We'll also get into this branch if the chain has been deleted but wallets are present. // In this case, the wallet tip will be null so the next statement will be skipped. if (this.walletTip != null && earliestWalletHeight.Value > this.walletTip.Height) { earliestWalletHeight = this.walletTip.Height; } this.SyncFromHeight(earliestWalletHeight.Value); } } }
static void Main(string[] args) { try { var options = new IndexerOptions(); if (args.Length == 0) { System.Console.WriteLine(options.GetUsage()); } if (Parser.Default.ParseArguments(args, options)) { System.Console.WriteLine("NBitcoin.Indexer " + typeof(AzureIndexer).Assembly.GetName().Version); if (options.All) { options.IndexAddresses = true; options.IndexBlocks = true; options.IndexWallets = true; options.IndexChain = true; options.IndexTransactions = true; } var indexer = AzureIndexer.CreateIndexer(); indexer.Configuration.EnsureSetup(); indexer.TaskScheduler = new CustomThreadPoolTaskScheduler(30, 100); indexer.CheckpointInterval = TimeSpan.Parse(options.CheckpointInterval); indexer.IgnoreCheckpoints = options.IgnoreCheckpoints; indexer.FromHeight = options.From; indexer.ToHeight = options.To; ChainBase chain = null; var checkpointRepository = indexer.GetCheckpointRepository(); checkpointRepository.CheckpointSet = null; if (options.ListCheckpoints) { foreach (var checkpoint in checkpointRepository.GetCheckpointsAsync().Result) { chain = chain ?? indexer.GetNodeChain(); var fork = chain.FindFork(checkpoint.BlockLocator); System.Console.WriteLine("Name : " + checkpoint.CheckpointName); if (fork != null) { System.Console.WriteLine("Height : " + fork.Height); System.Console.WriteLine("Hash : " + fork.HashBlock); } System.Console.WriteLine(); } } if (options.DeleteCheckpoint != null) { checkpointRepository.GetCheckpoint(options.DeleteCheckpoint).DeleteAsync().Wait(); System.Console.WriteLine("Checkpoint " + options.DeleteCheckpoint + " deleted"); } if (options.AddCheckpoint != null) { chain = chain ?? indexer.GetNodeChain(); var split = options.AddCheckpoint.Split(':'); var name = split[0]; var height = int.Parse(split[1]); var b = chain.GetBlock(height); var checkpoint = checkpointRepository.GetCheckpoint(name); checkpoint.SaveProgress(b.GetLocator()); System.Console.WriteLine("Checkpoint " + options.AddCheckpoint + " saved to height " + b.Height); } if (ConfigurationManager.AppSettings["MainDirectory"] != null) { System.Console.WriteLine("Warning : obsolete appsetting detected, MainDirectory"); string[] oldCheckpoints = new string[] { "transactions", "blocks", "wallets", "balances" }; foreach (var chk in oldCheckpoints) { var path = GetFilePath(indexer.Configuration, chk); if (File.Exists(path)) { var onlineCheckpoint = checkpointRepository.GetCheckpointsAsync().Result.FirstOrDefault(r => r.CheckpointName.ToLowerInvariant() == chk); if (onlineCheckpoint == null) { onlineCheckpoint = checkpointRepository.GetCheckpoint(indexer.Configuration.CheckpointSetName + "/" + chk); BlockLocator offlineLocator = new BlockLocator(); offlineLocator.FromBytes(File.ReadAllBytes(path)); onlineCheckpoint.SaveProgress(offlineLocator); System.Console.WriteLine("Local checkpoint " + chk + " saved in azure"); } File.Delete(path); System.Console.WriteLine("Checkpoint File deleted " + path); } } } if (options.IndexBlocks) { chain = chain ?? indexer.GetNodeChain(); indexer.IndexBlocks(chain); } if (options.IndexTransactions) { chain = chain ?? indexer.GetNodeChain(); indexer.IndexTransactions(chain); } if (options.IndexAddresses) { chain = chain ?? indexer.GetNodeChain(); indexer.IndexOrderedBalances(chain); } if (options.IndexWallets) { chain = chain ?? indexer.GetNodeChain(); indexer.IndexWalletBalances(chain); } if (options.IndexChain) { chain = chain ?? indexer.GetNodeChain(); indexer.IndexChain(chain); } } } catch (ConfigurationErrorsException ex) { System.Console.WriteLine("LocalSettings.config missing settings : " + ex.Message); } }