private async Task SaveMatches(Block block)
{
_LastBlockDownloaded = DateTimeOffset.UtcNow;
block.Header.PrecomputeHash(false, false);
foreach (var tx in block.Transactions)
{
tx.PrecomputeHash(false, true);
}
var blockHash = block.GetHash();
var delay = TimeSpan.FromSeconds(1);
retry:
try
{
DateTimeOffset now = DateTimeOffset.UtcNow;
var matches =
(await Repository.GetMatches(block.Transactions, blockHash, now, true))
.ToArray();
await SaveMatches(matches, blockHash, now, true);
var slimBlockHeader = Chain.GetBlock(blockHash);
if (slimBlockHeader != null)
{
var blockEvent = new Models.NewBlockEvent()
{
CryptoCode = _Repository.Network.CryptoCode,
Hash = blockHash,
Height = slimBlockHeader.Height,
PreviousBlockHash = slimBlockHeader.Previous
};
var saving = Repository.SaveEvent(blockEvent);
_EventAggregator.Publish(blockEvent);
await saving;
}
}
catch (ObjectDisposedException)
{
throw;
}
catch (Exception ex)
{
Logs.Explorer.LogWarning(ex, $"{Network.CryptoCode}: Error while saving block in database, retrying in {delay.TotalSeconds} seconds ({ex.Message})");
await Task.Delay(delay, _Cts.Token);
delay = delay * 2;
var maxDelay = TimeSpan.FromSeconds(60);
if (delay > maxDelay)
{
delay = maxDelay;
}
goto retry;
}
if (_InFlights.TryRemove(blockHash, out var unused) && _InFlights.IsEmpty)
{
var highestInFlight = _HighestInFlight;
_HighestInFlight = null;
if (highestInFlight != null)
{
CurrentLocation = highestInFlight;
await Repository.SetIndexProgress(highestInFlight);
}
AskBlocks();
}
}
private void TransferRecentStates(GetRecentStates getRecentStates)
{
BlockLocator baseLocator = getRecentStates.BaseLocator;
HashDigest <SHA256>? @base = BlockChain.FindBranchPoint(baseLocator);
HashDigest <SHA256> target = getRecentStates.TargetBlockHash;
IImmutableDictionary <HashDigest <SHA256>,
IImmutableDictionary <string, IValue>
> blockStates = null;
IImmutableDictionary <string, IImmutableList <HashDigest <SHA256> > > stateRefs = null;
long nextOffset = -1;
int iteration = 0;
if (BlockChain.StateStore is IBlockStatesStore blockStatesStore &&
BlockChain.ContainsBlock(target))
{
ReaderWriterLockSlim rwlock = BlockChain._rwlock;
rwlock.EnterReadLock();
try
{
Guid chainId = BlockChain.Id;
_logger.Debug(
"Getting state references from {Offset}",
getRecentStates.Offset);
long baseIndex =
(@base is HashDigest <SHA256> bbh &&
_store.GetBlockIndex(bbh) is long bbIdx)
? bbIdx
: 0;
long lowestIndex = baseIndex + getRecentStates.Offset;
long targetIndex =
(target is HashDigest <SHA256> tgt &&
_store.GetBlockIndex(tgt) is long tgtIdx)
? tgtIdx
: long.MaxValue;
iteration =
(int)Math.Ceiling(
(double)(targetIndex - baseIndex + 1) / FindNextStatesChunkSize);
long highestIndex = lowestIndex + FindNextStatesChunkSize - 1 > targetIndex
? targetIndex
: lowestIndex + FindNextStatesChunkSize - 1;
nextOffset = highestIndex == targetIndex
? -1
: getRecentStates.Offset + FindNextStatesChunkSize;
stateRefs = blockStatesStore.ListAllStateReferences(
chainId,
lowestIndex: lowestIndex,
highestIndex: highestIndex
);
if (_logger.IsEnabled(LogEventLevel.Verbose))
{
_logger.Verbose(
"List state references from {From} to {To}:\n{StateReferences}",
lowestIndex,
highestIndex,
string.Join(
"\n",
stateRefs.Select(kv => $"{kv.Key}: {string.Join(", ", kv.Value)}")
)
);
}
// GetBlockStates may return null since swarm may not have deep states.
blockStates = stateRefs.Values
.Select(refs => refs.Last())
.ToImmutableHashSet()
.Select(bh => (bh, blockStatesStore.GetBlockStates(bh)))
.Where(pair => !(pair.Item2 is null))
.ToImmutableDictionary(
pair => pair.Item1,
pair => (IImmutableDictionary <string, IValue>)pair.Item2
.ToImmutableDictionary(kv => kv.Key, kv => kv.Value)
);
}
finally
{
rwlock.ExitReadLock();
}
}
if (_logger.IsEnabled(LogEventLevel.Verbose))
{
if (BlockChain.ContainsBlock(target))
{
var baseString = @base is HashDigest <SHA256> h
? $"{BlockChain[h].Index}:{h}"
: null;
var targetString = $"{BlockChain[target].Index}:{target}";
_logger.Verbose(
"State references to send (preload): {StateReferences} ({Base}-{Target})",
stateRefs.Select(kv =>
(
kv.Key,
string.Join(", ", kv.Value.Select(v => v.ToString()))
)
).ToArray(),
baseString,
targetString
);
_logger.Verbose(
"Block states to send (preload): {BlockStates} ({Base}-{Target})",
blockStates.Select(kv => (kv.Key, kv.Value)).ToArray(),
baseString,
targetString
);
}
else
{
_logger.Verbose("Nothing to reply because {TargetHash} doesn't exist.", target);
}
}
var reply = new RecentStates(
target,
nextOffset,
iteration,
blockStates,
stateRefs?.ToImmutableDictionary())
{
Identity = getRecentStates.Identity,
};
Transport.ReplyMessage(reply);
}
public BlockLocator GetCurrentChainLocator()
{
List<StoredBlock> headers = new List<StoredBlock>();
using (BlockchainRepository repo = new BlockchainRepository(conn))
{
StoredBlock lastBlock = repo.GetLastBlockHeader();
if (lastBlock != null)
{
headers = repo.ReadHeadersWithHeight(BlockLocator.GetRequiredBlockHeights(lastBlock.Height));
}
}
BlockLocator locator = new BlockLocator();
foreach (StoredBlock header in headers)
{
locator.AddHash(header.Height, header.Hash);
}
return locator;
}
public GetHeadersPayload(BlockLocator locator)
{
this.BlockLocator = locator;
}
/// <summary>
/// Start a scan, if a scan is already running, will change only if the parameters are anterior
/// </summary>
/// <param name="locator"></param>
public void Scan(BlockLocator locator, DateTimeOffset skipBefore)
{
lock(cs)
{
if(_SkipBefore == default(DateTimeOffset) || skipBefore < _SkipBefore)
_SkipBefore = skipBefore;
if(_CurrentProgress == null || EarlierThanCurrentProgress(locator))
_CurrentProgress = locator;
}
}
/// <inheritdoc />
public Task <ChainedHeader> LoadAsync(ChainedHeader genesisHeader)
{
Task <ChainedHeader> task = Task.Run(() =>
{
ChainedHeader tip = null;
ChainData data = this.chainStore.GetChainData(0);
if (data == null)
{
genesisHeader.SetChainStore(this.chainStore);
return(genesisHeader);
}
Guard.Assert(data.Hash == genesisHeader.HashBlock); // can't swap networks
int height = 0;
while (true)
{
data = this.chainStore.GetChainData(height);
if (data == null)
{
break;
}
tip = new ChainedHeader(data.Hash, data.Work, tip);
if (tip.Height == 0)
{
tip.SetChainStore(this.chainStore);
}
if (height % 50_000 == 0)
{
if (this.signals != null)
{
this.signals.Publish(new FullNodeEvent()
{
Message = $"Loading chain at height {height}.", State = FullNodeState.Initializing.ToString()
});
}
this.logger.LogInformation($"Loading chain at height {height}.");
}
height++;
}
if (tip == null)
{
genesisHeader.SetChainStore(this.chainStore);
tip = genesisHeader;
}
else
{
// Confirm that the chain tip exists in the headers table.
this.chainStore.GetHeader(tip, tip.HashBlock);
}
this.locator = tip.GetLocator();
return(tip);
});
return(task);
}
public int Run(ChainBase chain = null)
{
ListenerTrace.Info("Start initial indexing");
int totalProcessed = 0;
using (var node = _Conf.Indexer.ConnectToNode(false))
{
ListenerTrace.Info("Handshaking...");
node.VersionHandshake();
ListenerTrace.Info("Handshaked");
chain = chain ?? node.GetChain();
ListenerTrace.Info("Current chain at height " + chain.Height);
var blockRepository = new NodeBlocksRepository(node);
var blobLock = GetInitBlob();
string lease = null;
try
{
blobLock.UploadText("Enqueuing");
lease = blobLock.AcquireLease(null, null);
}
catch (StorageException)
{
}
if (lease != null)
{
ListenerTrace.Info("Queueing index jobs");
EnqueueJobs(blockRepository, chain, blobLock, lease);
}
ListenerTrace.Info("Dequeuing index jobs");
while (true)
{
var msg = _Conf.Topics
.InitialIndexing
.ReceiveAsync(TimeSpan.FromMilliseconds(1000))
.Result;
var ns = _Conf.Topics.InitialIndexing.GetNamespace();
var description = ns.GetQueue(_Conf.Topics.InitialIndexing.Queue);
Console.WriteLine("Work remaining in the queue : " + description.MessageCountDetails.ActiveMessageCount);
if (msg == null)
{
var state = blobLock.DownloadText();
if (state == "Enqueuing" || description.MessageCountDetails.ActiveMessageCount != 0)
{
ListenerTrace.Info("Additional work will be enqueued...");
continue;
}
else
{
var locator = new BlockLocator();
locator.FromBytes(Encoders.Hex.DecodeData(state));
UpdateCheckpoints(locator);
break;
}
}
using (msg.Message)
{
var range = msg.Body;
using (var sched = new CustomThreadPoolTaskScheduler(50, 100, range.ToString()))
{
ListenerTrace.Info("Processing " + range.ToString());
totalProcessed++;
var task = _IndexTasks[range.Target];
BlockFetcher fetcher = new BlockFetcher(task.Item1, blockRepository, chain)
{
FromHeight = range.From,
ToHeight = range.From + range.Count - 1
};
try
{
task.Item2.SaveProgression = false;
task.Item2.EnsureIsSetup = totalProcessed == 0;
var index = Task.Factory.StartNew(() =>
{
task.Item2.Index(fetcher, sched);
}, TaskCreationOptions.LongRunning);
while (!index.Wait(TimeSpan.FromMinutes(4)))
{
msg.Message.RenewLock();
ListenerTrace.Info("Lock renewed");
}
}
catch (AggregateException aex)
{
ExceptionDispatchInfo.Capture(aex.InnerException).Throw();
throw;
}
range.Processed = true;
msg.Message.Complete();
}
}
}
}
ListenerTrace.Info("Initial indexing terminated");
return(totalProcessed);
}
public GetHeadersPayload(BlockLocator locator)
{
BlockLocators = locator;
}
private void TryUpdateLocation()
{
var group = _Group;
if(group != null)
{
var progress =
group.ConnectedNodes
.Select(f => f.Behaviors.Find<TrackerBehavior>().CurrentProgress)
.Where(p => p != null)
.Select(l => new
{
Locator = l,
Block = Chain.FindFork(l)
})
.OrderByDescending(o => o.Block.Height)
.Select(o => o.Block)
.FirstOrDefault();
if(progress != null)
{
progress = progress.EnumerateToGenesis().Skip(5).FirstOrDefault() ?? progress; //Step down 5 blocks, it does not cost a lot to rescan them in case we missed something
_ScanLocation = progress.GetLocator();
}
}
}
public GetBlocksPayload(BlockLocator locator)
{
BlockLocators = locator;
}
public GetBlocks(
BlockLocator locator, HashDigest <SHA256>?stop)
{
Locator = locator;
Stop = stop;
}
/// <summary>
/// Attempts to connect to a full BitCoin node and request any missing block headers.
/// </summary>
private static async Task <Node> ConnectNodeAndSyncHeaders(ConcurrentChain chain, CancellationToken ct)
{
logger.DebugFormat("Connecting to full node...");
ct.ThrowIfCancellationRequested();
ManualResetEventSlim headersSyncedSignal = new ManualResetEventSlim();
var parameters = new NodeConnectionParameters();
parameters.IsRelay = false;
var scanLocation = new BlockLocator();
scanLocation.Blocks.Add(chain.Tip != null ? chain.Tip.HashBlock : _network.GetGenesis().GetHash());
var node = Node.ConnectToLocal(_network, parameters);
logger.DebugFormat("Connected to node " + node.RemoteSocketEndpoint + ".");
node.MessageReceived += (node1, message) =>
{
ct.ThrowIfCancellationRequested();
switch (message.Message.Payload)
{
case HeadersPayload hdr:
if (hdr.Headers != null && hdr.Headers.Count > 0)
{
logger.DebugFormat("Received {0} blocks start {1} to {2} height {3}.", hdr.Headers.Count, hdr.Headers.First().BlockTime, hdr.Headers.Last().BlockTime, chain.Height);
scanLocation.Blocks.Clear();
scanLocation.Blocks.Add(hdr.Headers.Last().GetHash());
if (hdr != null)
{
var tip = chain.Tip;
foreach (var header in hdr.Headers)
{
var prev = tip.FindAncestorOrSelf(header.HashPrevBlock);
if (prev == null)
{
break;
}
tip = new ChainedBlock(header, header.GetHash(), prev);
chain.SetTip(tip);
ct.ThrowIfCancellationRequested();
}
}
var getHeadersPayload = new GetHeadersPayload(scanLocation);
node.SendMessageAsync(getHeadersPayload);
}
else
{
// Headers synchronised.
logger.DebugFormat("Block headers synchronised.");
headersSyncedSignal.Set();
}
break;
case InvPayload inv:
logger.DebugFormat("Inventory items {0}, first type {1}.", inv.Count(), inv.First().Type);
if (inv.Any(x => x.Type == InventoryType.MSG_BLOCK))
{
// New block available.
var getHeadersPayload = new GetHeadersPayload(scanLocation);
node.SendMessage(getHeadersPayload);
}
break;
case MerkleBlockPayload merkleBlk:
break;
case TxPayload tx:
break;
default:
logger.DebugFormat(message.Message.Command);
break;
}
};
node.Disconnected += n =>
{
logger.DebugFormat("Node disconnected, chain height " + chain.Height + ".");
};
node.VersionHandshake(ct);
node.PingPong(ct);
logger.DebugFormat("Requesting block headers greater than height {0}.", chain.Height);
node.SendMessage(new GetHeadersPayload(scanLocation));
logger.DebugFormat("Bitcoin node connected.");
await Task.Run(() => {
headersSyncedSignal.Wait(ct);
});
return(node);
}
#pragma warning disable MEN003
private async Task <BlockChain <T> > FillBlocksAsync(
BoundPeer peer,
BlockChain <T> blockChain,
BlockHash?stop,
IProgress <BlockDownloadState> progress,
long totalBlockCount,
long receivedBlockCount,
bool evaluateActions,
TimeSpan timeout,
int logSessionId,
CancellationToken cancellationToken
)
{
var sessionRandom = new Random();
const string fname = nameof(FillBlocksAsync);
BlockChain <T> workspace = blockChain;
var scope = new List <Guid>();
bool renderActions = evaluateActions;
bool renderBlocks = true;
try
{
while (!cancellationToken.IsCancellationRequested)
{
int subSessionId = sessionRandom.Next();
Block <T> tip = workspace?.Tip;
_logger.Debug(
"{SessionId}/{SubSessionId}: Trying to find branchpoint...",
logSessionId,
subSessionId
);
BlockLocator locator = workspace.GetBlockLocator();
_logger.Debug(
"{SessionId}/{SubSessionId}: Locator's length: {LocatorLength}",
logSessionId,
subSessionId,
locator.Count()
);
IAsyncEnumerable <Tuple <long, BlockHash> > hashesAsync = GetBlockHashes(
peer: peer,
locator: locator,
stop: stop,
timeout: timeout,
logSessionIds: (logSessionId, subSessionId),
cancellationToken: cancellationToken
);
IEnumerable <Tuple <long, BlockHash> > hashes = await hashesAsync.ToArrayAsync();
if (!hashes.Any())
{
_logger.Debug(
"{SessionId}/{SubSessionId}: Peer {0} returned no hashes; ignored.",
logSessionId,
subSessionId,
peer.Address.ToHex()
);
return(workspace);
}
hashes.First().Deconstruct(
out long branchIndex,
out BlockHash branchpoint
);
_logger.Debug(
"{SessionId}/{SubSessionId}: Branchpoint is #{BranchIndex} {BranchHash}.",
logSessionId,
subSessionId,
branchIndex,
branchpoint
);
if (tip is null || branchpoint.Equals(tip.Hash))
{
_logger.Debug(
"{SessionId}/{SubSessionId}: It doesn't need to fork.",
logSessionId,
subSessionId
);
}
else if (!workspace.ContainsBlock(branchpoint))
{
// FIXME: This behavior can unexpectedly terminate the swarm (and the game
// app) if it encounters a peer having a different blockchain, and therefore
// can be exploited to remotely shut down other nodes as well.
// Since the intention of this behavior is to prevent mistakes to try to
// connect incorrect seeds (by a user), this behavior should be limited for
// only seed peers.
var msg =
$"Since the genesis block is fixed to {BlockChain.Genesis} " +
"protocol-wise, the blockchain which does not share " +
"any mutual block is not acceptable.";
throw new InvalidGenesisBlockException(
branchpoint,
workspace.Genesis.Hash,
msg);
}
else
{
_logger.Debug(
"{SessionId}/{SubSessionId}: Needs to fork; trying to fork...",
logSessionId,
subSessionId
);
workspace = workspace.Fork(branchpoint);
Guid workChainId = workspace.Id;
scope.Add(workChainId);
renderActions = false;
renderBlocks = false;
_logger.Debug(
"{SessionId}/{SubSessionId}: Fork finished.",
logSessionId,
subSessionId
);
}
if (!(workspace.Tip is null))
{
hashes = hashes.Skip(1);
}
_logger.Debug(
"{SessionId}/{SubSessionId}: Trying to fill up previous blocks...",
logSessionId,
subSessionId
);
var hashesAsArray = hashes as Tuple <long, BlockHash>[] ?? hashes.ToArray();
if (!hashesAsArray.Any())
{
break;
}
int hashCount = hashesAsArray.Count();
_logger.Debug(
"{SessionId}/{SubSessionId}: Required {Hashes} hashes " +
"(tip: #{TipIndex} {TipHash}).",
logSessionId,
subSessionId,
hashCount,
workspace.Tip?.Index,
workspace.Tip?.Hash
);
totalBlockCount = Math.Max(totalBlockCount, receivedBlockCount + hashCount);
IAsyncEnumerable <Block <T> > blocks = GetBlocksAsync(
peer,
hashesAsArray.Select(pair => pair.Item2),
cancellationToken
);
var receivedBlockCountCurrentLoop = 0;
await foreach (Block <T> block in blocks)
{
const string startMsg =
"{SessionId}/{SubSessionId}: Try to append a block " +
"#{BlockIndex} {BlockHash}...";
_logger.Debug(
startMsg,
logSessionId,
subSessionId,
block.Index,
block.Hash
);
cancellationToken.ThrowIfCancellationRequested();
workspace.Append(
block,
DateTimeOffset.UtcNow,
evaluateActions: evaluateActions,
renderBlocks: renderBlocks,
renderActions: renderActions
);
receivedBlockCountCurrentLoop++;
progress?.Report(new BlockDownloadState
{
TotalBlockCount = totalBlockCount,
ReceivedBlockCount = receivedBlockCount + receivedBlockCountCurrentLoop,
ReceivedBlockHash = block.Hash,
SourcePeer = peer,
});
const string endMsg =
"{SessionId}/{SubSessionId}: Block #{BlockIndex} {BlockHash} " +
"was appended.";
_logger.Debug(endMsg, logSessionId, subSessionId, block.Index, block.Hash);
}
receivedBlockCount += receivedBlockCountCurrentLoop;
var isEndedFirstTime = receivedBlockCount == receivedBlockCountCurrentLoop &&
receivedBlockCount < FindNextHashesChunkSize - 1;
if (receivedBlockCountCurrentLoop < FindNextHashesChunkSize && isEndedFirstTime)
{
_logger.Debug(
"{SessionId}/{SubSessionId}: Got {Blocks} blocks from Peer {Peer} " +
"(tip: #{TipIndex} {TipHash})",
logSessionId,
subSessionId,
receivedBlockCountCurrentLoop,
peer.Address.ToHex(),
workspace.Tip?.Index,
workspace.Tip?.Hash
);
break;
}
}
public GetRecentStates(BlockLocator baseLocator, HashDigest <SHA256> target)
{
BaseLocator = baseLocator;
TargetBlockHash = target;
}
static void Main(string[] args)
{
try
{
var options = new IndexerOptions();
if (args.Length == 0)
System.Console.WriteLine(options.GetUsage());
if (Parser.Default.ParseArguments(args, options))
{
System.Console.WriteLine("NBitcoin.Indexer " + typeof(AzureIndexer).Assembly.GetName().Version);
if (options.All)
{
options.IndexAddresses = true;
options.IndexBlocks = true;
options.IndexWallets = true;
options.IndexChain = true;
options.IndexTransactions = true;
}
var indexer = AzureIndexer.CreateIndexer();
indexer.Configuration.EnsureSetup();
indexer.TaskScheduler = new CustomThreadPoolTaskScheduler(30, 100);
indexer.CheckpointInterval = TimeSpan.Parse(options.CheckpointInterval);
indexer.IgnoreCheckpoints = options.IgnoreCheckpoints;
indexer.FromHeight = options.From;
indexer.ToHeight = options.To;
ChainBase chain = null;
var checkpointRepository = indexer.GetCheckpointRepository();
checkpointRepository.CheckpointSet = null;
if (options.ListCheckpoints)
{
foreach (var checkpoint in checkpointRepository.GetCheckpointsAsync().Result)
{
chain = chain ?? indexer.GetNodeChain();
var fork = chain.FindFork(checkpoint.BlockLocator);
System.Console.WriteLine("Name : " + checkpoint.CheckpointName);
if (fork != null)
{
System.Console.WriteLine("Height : " + fork.Height);
System.Console.WriteLine("Hash : " + fork.HashBlock);
}
System.Console.WriteLine();
}
}
if (options.DeleteCheckpoint != null)
{
checkpointRepository.GetCheckpoint(options.DeleteCheckpoint).DeleteAsync().Wait();
System.Console.WriteLine("Checkpoint " + options.DeleteCheckpoint + " deleted");
}
if (options.AddCheckpoint != null)
{
chain = chain ?? indexer.GetNodeChain();
var split = options.AddCheckpoint.Split(':');
var name = split[0];
var height = int.Parse(split[1]);
var b = chain.GetBlock(height);
var checkpoint = checkpointRepository.GetCheckpoint(name);
checkpoint.SaveProgress(b.GetLocator());
System.Console.WriteLine("Checkpoint " + options.AddCheckpoint + " saved to height " + b.Height);
}
if (ConfigurationManager.AppSettings["MainDirectory"] != null)
{
System.Console.WriteLine("Warning : obsolete appsetting detected, MainDirectory");
string[] oldCheckpoints = new string[] { "transactions", "blocks", "wallets", "balances" };
foreach (var chk in oldCheckpoints)
{
var path = GetFilePath(indexer.Configuration, chk);
if (File.Exists(path))
{
var onlineCheckpoint = checkpointRepository.GetCheckpointsAsync().Result.FirstOrDefault(r => r.CheckpointName.ToLowerInvariant() == chk);
if (onlineCheckpoint == null)
{
onlineCheckpoint = checkpointRepository.GetCheckpoint(indexer.Configuration.CheckpointSetName + "/" + chk);
BlockLocator offlineLocator = new BlockLocator();
offlineLocator.FromBytes(File.ReadAllBytes(path));
onlineCheckpoint.SaveProgress(offlineLocator);
System.Console.WriteLine("Local checkpoint " + chk + " saved in azure");
}
File.Delete(path);
System.Console.WriteLine("Checkpoint File deleted " + path);
}
}
}
if (options.IndexBlocks)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexBlocks(chain);
}
if (options.IndexTransactions)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexTransactions(chain);
}
if (options.IndexAddresses)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexOrderedBalances(chain);
}
if (options.IndexWallets)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexWalletBalances(chain);
}
if (options.IndexChain)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexChain(chain);
}
}
}
catch (ConfigurationErrorsException ex)
{
System.Console.WriteLine("LocalSettings.config missing settings : " + ex.Message);
}
}
void LoadCore(Stream stream)
{
JObject obj = JObject.Load(new JsonTextReader(new StreamReader(stream))
{
DateParseHandling = DateParseHandling.DateTimeOffset
});
_CurrentIndex = (int)(long)obj["CurrentIndex"];
_KeyPoolSize = (int)(long)obj["KeyPoolSize"];
_LoadedKeys = (int)(long)obj["LoadedKeys"];
Created = (DateTimeOffset)obj["Created"];
_ScanLocation = new BlockLocator();
_ScanLocation.FromBytes(Encoders.Hex.DecodeData((string)obj["Location"]));
_Parameters = WalletCreation.FromJson((JObject)obj["Parameters"]);
_KnownScripts.Clear();
var knownScripts = (JArray)obj["KnownScripts"];
foreach(var known in knownScripts.OfType<JObject>())
{
Script script = Script.FromBytesUnsafe(Encoders.Hex.DecodeData((string)known["ScriptPubKey"]));
KeyPath keypath = KeyPath.Parse((string)known["KeyPath"]);
_KnownScripts.Add(script, keypath);
}
}
public GetBlockHashes(BlockLocator locator, BlockHash?stop)
{
Locator = locator;
Stop = stop;
}
public GetHeadersPayload(BlockLocator locator)
{
BlockLocators = locator;
}
private void AttachedNode_MessageReceived(Node node, IncomingMessage message)
{
message.Message.IfPayloadIs <InvPayload>(invs =>
{
var data = new GetDataPayload();
foreach (var inv in invs.Inventory)
{
inv.Type = node.AddSupportedOptions(inv.Type);
if (inv.Type.HasFlag(InventoryType.MSG_TX))
{
data.Inventory.Add(inv);
}
}
if (data.Inventory.Count != 0)
{
node.SendMessageAsync(data);
}
});
message.Message.IfPayloadIs <HeadersPayload>(headers =>
{
if (headers.Headers.Count == 0)
{
return;
}
AskBlocks();
});
message.Message.IfPayloadIs <BlockPayload>(block =>
{
block.Object.Header.PrecomputeHash(false, false);
Download o;
if (_InFlights.ContainsKey(block.Object.GetHash()))
{
var blockHeader = Chain.GetBlock(block.Object.GetHash());
if (blockHeader == null)
{
return;
}
var currentLocation = blockHeader.GetLocator();
_CurrentLocation = currentLocation;
if (_InFlights.TryRemove(block.Object.GetHash(), out o))
{
foreach (var tx in block.Object.Transactions)
{
tx.PrecomputeHash(false, true);
}
var matches =
block.Object.Transactions
.SelectMany(tx => Repository.GetMatches(tx))
.ToArray();
var blockHash = block.Object.GetHash();
SaveMatches(matches, blockHash);
//Save index progress everytimes if not synching, or once every 100 blocks otherwise
if (!IsSynching() || blockHash.GetLow32() % 100 == 0)
{
Repository.SetIndexProgress(currentLocation);
}
_EventAggregator.Publish(new Events.NewBlockEvent(this._Repository.Network.CryptoCode, blockHash));
}
if (_InFlights.Count == 0)
{
AskBlocks();
}
}
});
message.Message.IfPayloadIs <TxPayload>(txPayload =>
{
var matches = Repository.GetMatches(txPayload.Object).ToArray();
SaveMatches(matches, null);
});
}
private void AttachedNode_MessageReceived(Node node, IncomingMessage message)
{
message.Message.IfPayloadIs <InvPayload>(invs =>
{
var data = new GetDataPayload();
foreach (var inv in invs.Inventory)
{
inv.Type = node.AddSupportedOptions(inv.Type);
if (inv.Type.HasFlag(InventoryType.MSG_TX))
{
data.Inventory.Add(inv);
}
}
if (data.Inventory.Count != 0)
{
node.SendMessageAsync(data);
}
});
message.Message.IfPayloadIs <HeadersPayload>(headers =>
{
if (headers.Headers.Count == 0)
{
return;
}
AskBlocks();
});
message.Message.IfPayloadIs <BlockPayload>(block =>
{
block.Object.Header.CacheHashes();
Download o;
if (_InFlights.ContainsKey(block.Object.GetHash()))
{
var blockHeader = Chain.GetBlock(block.Object.GetHash());
if (blockHeader == null)
{
return;
}
var currentLocation = blockHeader.GetLocator();
_CurrentLocation = currentLocation;
if (_InFlights.TryRemove(block.Object.GetHash(), out o))
{
if (!IsSynching())
{
var unused = _Callbacks.SendCallbacks(block.Object.GetHash());
}
foreach (var tx in block.Object.Transactions)
{
tx.CacheHashes();
}
var matches =
block.Object.Transactions
.SelectMany(tx => GetMatches(tx))
.ToArray();
var blockHash = block.Object.GetHash();
SaveMatches(matches, blockHash);
if (!IsSynching())
{
var unused = _Callbacks.SendCallbacks(matches);
}
//Save index progress everytimes if not synching, or once every 100 blocks otherwise
if (!IsSynching() || blockHash.GetLow32() % 100 == 0)
{
Repository.SetIndexProgress(currentLocation);
}
Logs.Explorer.LogInformation($"Processed block {blockHash}");
}
if (_InFlights.Count == 0)
{
AskBlocks();
}
}
});
message.Message.IfPayloadIs <TxPayload>(txPayload =>
{
var matches = GetMatches(txPayload.Object).ToArray();
SaveMatches(matches, null);
});
}
public GetBlocksPayload(BlockLocator locator)
{
this.BlockLocators = locator;
}
public void SaveProgress(BlockLocator locator)
{
_BlockLocator = locator;
File.WriteAllBytes(_FileName, _BlockLocator.ToBytes());
}
private bool EarlierThanCurrentProgress(BlockLocator locator)
{
return(_Chain.FindFork(locator).Height < _Chain.FindFork(_CurrentProgress).Height);
}
/// <summary>
/// Start a scan, if a scan is already running, will change only if the parameters are anterior
/// </summary>
/// <param name="locator"></param>
public void Scan(BlockLocator locator, DateTimeOffset skipBefore)
{
lock(cs)
{
if(_SkipBefore == default(DateTimeOffset) || skipBefore < _SkipBefore)
_SkipBefore = skipBefore;
if(_CurrentProgress == null || _Chain.FindFork(locator).Height < _Chain.FindFork(_CurrentProgress).Height)
_CurrentProgress = locator;
}
}
private bool EarlierThanCurrentProgress(BlockLocator locator)
{
return _Chain.FindFork(locator).Height < _Chain.FindFork(_CurrentProgress).Height;
}
/// <summary>
/// Create a new wallet
/// </summary>
/// <param name="creation">Creation parameters</param>
/// <param name="keyPoolSize">The number of keys which will be pre-created</param>
public Wallet(WalletCreation creation, int keyPoolSize = 500)
{
if(creation == null)
throw new ArgumentNullException("creation");
_Parameters = creation;
_ScanLocation = new BlockLocator();
_ScanLocation.Blocks.Add(creation.Network.GetGenesis().GetHash());
_KeyPoolSize = keyPoolSize;
Created = DateTimeOffset.UtcNow;
}
public IEnumerable <ChainedHeader> GetHeadersFromFork(INetworkPeer peer, ChainedHeader currentTip, uint256 hashStop = null, CancellationToken cancellationToken = default(CancellationToken))
{
this.AssertStateAsync(peer, NetworkPeerState.HandShaked, cancellationToken).GetAwaiter().GetResult();
using (var listener = new NetworkPeerListener(peer, this.GetOrCreateAsyncProvider()))
{
int acceptMaxReorgDepth = 0;
while (true)
{
// Get before last so, at the end, we should only receive 1 header equals to this one (so we will not have race problems with concurrent GetChains).
BlockLocator awaited = currentTip.Previous == null?currentTip.GetLocator() : currentTip.Previous.GetLocator();
peer.SendMessageAsync(new GetHeadersPayload()
{
BlockLocator = awaited,
HashStop = hashStop
}, cancellationToken).GetAwaiter().GetResult();
while (true)
{
bool isOurs = false;
HeadersPayload headers = null;
using (CancellationTokenSource headersCancel = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
headersCancel.CancelAfter(TimeSpan.FromMinutes(1.0));
try
{
headers = listener.ReceivePayloadAsync <HeadersPayload>(headersCancel.Token).GetAwaiter().GetResult();
}
catch (OperationCanceledException)
{
acceptMaxReorgDepth += 6;
if (cancellationToken.IsCancellationRequested)
{
throw;
}
// Send a new GetHeaders.
break;
}
}
// In the special case where the remote node is at height 0 as well as us, then the headers count will be 0.
if ((headers.Headers.Count == 0) && (peer.PeerVersion.StartHeight == 0) && (currentTip.HashBlock == peer.Network.GenesisHash))
{
yield break;
}
if ((headers.Headers.Count == 1) && (headers.Headers[0].GetHash() == currentTip.HashBlock))
{
yield break;
}
foreach (BlockHeader header in headers.Headers)
{
uint256 hash = header.GetHash();
if (hash == currentTip.HashBlock)
{
continue;
}
// The previous headers request timeout, this can arrive in case of big reorg.
if (header.HashPrevBlock != currentTip.HashBlock)
{
int reorgDepth = 0;
ChainedHeader tempCurrentTip = currentTip;
while (reorgDepth != acceptMaxReorgDepth && tempCurrentTip != null && header.HashPrevBlock != tempCurrentTip.HashBlock)
{
reorgDepth++;
tempCurrentTip = tempCurrentTip.Previous;
}
if (reorgDepth != acceptMaxReorgDepth && tempCurrentTip != null)
{
currentTip = tempCurrentTip;
}
}
if (header.HashPrevBlock == currentTip.HashBlock)
{
isOurs = true;
currentTip = new ChainedHeader(header, hash, currentTip);
yield return(currentTip);
if (currentTip.HashBlock == hashStop)
{
yield break;
}
}
else
{
break; // Not our headers, continue receive.
}
}
if (isOurs)
{
break; //Go ask for next header.
}
}
}
}
}
private void TryUpdateLocation(IEnumerable<Node> nodes)
{
if(nodes != null)
{
var current = Chain.FindFork(_ScanLocation);
if(current == null)
return;
var progress =
nodes
.Select(f => f.Behaviors.Find<TrackerBehavior>())
.Where(f => f != null)
.Select(f => f.CurrentProgress)
.Where(p => p != null)
.Select(l => new
{
Locator = l,
Block = Chain.FindFork(l)
})
.Where(o => o.Block.Height > current.Height)
.OrderByDescending(o => o.Block.Height)
.Select(o => o.Block)
.FirstOrDefault();
if(progress != null)
{
_ScanLocation = progress.GetLocator();
}
}
}
public GetProvenHeadersPayload(BlockLocator locator)
{
this.BlockLocator = locator;
this.HashStop = uint256.Zero;
}
void LoadCore(Stream stream)
{
JObject obj = JObject.Load(new JsonTextReader(new StreamReader(stream))
{
DateParseHandling = DateParseHandling.DateTimeOffset
});
_Parameters = WalletCreation.FromJson((JObject)obj["Parameters"]);
_PathStates = new Dictionary<KeyPath, PathState>();
if(obj.Property("CurrentIndex") != null) //legacy
{
var idx = (int)(long)obj["CurrentIndex"];
var loadedKeys = (int)(long)obj["LoadedKeys"];
_PathStates.Add(_Parameters.DerivationPath.Derive(0), new PathState()
{
Next = idx,
Loaded = loadedKeys
});
_PathStates.Add(_Parameters.DerivationPath.Derive(1), new PathState()
{
Next = idx,
Loaded = loadedKeys
});
}
var indices = obj["Indices"] as JArray;
if(indices != null)
{
foreach(var indice in indices.OfType<JObject>())
{
_PathStates.Add(KeyPath.Parse((string)indice["KeyPath"]), new PathState()
{
Next = (int)(long)indice["Next"],
Loaded = (int)(long)indice["Loaded"]
});
}
}
_KeyPoolSize = (int)(long)obj["KeyPoolSize"];
Created = (DateTimeOffset)obj["Created"];
_ScanLocation = new BlockLocator();
_ScanLocation.FromBytes(Encoders.Hex.DecodeData((string)obj["Location"]));
_KnownScripts.Clear();
var knownScripts = (JArray)obj["KnownScripts"];
foreach(var known in knownScripts.OfType<JObject>())
{
Script script = Script.FromBytesUnsafe(Encoders.Hex.DecodeData((string)known["ScriptPubKey"]));
if(known["KeyPath"] != null) //Legacy data
{
KeyPath keypath = KeyPath.Parse((string)known["KeyPath"]);
_KnownScripts.Add(script, _Parameters.DerivationPath.Derive(keypath));
}
if(known["AbsoluteKeyPath"] != null)
{
KeyPath keypath = KeyPath.Parse((string)known["AbsoluteKeyPath"]);
_KnownScripts.Add(script, keypath);
}
}
}
/// <inheritdoc />
public void Start()
{
this.signals.OnTransactionReceived.Attach(this.ProcessTransaction);
this.signals.OnBlockConnected.Attach(this.OnBlockConnected);
// if there is no wallet created yet, the wallet tip is the chain tip.
if (!this.walletManager.ContainsWallets)
{
this.walletTip = this.chain.Tip;
}
else
{
this.walletTip = this.chain.GetBlock(this.walletManager.WalletTipHash);
if (this.walletTip == null && this.chain.Height > 0)
{
// the wallet tip was not found in the main chain.
// this can happen if the node crashes unexpectedly.
// to recover we need to find the first common fork
// with the best chain, as the wallet does not have a
// list of chain headers we use a BlockLocator and persist
// that in the wallet. the block locator will help finding
// a common fork and bringing the wallet back to a good
// state (behind the best chain)
ICollection <uint256> locators = this.walletManager.GetFirstWalletBlockLocator();
var blockLocator = new BlockLocator {
Blocks = locators.ToList()
};
ChainedHeader fork = this.chain.FindFork(blockLocator);
this.walletManager.RemoveBlocks(fork);
this.walletManager.WalletTipHash = fork.HashBlock;
this.walletTip = fork;
this.logger.LogWarning($"Wallet tip was out of sync, wallet tip reverted back to Height = {this.walletTip.Height} hash = {this.walletTip.HashBlock}.");
}
// we're looking from where to start syncing the wallets.
// we start by looking at the heights of the wallets and we start syncing from the oldest one (the smallest height).
// if for some reason we can't find a height, we look at the creation date of the wallets and we start syncing from the earliest date.
int?earliestWalletHeight = this.walletManager.GetEarliestWalletHeight();
if (earliestWalletHeight == null)
{
DateTimeOffset oldestWalletDate = this.walletManager.GetOldestWalletCreationTime();
if (oldestWalletDate > this.walletTip.Header.BlockTime)
{
oldestWalletDate = this.walletTip.Header.BlockTime;
}
this.SyncFromDate(oldestWalletDate.LocalDateTime);
}
else
{
// If we reorged and the fork point is before the earliest wallet height start to
// sync from the fork point.
// We'll also get into this branch if the chain has been deleted but wallets are present.
// In this case, the wallet tip will be null so the next statement will be skipped.
if (this.walletTip != null && earliestWalletHeight.Value > this.walletTip.Height)
{
earliestWalletHeight = this.walletTip.Height;
}
this.SyncFromHeight(earliestWalletHeight.Value);
}
}
}
static void Main(string[] args)
{
try
{
var options = new IndexerOptions();
if (args.Length == 0)
{
System.Console.WriteLine(options.GetUsage());
}
if (Parser.Default.ParseArguments(args, options))
{
System.Console.WriteLine("NBitcoin.Indexer " + typeof(AzureIndexer).Assembly.GetName().Version);
if (options.All)
{
options.IndexAddresses = true;
options.IndexBlocks = true;
options.IndexWallets = true;
options.IndexChain = true;
options.IndexTransactions = true;
}
var indexer = AzureIndexer.CreateIndexer();
indexer.Configuration.EnsureSetup();
indexer.TaskScheduler = new CustomThreadPoolTaskScheduler(30, 100);
indexer.CheckpointInterval = TimeSpan.Parse(options.CheckpointInterval);
indexer.IgnoreCheckpoints = options.IgnoreCheckpoints;
indexer.FromHeight = options.From;
indexer.ToHeight = options.To;
ChainBase chain = null;
var checkpointRepository = indexer.GetCheckpointRepository();
checkpointRepository.CheckpointSet = null;
if (options.ListCheckpoints)
{
foreach (var checkpoint in checkpointRepository.GetCheckpointsAsync().Result)
{
chain = chain ?? indexer.GetNodeChain();
var fork = chain.FindFork(checkpoint.BlockLocator);
System.Console.WriteLine("Name : " + checkpoint.CheckpointName);
if (fork != null)
{
System.Console.WriteLine("Height : " + fork.Height);
System.Console.WriteLine("Hash : " + fork.HashBlock);
}
System.Console.WriteLine();
}
}
if (options.DeleteCheckpoint != null)
{
checkpointRepository.GetCheckpoint(options.DeleteCheckpoint).DeleteAsync().Wait();
System.Console.WriteLine("Checkpoint " + options.DeleteCheckpoint + " deleted");
}
if (options.AddCheckpoint != null)
{
chain = chain ?? indexer.GetNodeChain();
var split = options.AddCheckpoint.Split(':');
var name = split[0];
var height = int.Parse(split[1]);
var b = chain.GetBlock(height);
var checkpoint = checkpointRepository.GetCheckpoint(name);
checkpoint.SaveProgress(b.GetLocator());
System.Console.WriteLine("Checkpoint " + options.AddCheckpoint + " saved to height " + b.Height);
}
if (ConfigurationManager.AppSettings["MainDirectory"] != null)
{
System.Console.WriteLine("Warning : obsolete appsetting detected, MainDirectory");
string[] oldCheckpoints = new string[] { "transactions", "blocks", "wallets", "balances" };
foreach (var chk in oldCheckpoints)
{
var path = GetFilePath(indexer.Configuration, chk);
if (File.Exists(path))
{
var onlineCheckpoint = checkpointRepository.GetCheckpointsAsync().Result.FirstOrDefault(r => r.CheckpointName.ToLowerInvariant() == chk);
if (onlineCheckpoint == null)
{
onlineCheckpoint = checkpointRepository.GetCheckpoint(indexer.Configuration.CheckpointSetName + "/" + chk);
BlockLocator offlineLocator = new BlockLocator();
offlineLocator.FromBytes(File.ReadAllBytes(path));
onlineCheckpoint.SaveProgress(offlineLocator);
System.Console.WriteLine("Local checkpoint " + chk + " saved in azure");
}
File.Delete(path);
System.Console.WriteLine("Checkpoint File deleted " + path);
}
}
}
if (options.IndexBlocks)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexBlocks(chain);
}
if (options.IndexTransactions)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexTransactions(chain);
}
if (options.IndexAddresses)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexOrderedBalances(chain);
}
if (options.IndexWallets)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexWalletBalances(chain);
}
if (options.IndexChain)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexChain(chain);
}
}
}
catch (ConfigurationErrorsException ex)
{
System.Console.WriteLine("LocalSettings.config missing settings : " + ex.Message);
}
}
