private void EnqueueRange(ChainBase chain, ChainedBlock startCumul, int blockCount)
{
ListenerTrace.Info("Enqueing from " + startCumul.Height + " " + blockCount + " blocks");
if (blockCount == 0)
{
return;
}
var tasks = _IndexTasks
.Where(t => chain.FindFork(t.Value.Item1.BlockLocator).Height <= startCumul.Height + blockCount)
.Select(t => new BlockRange()
{
From = startCumul.Height,
Count = blockCount,
Target = t.Key
})
.Select(t => _Conf.Topics.InitialIndexing.AddAsync(t))
.ToArray();
try
{
Task.WaitAll(tasks);
}
catch (AggregateException aex)
{
ExceptionDispatchInfo.Capture(aex.InnerException).Throw();
throw;
}
}
public void Cancel()
{
var blobLock = GetInitBlob();
try
{
try
{
blobLock.Delete();
}
catch
{
blobLock.BreakLease();
blobLock.Delete();
}
}
catch (StorageException ex)
{
if (ex.RequestInformation == null || ex.RequestInformation.HttpStatusCode != 404)
{
throw;
}
}
ListenerTrace.Info("Init blob deleted");
ListenerTrace.Info("Deleting queue...");
_Conf.Topics
.InitialIndexing
.GetNamespace().DeleteQueue(_Conf.Topics.InitialIndexing.Queue);
ListenerTrace.Info("Queue deleted");
}
void Ping(object state)
{
ListenerTrace.Verbose("Ping");
_Node.SendMessage(new PingPayload());
ListenerTrace.Verbose("Ping");
_SenderNode.SendMessage(new PingPayload());
}
private void UpdateCheckpoints(BlockLocator locator)
{
ListenerTrace.Info("Work finished, updating checkpoints");
foreach (var chk in _IndexTasks.Select(kv => kv.Value.Item1))
{
ListenerTrace.Info(chk.CheckpointName + "...");
chk.SaveProgress(locator);
}
ListenerTrace.Info("Checkpoints updated");
}
private void DeleteExpiredBroadcasted(CloudTableEvent evt, uint256 hash)
{
if (evt.AddedEntity != null)
{
if (DateTimeOffset.UtcNow - evt.AddedEntity.Timestamp > TimeSpan.FromHours(24.0))
{
ListenerTrace.Verbose("Cleaning expired broadcasted " + hash);
DeleteBroadcasted(evt.AddedEntity, hash);
}
}
}
void RunTask(string name, Action act, bool commonThread)
{
new Task(() =>
{
try
{
act();
}
catch (Exception ex)
{
ListenerTrace.Error("Error during task : " + name, ex);
LastException = ex;
}
}).Start(commonThread ? _Scheduler : TaskScheduler.Default);
}
private void DeleteBroadcasted(DynamicTableEntity entity, uint256 hash)
{
try
{
BroadcastedTransaction unused;
_BroadcastedTransactions.TryRemove(hash, out unused);
entity.ETag = "*";
Configuration.GetBroadcastedTransactionsListenable().CloudTable.Execute(TableOperation.Delete(entity));
}
catch (Exception ex)
{
StorageException storageEx = ex as StorageException;
if (storageEx == null || storageEx.RequestInformation == null || storageEx.RequestInformation.HttpStatusCode != 404)
{
ListenerTrace.Error("Error while cleaning up broadcasted transaction " + hash, ex);
}
}
}
void _SenderNode_MessageReceived(Node node, IncomingMessage message)
{
if (message.Message.Payload is GetDataPayload)
{
var getData = (GetDataPayload)message.Message.Payload;
foreach (var data in getData.Inventory)
{
if (data.Type == InventoryType.MSG_TX && _BroadcastedTransactions.ContainsKey(data.Hash))
{
var result = _BroadcastedTransactions[data.Hash];
var tx = new TxPayload(result.Transaction);
node.SendMessage(tx);
ListenerTrace.Info("Broadcasted " + data.Hash);
try
{
Configuration.GetBroadcastedTransactionsListenable().CloudTable.Execute(TableOperation.Delete(result.ToEntity()));
}
catch (StorageException)
{
}
}
}
}
if (message.Message.Payload is RejectPayload)
{
var reject = (RejectPayload)message.Message.Payload;
uint256 txId = reject.Hash;
if (txId != null && _BroadcastedTransactions.ContainsKey(txId))
{
ListenerTrace.Info("Broadcasted transaction rejected (" + reject.Code + ") " + txId);
DeleteBroadcasted(txId);
if (reject.Code != RejectCode.DUPLICATE)
{
UpdateBroadcastState(txId, reject.Code.ToString());
}
}
}
if (message.Message.Payload is PongPayload)
{
ListenerTrace.Verbose("Pong");
}
}
public int Run(ChainBase chain = null)
{
ListenerTrace.Info("Start initial indexing");
int totalProcessed = 0;
using (var node = _Conf.Indexer.ConnectToNode(false))
{
ListenerTrace.Info("Handshaking...");
node.VersionHandshake();
ListenerTrace.Info("Handshaked");
chain = chain ?? node.GetChain();
ListenerTrace.Info("Current chain at height " + chain.Height);
var blockRepository = new NodeBlocksRepository(node);
var blobLock = GetInitBlob();
string lease = null;
try
{
blobLock.UploadText("Enqueuing");
lease = blobLock.AcquireLease(null, null);
}
catch (StorageException)
{
}
if (lease != null)
{
ListenerTrace.Info("Queueing index jobs");
EnqueueJobs(blockRepository, chain, blobLock, lease);
}
ListenerTrace.Info("Dequeuing index jobs");
while (true)
{
var msg = _Conf.Topics
.InitialIndexing
.ReceiveAsync(TimeSpan.FromMilliseconds(1000))
.Result;
var ns = _Conf.Topics.InitialIndexing.GetNamespace();
var description = ns.GetQueue(_Conf.Topics.InitialIndexing.Queue);
Console.WriteLine("Work remaining in the queue : " + description.MessageCountDetails.ActiveMessageCount);
if (msg == null)
{
var state = blobLock.DownloadText();
if (state == "Enqueuing" || description.MessageCountDetails.ActiveMessageCount != 0)
{
ListenerTrace.Info("Additional work will be enqueued...");
continue;
}
else
{
var locator = new BlockLocator();
locator.FromBytes(Encoders.Hex.DecodeData(state));
UpdateCheckpoints(locator);
break;
}
}
using (msg.Message)
{
var range = msg.Body;
using (var sched = new CustomThreadPoolTaskScheduler(50, 100, range.ToString()))
{
ListenerTrace.Info("Processing " + range.ToString());
totalProcessed++;
var task = _IndexTasks[range.Target];
BlockFetcher fetcher = new BlockFetcher(task.Item1, blockRepository, chain)
{
FromHeight = range.From,
ToHeight = range.From + range.Count - 1
};
try
{
task.Item2.SaveProgression = false;
task.Item2.EnsureIsSetup = totalProcessed == 0;
var index = Task.Factory.StartNew(() =>
{
task.Item2.Index(fetcher, sched);
}, TaskCreationOptions.LongRunning);
while (!index.Wait(TimeSpan.FromMinutes(4)))
{
msg.Message.RenewLock();
ListenerTrace.Info("Lock renewed");
}
}
catch (AggregateException aex)
{
ExceptionDispatchInfo.Capture(aex.InnerException).Throw();
throw;
}
range.Processed = true;
msg.Message.Complete();
}
}
}
}
ListenerTrace.Info("Initial indexing terminated");
return(totalProcessed);
}
public void Listen()
{
_Evt.Reset();
_Scheduler = new SingleThreadTaskScheduler();
ListenerTrace.Info("Connecting to node " + Configuration.Indexer.Node + "...");
_Node = _Configuration.Indexer.ConnectToNode(true);
ListenerTrace.Info("Connected");
ListenerTrace.Info("Handshaking...");
_Node.VersionHandshake();
ListenerTrace.Info("Handshaked");
_Chain = new ConcurrentChain(_Configuration.Indexer.Network);
ListenerTrace.Info("Fetching headers...");
_Node.SynchronizeChain(_Chain);
ListenerTrace.Info("Headers fetched tip " + _Chain.Tip.Height);
_Indexer = Configuration.Indexer.CreateIndexer();
ListenerTrace.Info("Indexing indexer chain...");
_Indexer.IndexChain(_Chain);
_Node.MessageReceived += node_MessageReceived;
_Wallets = _Configuration.Indexer.CreateIndexerClient().GetAllWalletRules();
ListenerTrace.Info("Connecting and handshaking for the sender node...");
_SenderNode = _Configuration.Indexer.ConnectToNode(false);
_SenderNode.VersionHandshake();
_SenderNode.MessageReceived += _SenderNode_MessageReceived;
ListenerTrace.Info("Sender node handshaked");
ListenerTrace.Info("Fetching transactions to broadcast...");
_Disposables.Add(
Configuration
.GetBroadcastedTransactionsListenable()
.CreateConsumer()
.EnsureExists()
.OnMessage(evt =>
{
uint256 hash = null;
try
{
if (evt.Addition)
{
var tx = new BroadcastedTransaction(evt.AddedEntity);
hash = tx.Transaction.GetHash();
var value = _BroadcastedTransactions.GetOrAdd(hash, tx);
ListenerTrace.Info("Broadcasting " + hash);
if (value == tx) //Was not present before
{
_SenderNode.SendMessage(new InvPayload(tx.Transaction));
}
}
}
catch (Exception ex)
{
LastException = ex;
ListenerTrace.Error("Error for new broadcasted transaction " + hash, ex);
}
finally
{
DeleteExpiredBroadcasted(evt, hash);
}
}));
ListenerTrace.Info("Transactions to broadcast fetched");
ListenerTrace.Info("Fetching wallet rules...");
_Disposables.Add(Configuration
.GetWalletRuleListenable()
.CreateConsumer()
.EnsureExists()
.OnMessage(evt =>
{
ListenerTrace.Info("New wallet rule");
RunTask("New wallet rule", () =>
{
_Wallets.Add(new WalletRuleEntry(evt.AddedEntity, Configuration.Indexer.CreateIndexerClient()));
}, true);
}));
ListenerTrace.Info("Wallet rules fetched");
var ping = new Timer(Ping, null, 0, 1000 * 60);
_Disposables.Add(ping);
}
void node_MessageReceived(Node node, IncomingMessage message)
{
if (message.Message.Payload is InvPayload)
{
var inv = (InvPayload)message.Message.Payload;
foreach (var inventory in inv.Inventory.Where(i => _BroadcastedTransactions.ContainsKey(i.Hash)))
{
ListenerTrace.Info("Broadcasted reached mempool " + inventory);
}
node.SendMessage(new GetDataPayload(inv.Inventory.ToArray()));
}
if (message.Message.Payload is TxPayload)
{
var tx = ((TxPayload)message.Message.Payload).Object;
ListenerTrace.Verbose("Received Transaction " + tx.GetHash());
RunTask("New transaction", () =>
{
var txId = tx.GetHash();
_Indexer.Index(new TransactionEntry.Entity(txId, tx, null));
_Indexer.IndexOrderedBalance(tx);
RunTask("New transaction", () =>
{
var balances =
OrderedBalanceChange
.ExtractWalletBalances(txId, tx, null, null, int.MaxValue, _Wallets)
.GroupBy(b => b.PartitionKey);
foreach (var b in balances)
{
_Indexer.Index(b);
}
}, true);
}, false);
}
if (message.Message.Payload is BlockPayload)
{
var block = ((BlockPayload)message.Message.Payload).Object;
ListenerTrace.Info("Received block " + block.GetHash());
RunTask("New block", () =>
{
var blockId = block.GetHash();
node.SynchronizeChain(_Chain);
_Indexer.IndexChain(_Chain);
ListenerTrace.Info("New height : " + _Chain.Height);
var header = _Chain.GetBlock(blockId);
if (header == null)
{
return;
}
_Indexer.IndexWalletOrderedBalance(header.Height, block, _Wallets);
RunTask("New block", () =>
{
_Indexer.Index(block);
}, false);
RunTask("New block", () =>
{
_Indexer.IndexTransactions(header.Height, block);
}, false);
RunTask("New block", () =>
{
_Indexer.IndexOrderedBalance(header.Height, block);
}, false);
}, true);
}
if (message.Message.Payload is PongPayload)
{
ListenerTrace.Verbose("Pong");
}
}