void ExceptionOnMessagePump(Exception ex)
{
if (!_Disposed)
{
ListenerTrace.Error("Error on azure message pumped", ex);
LastException = ex;
}
}
void HandleException(Task t)
{
if (t.IsFaulted)
{
if (!_Disposed)
{
ListenerTrace.Error("Error during asynchronous task", t.Exception);
LastException = t.Exception;
}
}
}
void AttachedNode_StateChanged(Node node, NodeState oldState)
{
ListenerTrace.Info("State change " + node.State);
if (node.State == NodeState.HandShaked)
{
ListenerTrace.Info("Node handshaked");
AttachedNode.MessageReceived += _Listener.node_MessageReceived;
AttachedNode.Disconnected += AttachedNode_Disconnected;
ListenerTrace.Info("Connection count : " + NodesGroup.GetNodeGroup(node).ConnectedNodes.Count);
}
}
public IDisposable OnMessageAsync(Func <T, MessageControl, Task> evt, OnMessageOptions options = null)
{
if (options == null)
{
options = new OnMessageOptions()
{
AutoComplete = true,
MaxConcurrentCalls = 10,
AutoRenewTimeout = TimeSpan.Zero
}
}
;
return(OnMessageAsyncCore(async bm =>
{
var control = new MessageControl();
control.Options = options;
var obj = ToObject(bm);
if (obj == null)
{
return;
}
await evt(obj, control).ConfigureAwait(false);
if (control._Scheduled != null)
{
BrokeredMessage message = new BrokeredMessage(Serializer.ToString(obj));
message.MessageId = Encoders.Hex.EncodeData(RandomUtils.GetBytes(32));
message.ScheduledEnqueueTimeUtc = control._Scheduled.Value;
await SendAsync(message).ConfigureAwait(false);
if (!options.AutoComplete)
{
if (control._Complete)
{
try
{
await bm.CompleteAsync().ConfigureAwait(false);
}
catch (ObjectDisposedException)
{
ListenerTrace.Error("Brokered message already disposed", null);
}
}
}
}
}, options));
}
private async Task SendMessageAsync(Transaction tx)
{
var payload = new InvPayload(tx);
int[] delays = new int[] { 50, 100, 200, 300, 1000, 2000, 3000, 6000, 12000 };
int i = 0;
var rpc = Configuration.TryCreateRPCClient();
var txId = tx.GetHash();
if (rpc != null)
{
try
{
ListenerTrace.Info("Sending " + txId + " via RPC");
await rpc.SendRawTransactionAsync(tx);
ListenerTrace.Info("Successfully broadcasted");
Transaction unused;
_Broadcasting.TryRemove(txId, out unused);
}
catch (NBitcoin.RPC.RPCException ex)
{
ListenerTrace.Info("Broadcasted transaction rejected (" + ex.Message + ") " + txId);
var reject = new RejectPayload()
{
Message = "tx",
Reason = ex.Message,
Code = RejectCode.INVALID,
Hash = txId
};
Transaction unused;
_Broadcasting.TryRemove(txId, out unused);
}
return;
}
while (_Group.ConnectedNodes.Count != 2)
{
i++;
i = Math.Min(i, delays.Length - 1);
await Task.Delay(delays[i]).ConfigureAwait(false);
}
await _Group.ConnectedNodes.First().SendMessageAsync(payload).ConfigureAwait(false);
}
Task Async(Action act)
{
var t = new Task(() =>
{
try
{
act();
}
catch (Exception ex)
{
if (!_Disposed)
{
ListenerTrace.Error("Error during task.", ex);
LastException = ex;
}
}
});
t.Start(TaskScheduler.Default);
return(t);
}
private void UpdateHDState(OrderedBalanceChange entry)
{
var repo = Configuration.CreateWalletRepository();
IDisposable walletLock = null;
try
{
foreach (var matchedAddress in entry.MatchedRules.Select(m => WalletAddress.TryParse(m.Rule.CustomData)).Where(m => m != null))
{
if (matchedAddress.HDKeySet == null)
{
return;
}
var keySet = repo.GetKeySetData(matchedAddress.WalletName, matchedAddress.HDKeySet.Name);
if (keySet == null)
{
return;
}
var keyIndex = (int)matchedAddress.HDKey.Path.Indexes.Last();
if (keyIndex < keySet.State.NextUnused)
{
return;
}
walletLock = walletLock ?? _WalletsSlimLock.LockWrite();
foreach (var address in repo.Scan(matchedAddress.WalletName, keySet, keyIndex + 1, 20))
{
ListenerTrace.Info("New wallet rule");
var walletEntry = address.CreateWalletRuleEntry();
_Wallets.Add(walletEntry);
}
}
}
finally
{
if (walletLock != null)
{
walletLock.Dispose();
}
}
}
void AttachedNode_Disconnected(Node node)
{
ListenerTrace.Info("Node Connection dropped : " + ToString(node.DisconnectReason));
}
void node_MessageReceived(Node node, IncomingMessage message)
{
if (_KnownInvs.Count == 1000)
{
_KnownInvs.Clear();
}
if (message.Message.Payload is InvPayload)
{
var inv = (InvPayload)message.Message.Payload;
foreach (var inventory in inv.Inventory)
{
Transaction tx;
if (_Broadcasting.TryRemove(inventory.Hash, out tx))
{
ListenerTrace.Info("Broadcasted reached mempool " + inventory);
}
}
var getdata = new GetDataPayload(inv.Inventory.Where(i => i.Type == InventoryType.MSG_TX && _KnownInvs.TryAdd(i.Hash, i.Hash)).ToArray());
foreach (var data in getdata.Inventory)
{
data.Type = node.AddSupportedOptions(InventoryType.MSG_TX);
}
if (getdata.Inventory.Count > 0)
{
node.SendMessageAsync(getdata);
}
}
if (message.Message.Payload is TxPayload)
{
var tx = ((TxPayload)message.Message.Payload).Object;
ListenerTrace.Verbose("Received Transaction " + tx.GetHash());
_Indexer.IndexAsync(new TransactionEntry.Entity(tx.GetHash(), tx, null))
.ContinueWith(HandleException);
_Indexer.IndexOrderedBalanceAsync(tx)
.ContinueWith(HandleException);
Async(() =>
{
var txId = tx.GetHash();
List <OrderedBalanceChange> balances;
using (_WalletsSlimLock.LockRead())
{
balances =
OrderedBalanceChange
.ExtractWalletBalances(txId, tx, null, null, int.MaxValue, _Wallets)
.AsEnumerable()
.ToList();
}
UpdateHDState(balances);
_Indexer.IndexAsync(balances)
.ContinueWith(HandleException);
Task notify = null;
using (_SubscriptionSlimLock.LockRead())
{
var topic = Configuration.Topics.SendNotifications;
notify = Task.WhenAll(_Subscriptions
.GetNewTransactions()
.Select(t => topic.AddAsync(new Notify()
{
SendAndForget = true,
Notification = new Notification()
{
Subscription = t,
Data = new NewTransactionNotificationData()
{
TransactionId = txId
}
}
})).ToArray());
}
notify.Wait();
});
var unused = Configuration.Topics.NewTransactions.AddAsync(tx)
.ContinueWith(HandleException);
}
if (message.Message.Payload is BlockPayload)
{
var block = ((BlockPayload)message.Message.Payload).Object;
var blockId = block.GetHash();
List <OrderedBalanceChange> balances = null;
using (_WalletsSlimLock.LockRead())
{
balances = block.Transactions.SelectMany(tx => OrderedBalanceChange.ExtractWalletBalances(null, tx, null, null, 0, _Wallets)).ToList();
}
UpdateHDState(balances);
}
if (message.Message.Payload is HeadersPayload)
{
if (_Chain.Tip.HashBlock != _LastChainTip)
{
var header = _Chain.Tip.Header;
_LastChainTip = _Chain.Tip.HashBlock;
Configuration.Indexer.CreateIndexer().IndexChain(_Chain);
Async(() =>
{
CancellationTokenSource cancel = new CancellationTokenSource(TimeSpan.FromMinutes(30));
var repo = new CacheBlocksRepository(new NodeBlocksRepository(node));
TryLock(_LockBlocks, () =>
{
new IndexBlocksTask(Configuration.Indexer)
{
EnsureIsSetup = false,
}.Index(new BlockFetcher(_Indexer.GetCheckpoint(IndexerCheckpoints.Blocks), repo, _Chain)
{
CancellationToken = cancel.Token
}, _Indexer.TaskScheduler);
});
TryLock(_LockTransactions, () =>
{
new IndexTransactionsTask(Configuration.Indexer)
{
EnsureIsSetup = false
}
.Index(new BlockFetcher(_Indexer.GetCheckpoint(IndexerCheckpoints.Transactions), repo, _Chain)
{
CancellationToken = cancel.Token
}, _Indexer.TaskScheduler);
});
TryLock(_LockWallets, () =>
{
using (_WalletsSlimLock.LockRead())
{
new IndexBalanceTask(Configuration.Indexer, _Wallets)
{
EnsureIsSetup = false
}
.Index(new BlockFetcher(_Indexer.GetCheckpoint(IndexerCheckpoints.Wallets), repo, _Chain)
{
CancellationToken = cancel.Token
}, _Indexer.TaskScheduler);
}
});
TryLock(_LockBalance, () =>
{
new IndexBalanceTask(Configuration.Indexer, null)
{
EnsureIsSetup = false
}.Index(new BlockFetcher(_Indexer.GetCheckpoint(IndexerCheckpoints.Balances), repo, _Chain)
{
CancellationToken = cancel.Token
}, _Indexer.TaskScheduler);
});
TryLock(_LockSubscriptions, () =>
{
using (_SubscriptionSlimLock.LockRead())
{
new IndexNotificationsTask(Configuration, _Subscriptions)
{
EnsureIsSetup = false,
}
.Index(new BlockFetcher(_Indexer.GetCheckpointRepository().GetCheckpoint("subscriptions"), repo, _Chain)
{
CancellationToken = cancel.Token
}, _Indexer.TaskScheduler);
}
});
cancel.Dispose();
var unused = _Configuration.Topics.NewBlocks.AddAsync(header);
});
}
}
if (message.Message.Payload is GetDataPayload)
{
var getData = (GetDataPayload)message.Message.Payload;
foreach (var data in getData.Inventory)
{
Transaction tx = null;
if (data.Type == InventoryType.MSG_TX && _Broadcasting.TryGetValue(data.Hash, out tx))
{
var payload = new TxPayload(tx);
node.SendMessageAsync(payload);
ListenerTrace.Info("Broadcasted " + data.Hash);
}
}
}
if (message.Message.Payload is RejectPayload)
{
var reject = (RejectPayload)message.Message.Payload;
uint256 txId = reject.Hash;
if (txId != null)
{
ListenerTrace.Info("Broadcasted transaction rejected (" + reject.Code + ") " + txId);
if (reject.Code != RejectCode.DUPLICATE)
{
Configuration.GetRejectTable().Create(txId.ToString(), reject);
}
Transaction tx;
_Broadcasting.TryRemove(txId, out tx);
}
}
}
public void Listen(ConcurrentChain chain = null)
{
ListenerTrace.Info($"Connecting to node {_Configuration.Indexer.Node}");
var ip = Utils.ParseIpEndpoint(_Configuration.Indexer.Node, Configuration.Indexer.Network.DefaultPort);
ListenerTrace.Info($"Connecting to node ip {ip.ToString()}");
var node = Node.Connect(Configuration.Indexer.Network, ip);
ListenerTrace.Info($"Connected, trying handshake...");
node.VersionHandshake();
ListenerTrace.Info($"Hanshaked");
node.Disconnect();
_Chain = new ConcurrentChain(_Configuration.Indexer.Network);
_Indexer = Configuration.Indexer.CreateIndexer();
if (chain == null)
{
chain = new ConcurrentChain(_Configuration.Indexer.Network);
}
_Chain = chain;
ListenerTrace.Info("Fetching headers from " + _Chain.Tip.Height + " (from azure)");
var client = Configuration.Indexer.CreateIndexerClient();
client.SynchronizeChain(chain);
ListenerTrace.Info("Headers fetched tip " + _Chain.Tip.Height);
_Disposables.Add(_IndexerScheduler = new CustomThreadPoolTaskScheduler(50, 100, "Indexing Threads"));
_Indexer.TaskScheduler = _IndexerScheduler;
_Group = new NodesGroup(Configuration.Indexer.Network);
_Disposables.Add(_Group);
_Group.AllowSameGroup = true;
_Group.MaximumNodeConnection = 2;
AddressManager addrman = new AddressManager();
addrman.Add(new NetworkAddress(ip), IPAddress.Parse("127.0.0.1"));
_Group.NodeConnectionParameters.TemplateBehaviors.Add(new AddressManagerBehavior(addrman)
{
Mode = AddressManagerBehaviorMode.None
});
_Group.NodeConnectionParameters.TemplateBehaviors.Add(new ChainBehavior(_Chain)
{
SkipPoWCheck = true
});
_Group.NodeConnectionParameters.TemplateBehaviors.Add(new Behavior(this));
ListenerTrace.Info("Fetching wallet rules...");
_Wallets = _Configuration.Indexer.CreateIndexerClient().GetAllWalletRules();
ListenerTrace.Info("Wallet rules fetched");
ListenerTrace.Info("Fetching wallet subscriptions...");
_Subscriptions = new SubscriptionCollection(_Configuration.GetSubscriptionsTable().Read());
ListenerTrace.Info("Subscriptions fetched");
_Group.Connect();
ListenerTrace.Info("Fetching transactions to broadcast...");
_Disposables.Add(
Configuration
.Topics
.BroadcastedTransactions
.CreateConsumer("listener", true)
.EnsureSubscriptionExists()
.OnMessage((tx, ctl) =>
{
uint256 hash = null;
var repo = Configuration.Indexer.CreateIndexerClient();
var rejects = Configuration.GetRejectTable();
try
{
hash = tx.Transaction.GetHash();
var indexedTx = repo.GetTransaction(hash);
ListenerTrace.Info("Broadcasting " + hash);
var reject = rejects.ReadOne(hash.ToString());
if (reject != null)
{
ListenerTrace.Info("Abort broadcasting of rejected");
return;
}
if (_Broadcasting.Count > 1000)
{
_Broadcasting.Clear();
}
_Broadcasting.TryAdd(hash, tx.Transaction);
if (indexedTx == null || !indexedTx.BlockIds.Any(id => Chain.Contains(id)))
{
var unused = SendMessageAsync(tx.Transaction);
}
var reschedule = new[]
{
TimeSpan.FromMinutes(5),
TimeSpan.FromMinutes(10),
TimeSpan.FromHours(1),
TimeSpan.FromHours(6),
TimeSpan.FromHours(24),
};
if (tx.Tried <= reschedule.Length - 1)
{
ctl.RescheduleIn(reschedule[tx.Tried]);
tx.Tried++;
}
}
catch (Exception ex)
{
if (!_Disposed)
{
LastException = ex;
ListenerTrace.Error("Error for new broadcasted transaction " + hash, ex);
throw;
}
}
}));
ListenerTrace.Info("Transactions to broadcast fetched");
_Disposables.Add(_Configuration
.Topics
.SubscriptionChanges
.EnsureSubscriptionExists()
.AddUnhandledExceptionHandler(ExceptionOnMessagePump)
.OnMessage(c =>
{
using (_SubscriptionSlimLock.LockWrite())
{
if (c.Added)
{
_Subscriptions.Add(c.Subscription);
}
else
{
_Subscriptions.Remove(c.Subscription.Id);
}
}
}));
_Disposables.Add(_Configuration
.Topics
.SendNotifications
.AddUnhandledExceptionHandler(ExceptionOnMessagePump)
.OnMessageAsync((n, act) =>
{
return(SendAsync(n, act).ContinueWith(t =>
{
if (!_Disposed)
{
if (t.Exception != null)
{
LastException = t.Exception;
}
}
}));
}, new OnMessageOptions()
{
MaxConcurrentCalls = 1000,
AutoComplete = true,
AutoRenewTimeout = TimeSpan.Zero
}));
_Disposables.Add(Configuration
.Topics
.AddedAddresses
.CreateConsumer("updater", true)
.EnsureSubscriptionExists()
.AddUnhandledExceptionHandler(ExceptionOnMessagePump)
.OnMessage(evt =>
{
if (evt == null)
{
return;
}
ListenerTrace.Info("New wallet rule");
using (_WalletsSlimLock.LockWrite())
{
foreach (var address in evt)
{
_Wallets.Add(address.CreateWalletRuleEntry());
}
}
}));
}
static void Main(string[] args)
{
var options = new ListenerOptions();
if (args.Length == 0)
{
System.Console.WriteLine(options.GetUsage());
}
if (Parser.Default.ParseArguments(args, options))
{
if (options.Configuration != null)
{
if (!File.Exists(options.Configuration))
{
System.Console.WriteLine("File " + new FileInfo(options.Configuration).FullName + " not found");
return;
}
AppDomain.CurrentDomain.SetData("APP_CONFIG_FILE", new FileInfo(options.Configuration).FullName);
}
var iconf = new ConfigurationManagerConfiguration();
var conf = QBitNinjaConfiguration.FromConfiguration(iconf);
conf.EnsureSetup();
if (options.CancelInit)
{
var indexer = new InitialIndexer(conf);
indexer.Cancel();
}
if (options.Init)
{
var indexer = new InitialIndexer(conf);
while (true)
{
try
{
indexer.Run();
break;
}
catch (Exception ex)
{
ListenerTrace.Error("Error while Running the initial sync indexer, retrying...", ex);
System.Threading.Thread.Sleep(5000);
}
}
}
List <IDisposable> dispo = new List <IDisposable>();
List <Task> running = new List <Task>();
try
{
if (options.Listen)
{
QBitNinjaNodeListener listener = new QBitNinjaNodeListener(conf);
dispo.Add(listener);
listener.Listen();
running.Add(listener.Running);
}
if (options.Web)
{
System.Console.WriteLine("Trying to listen on http://*:" + options.Port + "/");
var server = WebApp.Start("http://*:" + options.Port, appBuilder =>
{
var config = new HttpConfiguration();
var qbit = QBitNinjaConfiguration.FromConfiguration(iconf);
qbit.EnsureSetup();
WebApiConfig.Register(config, qbit);
UpdateChainListener listener = new UpdateChainListener();
dispo.Add(listener);
listener.Listen(config);
appBuilder.UseWebApi(config);
running.Add(new TaskCompletionSource <int>().Task);
});
dispo.Add(server);
System.Console.WriteLine("Server started");
Process.Start("http://localhost:" + options.Port + "/blocks/tip");
}
if (running.Count != 0)
{
try
{
running.Add(WaitInput());
Task.WaitAny(running.ToArray());
}
catch (AggregateException aex)
{
ExceptionDispatchInfo.Capture(aex.InnerException).Throw();
throw;
}
}
}
finally
{
foreach (var d in dispo)
{
d.Dispose();
}
}
}
}
