public virtual void Start() { if (_isRunning) { throw new InvalidOperationException("Unable to start, the bus is already running"); } Starting(); _completionResultTaskScheduler = new CustomThreadPoolTaskScheduler(4); _logger.DebugFormat("Loading invokers..."); _messageDispatcher.LoadMessageHandlerInvokers(); PerformAutoSubscribe(); _logger.DebugFormat("Starting message dispatcher..."); _messageDispatcher.Start(); _logger.DebugFormat("Starting transport..."); _transport.Start(); _isRunning = true; _logger.DebugFormat("Registering on directory..."); var self = new Peer(PeerId, EndPoint); _directory.Register(this, self, GetSubscriptions()); _transport.OnRegistered(); Started(); }
public DBreezeSingleThreadSession(string threadName, string folder) { Guard.NotEmpty(threadName, nameof(threadName)); Guard.NotEmpty(folder, nameof(folder)); _SingleThread = new CustomThreadPoolTaskScheduler(1, 100, threadName); new Task(() => { DBreeze.Utils.CustomSerializator.ByteArraySerializator = NBitcoinSerialize; DBreeze.Utils.CustomSerializator.ByteArrayDeSerializator = NBitcoinDeserialize; _Engine = new DBreezeEngine(folder); _Transaction = _Engine.GetTransaction(); }).Start(_SingleThread); }
public int Run(ChainBase chain = null) { ListenerTrace.Info("Start initial indexing"); int totalProcessed = 0; using (var node = _Conf.Indexer.ConnectToNode(false)) { ListenerTrace.Info("Handshaking..."); node.VersionHandshake(); ListenerTrace.Info("Handshaked"); chain = chain ?? node.GetChain(); ListenerTrace.Info("Current chain at height " + chain.Height); var blockRepository = new NodeBlocksRepository(node); var blobLock = GetInitBlob(); string lease = null; try { blobLock.UploadText("Enqueuing"); lease = blobLock.AcquireLease(null, null); } catch (StorageException) { } if (lease != null) { ListenerTrace.Info("Queueing index jobs"); EnqueueJobs(blockRepository, chain, blobLock, lease); } ListenerTrace.Info("Dequeuing index jobs"); while (true) { var msg = _Conf.Topics .InitialIndexing .ReceiveAsync(TimeSpan.FromMilliseconds(1000)) .Result; var ns = _Conf.Topics.InitialIndexing.GetNamespace(); var description = ns.GetQueue(_Conf.Topics.InitialIndexing.Queue); Console.WriteLine("Work remaining in the queue : " + description.MessageCountDetails.ActiveMessageCount); if (msg == null) { var state = blobLock.DownloadText(); if (state == "Enqueuing" || description.MessageCountDetails.ActiveMessageCount != 0) { ListenerTrace.Info("Additional work will be enqueued..."); continue; } else { var locator = new BlockLocator(); locator.FromBytes(Encoders.Hex.DecodeData(state)); UpdateCheckpoints(locator); break; } } using (msg.Message) { var range = msg.Body; using (var sched = new CustomThreadPoolTaskScheduler(50, 100, range.ToString())) { ListenerTrace.Info("Processing " + range.ToString()); totalProcessed++; var task = _IndexTasks[range.Target]; BlockFetcher fetcher = new BlockFetcher(task.Item1, blockRepository, chain) { FromHeight = range.From, ToHeight = range.From + range.Count - 1 }; try { task.Item2.SaveProgression = false; task.Item2.EnsureIsSetup = totalProcessed == 0; var index = Task.Factory.StartNew(() => { task.Item2.Index(fetcher, sched); }, TaskCreationOptions.LongRunning); while (!index.Wait(TimeSpan.FromMinutes(4))) { msg.Message.RenewLock(); ListenerTrace.Info("Lock renewed"); } } catch (AggregateException aex) { ExceptionDispatchInfo.Capture(aex.InnerException).Throw(); throw; } range.Processed = true; msg.Message.Complete(); } } } } ListenerTrace.Info("Initial indexing terminated"); return(totalProcessed); }
public void Listen(ConcurrentChain chain = null) { ListenerTrace.Info($"Connecting to node {_Configuration.Indexer.Node}"); var ip = Utils.ParseIpEndpoint(_Configuration.Indexer.Node, Configuration.Indexer.Network.DefaultPort); ListenerTrace.Info($"Connecting to node ip {ip.ToString()}"); var node = Node.Connect(Configuration.Indexer.Network, ip); ListenerTrace.Info($"Connected, trying handshake..."); node.VersionHandshake(); ListenerTrace.Info($"Hanshaked"); node.Disconnect(); _Chain = new ConcurrentChain(_Configuration.Indexer.Network); _Indexer = Configuration.Indexer.CreateIndexer(); if (chain == null) { chain = new ConcurrentChain(_Configuration.Indexer.Network); } _Chain = chain; ListenerTrace.Info("Fetching headers from " + _Chain.Tip.Height + " (from azure)"); var client = Configuration.Indexer.CreateIndexerClient(); client.SynchronizeChain(chain); ListenerTrace.Info("Headers fetched tip " + _Chain.Tip.Height); _Disposables.Add(_IndexerScheduler = new CustomThreadPoolTaskScheduler(50, 100, "Indexing Threads")); _Indexer.TaskScheduler = _IndexerScheduler; _Group = new NodesGroup(Configuration.Indexer.Network); _Disposables.Add(_Group); _Group.AllowSameGroup = true; _Group.MaximumNodeConnection = 2; AddressManager addrman = new AddressManager(); addrman.Add(new NetworkAddress(ip), IPAddress.Parse("127.0.0.1")); _Group.NodeConnectionParameters.TemplateBehaviors.Add(new AddressManagerBehavior(addrman) { Mode = AddressManagerBehaviorMode.None }); _Group.NodeConnectionParameters.TemplateBehaviors.Add(new ChainBehavior(_Chain) { SkipPoWCheck = true }); _Group.NodeConnectionParameters.TemplateBehaviors.Add(new Behavior(this)); ListenerTrace.Info("Fetching wallet rules..."); _Wallets = _Configuration.Indexer.CreateIndexerClient().GetAllWalletRules(); ListenerTrace.Info("Wallet rules fetched"); ListenerTrace.Info("Fetching wallet subscriptions..."); _Subscriptions = new SubscriptionCollection(_Configuration.GetSubscriptionsTable().Read()); ListenerTrace.Info("Subscriptions fetched"); _Group.Connect(); ListenerTrace.Info("Fetching transactions to broadcast..."); _Disposables.Add( Configuration .Topics .BroadcastedTransactions .CreateConsumer("listener", true) .EnsureSubscriptionExists() .OnMessage((tx, ctl) => { uint256 hash = null; var repo = Configuration.Indexer.CreateIndexerClient(); var rejects = Configuration.GetRejectTable(); try { hash = tx.Transaction.GetHash(); var indexedTx = repo.GetTransaction(hash); ListenerTrace.Info("Broadcasting " + hash); var reject = rejects.ReadOne(hash.ToString()); if (reject != null) { ListenerTrace.Info("Abort broadcasting of rejected"); return; } if (_Broadcasting.Count > 1000) { _Broadcasting.Clear(); } _Broadcasting.TryAdd(hash, tx.Transaction); if (indexedTx == null || !indexedTx.BlockIds.Any(id => Chain.Contains(id))) { var unused = SendMessageAsync(tx.Transaction); } var reschedule = new[] { TimeSpan.FromMinutes(5), TimeSpan.FromMinutes(10), TimeSpan.FromHours(1), TimeSpan.FromHours(6), TimeSpan.FromHours(24), }; if (tx.Tried <= reschedule.Length - 1) { ctl.RescheduleIn(reschedule[tx.Tried]); tx.Tried++; } } catch (Exception ex) { if (!_Disposed) { LastException = ex; ListenerTrace.Error("Error for new broadcasted transaction " + hash, ex); throw; } } })); ListenerTrace.Info("Transactions to broadcast fetched"); _Disposables.Add(_Configuration .Topics .SubscriptionChanges .EnsureSubscriptionExists() .AddUnhandledExceptionHandler(ExceptionOnMessagePump) .OnMessage(c => { using (_SubscriptionSlimLock.LockWrite()) { if (c.Added) { _Subscriptions.Add(c.Subscription); } else { _Subscriptions.Remove(c.Subscription.Id); } } })); _Disposables.Add(_Configuration .Topics .SendNotifications .AddUnhandledExceptionHandler(ExceptionOnMessagePump) .OnMessageAsync((n, act) => { return(SendAsync(n, act).ContinueWith(t => { if (!_Disposed) { if (t.Exception != null) { LastException = t.Exception; } } })); }, new OnMessageOptions() { MaxConcurrentCalls = 1000, AutoComplete = true, AutoRenewTimeout = TimeSpan.Zero })); _Disposables.Add(Configuration .Topics .AddedAddresses .CreateConsumer("updater", true) .EnsureSubscriptionExists() .AddUnhandledExceptionHandler(ExceptionOnMessagePump) .OnMessage(evt => { if (evt == null) { return; } ListenerTrace.Info("New wallet rule"); using (_WalletsSlimLock.LockWrite()) { foreach (var address in evt) { _Wallets.Add(address.CreateWalletRuleEntry()); } } })); }
public void CustomThreadPoolTaskWorks() { TaskCompletionSource<int> completion = new TaskCompletionSource<int>(); var scheduler = new CustomThreadPoolTaskScheduler(10, 20); for(int i = 0; i < 30; i++) { new Task(() => Task.WaitAll(completion.Task)).Start(scheduler); } Assert.Equal(0, scheduler.AvailableThreads); Assert.Equal(20, scheduler.QueuedCount); Assert.Equal(10, scheduler.ThreadsCount); completion.SetResult(1); scheduler.WaitFinished(); Assert.Equal(10, scheduler.AvailableThreads); Assert.Equal(0, scheduler.QueuedCount); Assert.Equal(10, scheduler.ThreadsCount); }