public virtual void Start()
{
if (_isRunning)
{
throw new InvalidOperationException("Unable to start, the bus is already running");
}
Starting();
_completionResultTaskScheduler = new CustomThreadPoolTaskScheduler(4);
_logger.DebugFormat("Loading invokers...");
_messageDispatcher.LoadMessageHandlerInvokers();
PerformAutoSubscribe();
_logger.DebugFormat("Starting message dispatcher...");
_messageDispatcher.Start();
_logger.DebugFormat("Starting transport...");
_transport.Start();
_isRunning = true;
_logger.DebugFormat("Registering on directory...");
var self = new Peer(PeerId, EndPoint);
_directory.Register(this, self, GetSubscriptions());
_transport.OnRegistered();
Started();
}
public DBreezeSingleThreadSession(string threadName, string folder)
{
Guard.NotEmpty(threadName, nameof(threadName));
Guard.NotEmpty(folder, nameof(folder));
_SingleThread = new CustomThreadPoolTaskScheduler(1, 100, threadName);
new Task(() =>
{
DBreeze.Utils.CustomSerializator.ByteArraySerializator = NBitcoinSerialize;
DBreeze.Utils.CustomSerializator.ByteArrayDeSerializator = NBitcoinDeserialize;
_Engine = new DBreezeEngine(folder);
_Transaction = _Engine.GetTransaction();
}).Start(_SingleThread);
}
public int Run(ChainBase chain = null)
{
ListenerTrace.Info("Start initial indexing");
int totalProcessed = 0;
using (var node = _Conf.Indexer.ConnectToNode(false))
{
ListenerTrace.Info("Handshaking...");
node.VersionHandshake();
ListenerTrace.Info("Handshaked");
chain = chain ?? node.GetChain();
ListenerTrace.Info("Current chain at height " + chain.Height);
var blockRepository = new NodeBlocksRepository(node);
var blobLock = GetInitBlob();
string lease = null;
try
{
blobLock.UploadText("Enqueuing");
lease = blobLock.AcquireLease(null, null);
}
catch (StorageException)
{
}
if (lease != null)
{
ListenerTrace.Info("Queueing index jobs");
EnqueueJobs(blockRepository, chain, blobLock, lease);
}
ListenerTrace.Info("Dequeuing index jobs");
while (true)
{
var msg = _Conf.Topics
.InitialIndexing
.ReceiveAsync(TimeSpan.FromMilliseconds(1000))
.Result;
var ns = _Conf.Topics.InitialIndexing.GetNamespace();
var description = ns.GetQueue(_Conf.Topics.InitialIndexing.Queue);
Console.WriteLine("Work remaining in the queue : " + description.MessageCountDetails.ActiveMessageCount);
if (msg == null)
{
var state = blobLock.DownloadText();
if (state == "Enqueuing" || description.MessageCountDetails.ActiveMessageCount != 0)
{
ListenerTrace.Info("Additional work will be enqueued...");
continue;
}
else
{
var locator = new BlockLocator();
locator.FromBytes(Encoders.Hex.DecodeData(state));
UpdateCheckpoints(locator);
break;
}
}
using (msg.Message)
{
var range = msg.Body;
using (var sched = new CustomThreadPoolTaskScheduler(50, 100, range.ToString()))
{
ListenerTrace.Info("Processing " + range.ToString());
totalProcessed++;
var task = _IndexTasks[range.Target];
BlockFetcher fetcher = new BlockFetcher(task.Item1, blockRepository, chain)
{
FromHeight = range.From,
ToHeight = range.From + range.Count - 1
};
try
{
task.Item2.SaveProgression = false;
task.Item2.EnsureIsSetup = totalProcessed == 0;
var index = Task.Factory.StartNew(() =>
{
task.Item2.Index(fetcher, sched);
}, TaskCreationOptions.LongRunning);
while (!index.Wait(TimeSpan.FromMinutes(4)))
{
msg.Message.RenewLock();
ListenerTrace.Info("Lock renewed");
}
}
catch (AggregateException aex)
{
ExceptionDispatchInfo.Capture(aex.InnerException).Throw();
throw;
}
range.Processed = true;
msg.Message.Complete();
}
}
}
}
ListenerTrace.Info("Initial indexing terminated");
return(totalProcessed);
}
public void Listen(ConcurrentChain chain = null)
{
ListenerTrace.Info($"Connecting to node {_Configuration.Indexer.Node}");
var ip = Utils.ParseIpEndpoint(_Configuration.Indexer.Node, Configuration.Indexer.Network.DefaultPort);
ListenerTrace.Info($"Connecting to node ip {ip.ToString()}");
var node = Node.Connect(Configuration.Indexer.Network, ip);
ListenerTrace.Info($"Connected, trying handshake...");
node.VersionHandshake();
ListenerTrace.Info($"Hanshaked");
node.Disconnect();
_Chain = new ConcurrentChain(_Configuration.Indexer.Network);
_Indexer = Configuration.Indexer.CreateIndexer();
if (chain == null)
{
chain = new ConcurrentChain(_Configuration.Indexer.Network);
}
_Chain = chain;
ListenerTrace.Info("Fetching headers from " + _Chain.Tip.Height + " (from azure)");
var client = Configuration.Indexer.CreateIndexerClient();
client.SynchronizeChain(chain);
ListenerTrace.Info("Headers fetched tip " + _Chain.Tip.Height);
_Disposables.Add(_IndexerScheduler = new CustomThreadPoolTaskScheduler(50, 100, "Indexing Threads"));
_Indexer.TaskScheduler = _IndexerScheduler;
_Group = new NodesGroup(Configuration.Indexer.Network);
_Disposables.Add(_Group);
_Group.AllowSameGroup = true;
_Group.MaximumNodeConnection = 2;
AddressManager addrman = new AddressManager();
addrman.Add(new NetworkAddress(ip), IPAddress.Parse("127.0.0.1"));
_Group.NodeConnectionParameters.TemplateBehaviors.Add(new AddressManagerBehavior(addrman)
{
Mode = AddressManagerBehaviorMode.None
});
_Group.NodeConnectionParameters.TemplateBehaviors.Add(new ChainBehavior(_Chain)
{
SkipPoWCheck = true
});
_Group.NodeConnectionParameters.TemplateBehaviors.Add(new Behavior(this));
ListenerTrace.Info("Fetching wallet rules...");
_Wallets = _Configuration.Indexer.CreateIndexerClient().GetAllWalletRules();
ListenerTrace.Info("Wallet rules fetched");
ListenerTrace.Info("Fetching wallet subscriptions...");
_Subscriptions = new SubscriptionCollection(_Configuration.GetSubscriptionsTable().Read());
ListenerTrace.Info("Subscriptions fetched");
_Group.Connect();
ListenerTrace.Info("Fetching transactions to broadcast...");
_Disposables.Add(
Configuration
.Topics
.BroadcastedTransactions
.CreateConsumer("listener", true)
.EnsureSubscriptionExists()
.OnMessage((tx, ctl) =>
{
uint256 hash = null;
var repo = Configuration.Indexer.CreateIndexerClient();
var rejects = Configuration.GetRejectTable();
try
{
hash = tx.Transaction.GetHash();
var indexedTx = repo.GetTransaction(hash);
ListenerTrace.Info("Broadcasting " + hash);
var reject = rejects.ReadOne(hash.ToString());
if (reject != null)
{
ListenerTrace.Info("Abort broadcasting of rejected");
return;
}
if (_Broadcasting.Count > 1000)
{
_Broadcasting.Clear();
}
_Broadcasting.TryAdd(hash, tx.Transaction);
if (indexedTx == null || !indexedTx.BlockIds.Any(id => Chain.Contains(id)))
{
var unused = SendMessageAsync(tx.Transaction);
}
var reschedule = new[]
{
TimeSpan.FromMinutes(5),
TimeSpan.FromMinutes(10),
TimeSpan.FromHours(1),
TimeSpan.FromHours(6),
TimeSpan.FromHours(24),
};
if (tx.Tried <= reschedule.Length - 1)
{
ctl.RescheduleIn(reschedule[tx.Tried]);
tx.Tried++;
}
}
catch (Exception ex)
{
if (!_Disposed)
{
LastException = ex;
ListenerTrace.Error("Error for new broadcasted transaction " + hash, ex);
throw;
}
}
}));
ListenerTrace.Info("Transactions to broadcast fetched");
_Disposables.Add(_Configuration
.Topics
.SubscriptionChanges
.EnsureSubscriptionExists()
.AddUnhandledExceptionHandler(ExceptionOnMessagePump)
.OnMessage(c =>
{
using (_SubscriptionSlimLock.LockWrite())
{
if (c.Added)
{
_Subscriptions.Add(c.Subscription);
}
else
{
_Subscriptions.Remove(c.Subscription.Id);
}
}
}));
_Disposables.Add(_Configuration
.Topics
.SendNotifications
.AddUnhandledExceptionHandler(ExceptionOnMessagePump)
.OnMessageAsync((n, act) =>
{
return(SendAsync(n, act).ContinueWith(t =>
{
if (!_Disposed)
{
if (t.Exception != null)
{
LastException = t.Exception;
}
}
}));
}, new OnMessageOptions()
{
MaxConcurrentCalls = 1000,
AutoComplete = true,
AutoRenewTimeout = TimeSpan.Zero
}));
_Disposables.Add(Configuration
.Topics
.AddedAddresses
.CreateConsumer("updater", true)
.EnsureSubscriptionExists()
.AddUnhandledExceptionHandler(ExceptionOnMessagePump)
.OnMessage(evt =>
{
if (evt == null)
{
return;
}
ListenerTrace.Info("New wallet rule");
using (_WalletsSlimLock.LockWrite())
{
foreach (var address in evt)
{
_Wallets.Add(address.CreateWalletRuleEntry());
}
}
}));
}
public void CustomThreadPoolTaskWorks()
{
TaskCompletionSource<int> completion = new TaskCompletionSource<int>();
var scheduler = new CustomThreadPoolTaskScheduler(10, 20);
for(int i = 0; i < 30; i++)
{
new Task(() => Task.WaitAll(completion.Task)).Start(scheduler);
}
Assert.Equal(0, scheduler.AvailableThreads);
Assert.Equal(20, scheduler.QueuedCount);
Assert.Equal(10, scheduler.ThreadsCount);
completion.SetResult(1);
scheduler.WaitFinished();
Assert.Equal(10, scheduler.AvailableThreads);
Assert.Equal(0, scheduler.QueuedCount);
Assert.Equal(10, scheduler.ThreadsCount);
}
