public void SetUp(Action <IHttpService> bootstrap = null)
{
_bus = new InMemoryBus($"bus_{_serverEndPoint.Port}");
var pipelineBus = InMemoryBus.CreateTest();
var queue = new QueuedHandlerThreadPool(pipelineBus, "Test", new QueueStatsManager(), true, TimeSpan.FromMilliseconds(50));
_multiQueuedHandler = new MultiQueuedHandler(new IQueuedHandler[] { queue }, null);
_multiQueuedHandler.Start();
_service = new KestrelHttpService(ServiceAccessibility.Private, _bus, new NaiveUriRouter(),
_multiQueuedHandler, false, null, 0, false, _serverEndPoint);
_internalDispatcher = new InternalDispatcherEndpoint(queue, _multiQueuedHandler);
_bus.Subscribe(_internalDispatcher);
KestrelHttpService.CreateAndSubscribePipeline(pipelineBus);
bootstrap?.Invoke(_service);
_server = new TestServer(
new WebHostBuilder()
.UseStartup(new ClusterVNodeStartup(Array.Empty <ISubsystem>(), queue, _bus, _multiQueuedHandler,
new TestAuthenticationProvider(),
new IHttpAuthenticationProvider[] {
new BasicHttpAuthenticationProvider(new TestAuthenticationProvider()),
new AnonymousHttpAuthenticationProvider(),
}, new TestAuthorizationProvider(), new FakeReadIndex(_ => false), 1024 * 1024, _service)));
_httpMessageHandler = _server.CreateHandler();
_client = new HttpAsyncClient(_timeout, _httpMessageHandler);
HttpBootstrap.Subscribe(_bus, _service);
}
public void of_http_requests_routing()
{
const int iterations = 100000;
IPublisher inputBus = new NoopPublisher();
var bus = InMemoryBus.CreateTest();
var queue = new QueuedHandlerThreadPool(bus, "Test", true, TimeSpan.FromMilliseconds(50));
var multiQueuedHandler = new MultiQueuedHandler(new IQueuedHandler[] { queue }, null);
var providers = new HttpAuthenticationProvider[] { new AnonymousHttpAuthenticationProvider() };
var httpService = new HttpService(ServiceAccessibility.Public, inputBus,
new TrieUriRouter(), multiQueuedHandler, false, "http://localhost:12345/");
HttpService.CreateAndSubscribePipeline(bus, providers);
var fakeController = new FakeController(iterations, null);
httpService.SetupController(fakeController);
httpService.Handle(new SystemMessage.SystemInit());
var rnd = new Random();
var sw = Stopwatch.StartNew();
var httpClient = new HttpAsyncClient();
for (int i = 0; i < iterations; ++i)
{
var route = fakeController.BoundRoutes[rnd.Next(0, fakeController.BoundRoutes.Count)];
switch (route.Item2)
{
case HttpMethod.Get:
httpClient.Get(route.Item1, TimeSpan.FromMilliseconds(10000), x => { }, x => { throw new Exception(); });
break;
case HttpMethod.Post:
httpClient.Post(route.Item1, "abracadabra", ContentType.Json, TimeSpan.FromMilliseconds(10000), x => { }, x => { throw new Exception(); });
break;
case HttpMethod.Delete:
httpClient.Delete(route.Item1, TimeSpan.FromMilliseconds(10000), x => { }, x => { throw new Exception(); });
break;
case HttpMethod.Put:
httpClient.Put(route.Item1, "abracadabra", ContentType.Json, TimeSpan.FromMilliseconds(10000), x => { }, x => { throw new Exception(); });
break;
default:
throw new Exception();
}
}
fakeController.CountdownEvent.Wait();
sw.Stop();
Console.WriteLine("{0} request done in {1} ({2:0.00} per sec)", iterations, sw.Elapsed, 1000.0 * iterations / sw.ElapsedMilliseconds);
httpService.Shutdown();
multiQueuedHandler.Stop();
}
public ClusterVNodeStartup(
ISubsystem[] subsystems,
IPublisher mainQueue,
ISubscriber mainBus,
MultiQueuedHandler httpMessageHandler,
IReadOnlyList <IHttpAuthenticationProvider> httpAuthenticationProviders,
IAuthorizationProvider authorizationProvider,
IReadIndex readIndex,
int maxAppendSize,
KestrelHttpService httpService)
{
if (subsystems == null)
{
throw new ArgumentNullException(nameof(subsystems));
}
if (mainQueue == null)
{
throw new ArgumentNullException(nameof(mainQueue));
}
if (httpAuthenticationProviders == null)
{
throw new ArgumentNullException(nameof(httpAuthenticationProviders));
}
if (authorizationProvider == null)
{
throw new ArgumentNullException(nameof(authorizationProvider));
}
if (readIndex == null)
{
throw new ArgumentNullException(nameof(readIndex));
}
Ensure.Positive(maxAppendSize, nameof(maxAppendSize));
if (httpService == null)
{
throw new ArgumentNullException(nameof(httpService));
}
if (mainBus == null)
{
throw new ArgumentNullException(nameof(mainBus));
}
_subsystems = subsystems;
_mainQueue = mainQueue;
_mainBus = mainBus;
_httpMessageHandler = httpMessageHandler;
_httpAuthenticationProviders = httpAuthenticationProviders;
_authorizationProvider = authorizationProvider;
_readIndex = readIndex;
_maxAppendSize = maxAppendSize;
_httpService = httpService;
_statusCheck = new StatusCheck(this);
}
public StorageReaderService(
IPublisher bus,
ISubscriber subscriber,
IReadIndex readIndex,
int threadCount,
IReadOnlyCheckpoint writerCheckpoint,
QueueStatsManager queueStatsManager)
{
Ensure.NotNull(bus, "bus");
Ensure.NotNull(subscriber, "subscriber");
Ensure.NotNull(readIndex, "readIndex");
Ensure.Positive(threadCount, "threadCount");
Ensure.NotNull(writerCheckpoint, "writerCheckpoint");
_bus = bus;
_readIndex = readIndex;
_threadCount = threadCount;
StorageReaderWorker[] readerWorkers = new StorageReaderWorker[threadCount];
InMemoryBus[] storageReaderBuses = new InMemoryBus[threadCount];
for (var i = 0; i < threadCount; i++)
{
readerWorkers[i] = new StorageReaderWorker(bus, readIndex, writerCheckpoint, i);
storageReaderBuses[i] = new InMemoryBus("StorageReaderBus", watchSlowMsg: false);
storageReaderBuses[i].Subscribe <ClientMessage.ReadEvent>(readerWorkers[i]);
storageReaderBuses[i].Subscribe <ClientMessage.ReadStreamEventsBackward>(readerWorkers[i]);
storageReaderBuses[i].Subscribe <ClientMessage.ReadStreamEventsForward>(readerWorkers[i]);
storageReaderBuses[i].Subscribe <ClientMessage.ReadAllEventsForward>(readerWorkers[i]);
storageReaderBuses[i].Subscribe <ClientMessage.ReadAllEventsBackward>(readerWorkers[i]);
storageReaderBuses[i].Subscribe <ClientMessage.FilteredReadAllEventsForward>(readerWorkers[i]);
storageReaderBuses[i].Subscribe <ClientMessage.FilteredReadAllEventsBackward>(readerWorkers[i]);
storageReaderBuses[i].Subscribe <StorageMessage.BatchLogExpiredMessages>(readerWorkers[i]);
storageReaderBuses[i].Subscribe <StorageMessage.EffectiveStreamAclRequest>(readerWorkers[i]);
storageReaderBuses[i].Subscribe <StorageMessage.StreamIdFromTransactionIdRequest>(readerWorkers[i]);
}
_workersMultiHandler = new MultiQueuedHandler(
_threadCount,
queueNum => new QueuedHandlerThreadPool(storageReaderBuses[queueNum],
string.Format("StorageReaderQueue #{0}", queueNum + 1),
queueStatsManager,
groupName: "StorageReaderQueue",
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200)));
_workersMultiHandler.Start();
subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadEvent, Message>());
subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadStreamEventsBackward, Message>());
subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadStreamEventsForward, Message>());
subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadAllEventsForward, Message>());
subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadAllEventsBackward, Message>());
subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.FilteredReadAllEventsForward, Message>());
subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.FilteredReadAllEventsBackward, Message>());
subscriber.Subscribe(_workersMultiHandler.WidenFrom <StorageMessage.BatchLogExpiredMessages, Message>());
subscriber.Subscribe(_workersMultiHandler.WidenFrom <StorageMessage.EffectiveStreamAclRequest, Message>());
subscriber.Subscribe(_workersMultiHandler.WidenFrom <StorageMessage.StreamIdFromTransactionIdRequest, Message>());
}
public HttpService(ServiceAccessibility accessibility, IPublisher inputBus, IUriRouter uriRouter,
MultiQueuedHandler multiQueuedHandler, params string[] prefixes)
{
Ensure.NotNull(inputBus, "inputBus");
Ensure.NotNull(uriRouter, "uriRouter");
Ensure.NotNull(prefixes, "prefixes");
_accessibility = accessibility;
_inputBus = inputBus;
_uriRouter = uriRouter;
_publishEnvelope = new PublishEnvelope(inputBus);
_requestsMultiHandler = multiQueuedHandler;
_server = new HttpAsyncServer(prefixes);
_server.RequestReceived += RequestReceived;
}
public void SetUp()
{
_bus = new InMemoryBus(string.Format("bus_{0}", _serverEndPoint.Port));
{
var pipelineBus = InMemoryBus.CreateTest();
var queue = new QueuedHandlerThreadPool(pipelineBus, "Test", true, TimeSpan.FromMilliseconds(50));
_multiQueuedHandler = new MultiQueuedHandler(new IQueuedHandler[] { queue }, null);
var httpAuthenticationProviders = new HttpAuthenticationProvider[] { new AnonymousHttpAuthenticationProvider() };
_service = new HttpService(ServiceAccessibility.Private, _bus, new NaiveUriRouter(),
_multiQueuedHandler, false, _serverEndPoint.ToHttpUrl());
HttpService.CreateAndSubscribePipeline(pipelineBus, httpAuthenticationProviders);
_client = new HttpAsyncClient();
}
HttpBootstrap.Subscribe(_bus, _service);
}
public KestrelHttpService(ServiceAccessibility accessibility, IPublisher inputBus, IUriRouter uriRouter,
MultiQueuedHandler multiQueuedHandler, bool logHttpRequests, IPAddress advertiseAsAddress,
int advertiseAsPort, bool disableAuthorization, params IPEndPoint[] endPoints)
{
Ensure.NotNull(inputBus, nameof(inputBus));
Ensure.NotNull(uriRouter, nameof(uriRouter));
Ensure.NotNull(endPoints, nameof(endPoints));
_accessibility = accessibility;
_inputBus = inputBus;
UriRouter = uriRouter;
LogHttpRequests = logHttpRequests;
AdvertiseAsAddress = advertiseAsAddress;
AdvertiseAsPort = advertiseAsPort;
EndPoints = endPoints;
}
public void SetUp()
{
_bus = new InMemoryBus(string.Format("bus_{0}", _serverEndPoint.Port));
{
var pipelineBus = InMemoryBus.CreateTest();
var queue = new QueuedHandlerThreadPool(pipelineBus, "Test", true, TimeSpan.FromMilliseconds(50));
_multiQueuedHandler = new MultiQueuedHandler(new IQueuedHandler[]{queue}, null);
var httpAuthenticationProviders = new HttpAuthenticationProvider[] {new AnonymousHttpAuthenticationProvider()};
_service = new HttpService(ServiceAccessibility.Private, _bus, new NaiveUriRouter(),
_multiQueuedHandler, _serverEndPoint.ToHttpUrl());
HttpService.CreateAndSubscribePipeline(pipelineBus, httpAuthenticationProviders);
_client = new HttpAsyncClient();
}
HttpBootstrap.Subscribe(_bus, _service);
}
public HttpService(ServiceAccessibility accessibility, IPublisher inputBus, IUriRouter uriRouter,
MultiQueuedHandler multiQueuedHandler, bool logHttpRequests, IPAddress advertiseAsAddress, int advertiseAsPort, params string[] prefixes)
{
Ensure.NotNull(inputBus, "inputBus");
Ensure.NotNull(uriRouter, "uriRouter");
Ensure.NotNull(prefixes, "prefixes");
_accessibility = accessibility;
_inputBus = inputBus;
_uriRouter = uriRouter;
_publishEnvelope = new PublishEnvelope(inputBus);
_requestsMultiHandler = multiQueuedHandler;
_logHttpRequests = logHttpRequests;
_server = new HttpAsyncServer(prefixes);
_server.RequestReceived += RequestReceived;
_advertiseAsAddress = advertiseAsAddress;
_advertiseAsPort = advertiseAsPort;
}
public void SetUp(Action <IHttpService> bootstrap = null)
{
_bus = new InMemoryBus($"bus_{_serverEndPoint.Port}");
var pipelineBus = InMemoryBus.CreateTest();
var queue = new QueuedHandlerThreadPool(pipelineBus, "Test", new QueueStatsManager(), true, TimeSpan.FromMilliseconds(50));
_multiQueuedHandler = new MultiQueuedHandler(new IQueuedHandler[] { queue }, null);
_multiQueuedHandler.Start();
_service = new KestrelHttpService(ServiceAccessibility.Private, _bus, new NaiveUriRouter(),
_multiQueuedHandler, false, null, 0, false, _serverEndPoint);
KestrelHttpService.CreateAndSubscribePipeline(pipelineBus);
bootstrap?.Invoke(_service);
_server = new TestServer(
new WebHostBuilder()
.UseStartup(new HttpServiceStartup(_service)));
_httpMessageHandler = _server.CreateHandler();
_client = new HttpAsyncClient(_timeout, _httpMessageHandler);
HttpBootstrap.Subscribe(_bus, _service);
}
protected CommandQueueSpecification(
int queueCount = 1,
int slowCmdMs = 500,
int slowAckMs = 500)
{
Bus = new CommandBus(
"Fixture Bus",
slowCmdThreshold: TimeSpan.FromMilliseconds(slowCmdMs),
slowMsgThreshold: TimeSpan.FromMilliseconds(slowAckMs));
Queue = new MultiQueuedHandler(
queueCount,
index => new QueuedHandler(
new AdHocHandler <Message>(
msg =>
{
if (msg is Command)
{
Bus.TryFire((Command)msg);
}
else
{
Bus.Publish(msg);
}
}),
$"Queue {index}"
)
);
Queue.Start();
TestQueue = new TestQueue(Bus);
try
{
Given();
When();
}
catch (Exception)
{
throw;
}
}
public StorageReaderService(IPublisher bus, ISubscriber subscriber, IReadIndex readIndex, int threadCount, ICheckpoint writerCheckpoint)
{
Ensure.NotNull(bus, "bus");
Ensure.NotNull(subscriber, "subscriber");
Ensure.NotNull(readIndex, "readIndex");
Ensure.Positive(threadCount, "threadCount");
Ensure.NotNull(writerCheckpoint, "writerCheckpoint");
_bus = bus;
_readIndex = readIndex;
_threadCount = threadCount;
var readerWorker = new StorageReaderWorker(bus, readIndex, writerCheckpoint);
var storageReaderBus = new InMemoryBus("StorageReaderBus", watchSlowMsg: false);
storageReaderBus.Subscribe <ClientMessage.ReadEvent>(readerWorker);
storageReaderBus.Subscribe <ClientMessage.ReadStreamEventsBackward>(readerWorker);
storageReaderBus.Subscribe <ClientMessage.ReadStreamEventsForward>(readerWorker);
storageReaderBus.Subscribe <ClientMessage.ReadAllEventsForward>(readerWorker);
storageReaderBus.Subscribe <ClientMessage.ReadAllEventsBackward>(readerWorker);
storageReaderBus.Subscribe <StorageMessage.CheckStreamAccess>(readerWorker);
_workersMultiHandler = new MultiQueuedHandler(
_threadCount,
queueNum => new QueuedHandlerThreadPool(storageReaderBus,
string.Format("StorageReaderQueue #{0}", queueNum + 1),
groupName: "StorageReaderQueue",
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200)));
_workersMultiHandler.Start();
subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadEvent, Message>());
subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadStreamEventsBackward, Message>());
subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadStreamEventsForward, Message>());
subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadAllEventsForward, Message>());
subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadAllEventsBackward, Message>());
subscriber.Subscribe(_workersMultiHandler.WidenFrom <StorageMessage.CheckStreamAccess, Message>());
}
public SingleVNode(TFChunkDb db,
SingleVNodeSettings vNodeSettings,
bool dbVerifyHashes,
int memTableEntryCount,
params ISubsystem[] subsystems)
{
Ensure.NotNull(db, "db");
Ensure.NotNull(vNodeSettings, "vNodeSettings");
_settings = vNodeSettings;
_mainBus = new InMemoryBus("MainBus");
_controller = new SingleVNodeController(_mainBus, _settings.ExternalHttpEndPoint, db, this);
_mainQueue = new QueuedHandler(_controller, "MainQueue");
_controller.SetMainQueue(_mainQueue);
_subsystems = subsystems;
// MONITORING
var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
var monitoringQueue = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100));
var monitoring = new MonitoringService(monitoringQueue,
monitoringRequestBus,
_mainQueue,
db.Config.WriterCheckpoint,
db.Config.Path,
vNodeSettings.StatsPeriod,
_settings.ExternalHttpEndPoint,
vNodeSettings.StatsStorage);
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.SystemInit, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.StateChangeMessage, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShutdown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<ClientMessage.WriteEventsCompleted, Message>());
monitoringInnerBus.Subscribe<SystemMessage.SystemInit>(monitoring);
monitoringInnerBus.Subscribe<SystemMessage.StateChangeMessage>(monitoring);
monitoringInnerBus.Subscribe<SystemMessage.BecomeShuttingDown>(monitoring);
monitoringInnerBus.Subscribe<SystemMessage.BecomeShutdown>(monitoring);
monitoringInnerBus.Subscribe<ClientMessage.WriteEventsCompleted>(monitoring);
monitoringInnerBus.Subscribe<MonitoringMessage.GetFreshStats>(monitoring);
var truncPos = db.Config.TruncateCheckpoint.Read();
if (truncPos != -1)
{
Log.Info("Truncate checkpoint is present. Truncate: {0} (0x{0:X}), Writer: {1} (0x{1:X}), Chaser: {2} (0x{2:X}), Epoch: {3} (0x{3:X})",
truncPos, db.Config.WriterCheckpoint.Read(), db.Config.ChaserCheckpoint.Read(), db.Config.EpochCheckpoint.Read());
var truncator = new TFChunkDbTruncator(db.Config);
truncator.TruncateDb(truncPos);
}
db.Open(dbVerifyHashes);
// STORAGE SUBSYSTEM
var indexPath = Path.Combine(db.Config.Path, "index");
var readerPool = new ObjectPool<ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(db, db.Config.WriterCheckpoint));
var tableIndex = new TableIndex(indexPath,
() => new HashListMemTable(maxSize: memTableEntryCount * 2),
() => new TFReaderLease(readerPool),
maxSizeForMemory: memTableEntryCount,
maxTablesPerLevel: 2,
inMem: db.Config.InMemDb);
var hasher = new XXHashUnsafe();
var readIndex = new ReadIndex(_mainQueue,
readerPool,
tableIndex,
hasher,
ESConsts.StreamInfoCacheCapacity,
Application.IsDefined(Application.AdditionalCommitChecks),
Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1);
var writer = new TFChunkWriter(db);
var epochManager = new EpochManager(ESConsts.CachedEpochCount,
db.Config.EpochCheckpoint,
writer,
initialReaderCount: 1,
maxReaderCount: 5,
readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint));
epochManager.Init();
var storageWriter = new StorageWriterService(_mainQueue, _mainBus, _settings.MinFlushDelay,
db, writer, readIndex.IndexWriter, epochManager); // subscribes internally
monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageWriter);
var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, ESConsts.StorageReaderThreadCount, db.Config.WriterCheckpoint);
_mainBus.Subscribe<SystemMessage.SystemInit>(storageReader);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageReader);
_mainBus.Subscribe<SystemMessage.BecomeShutdown>(storageReader);
monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageReader);
var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, readIndex.IndexCommitter, epochManager);
_mainBus.Subscribe<SystemMessage.SystemInit>(storageChaser);
_mainBus.Subscribe<SystemMessage.SystemStart>(storageChaser);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageChaser);
var storageScavenger = new StorageScavenger(db,
tableIndex,
hasher,
readIndex,
Application.IsDefined(Application.AlwaysKeepScavenged),
mergeChunks: false /*!Application.IsDefined(Application.DisableMergeChunks)*/);
// ReSharper disable RedundantTypeArgumentsOfMethod
_mainBus.Subscribe<ClientMessage.ScavengeDatabase>(storageScavenger);
// ReSharper restore RedundantTypeArgumentsOfMethod
// MISC WORKERS
_workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray();
_workersHandler = new MultiQueuedHandler(
vNodeSettings.WorkerThreads,
queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
string.Format("Worker #{0}", queueNum + 1),
groupName: "Workers",
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200)));
// AUTHENTICATION INFRASTRUCTURE
var passwordHashAlgorithm = new Rfc2898PasswordHashAlgorithm();
var dispatcher = new IODispatcher(_mainQueue, new PublishEnvelope(_workersHandler, crossThread: true));
var internalAuthenticationProvider = new InternalAuthenticationProvider(dispatcher, passwordHashAlgorithm, ESConsts.CachedPrincipalCount);
var passwordChangeNotificationReader = new PasswordChangeNotificationReader(_mainQueue, dispatcher);
_mainBus.Subscribe<SystemMessage.SystemStart>(passwordChangeNotificationReader);
_mainBus.Subscribe<SystemMessage.BecomeShutdown>(passwordChangeNotificationReader);
_mainBus.Subscribe(internalAuthenticationProvider);
SubscribeWorkers(bus =>
{
bus.Subscribe(dispatcher.ForwardReader);
bus.Subscribe(dispatcher.BackwardReader);
bus.Subscribe(dispatcher.Writer);
bus.Subscribe(dispatcher.StreamDeleter);
bus.Subscribe(dispatcher);
});
// TCP
{
var tcpService = new TcpService(_mainQueue, _settings.ExternalTcpEndPoint, _workersHandler,
TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(),
ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
internalAuthenticationProvider, null);
_mainBus.Subscribe<SystemMessage.SystemInit>(tcpService);
_mainBus.Subscribe<SystemMessage.SystemStart>(tcpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(tcpService);
}
// SECURE TCP
if (vNodeSettings.ExternalSecureTcpEndPoint != null)
{
var secureTcpService = new TcpService(_mainQueue, _settings.ExternalSecureTcpEndPoint, _workersHandler,
TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(),
ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
internalAuthenticationProvider, _settings.Certificate);
_mainBus.Subscribe<SystemMessage.SystemInit>(secureTcpService);
_mainBus.Subscribe<SystemMessage.SystemStart>(secureTcpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(secureTcpService);
}
SubscribeWorkers(bus =>
{
var tcpSendService = new TcpSendService();
// ReSharper disable RedundantTypeArgumentsOfMethod
bus.Subscribe<TcpMessage.TcpSend>(tcpSendService);
// ReSharper restore RedundantTypeArgumentsOfMethod
});
// HTTP
var httpAuthenticationProviders = new List<HttpAuthenticationProvider>
{
new BasicHttpAuthenticationProvider(internalAuthenticationProvider),
};
if (_settings.EnableTrustedAuth)
httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider());
httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider());
var httpPipe = new HttpMessagePipe();
var httpSendService = new HttpSendService(httpPipe, forwardRequests: false);
_mainBus.Subscribe<SystemMessage.StateChangeMessage>(httpSendService);
_mainBus.Subscribe(new WideningHandler<HttpMessage.SendOverHttp, Message>(_workersHandler));
SubscribeWorkers(bus =>
{
bus.Subscribe<HttpMessage.HttpSend>(httpSendService);
bus.Subscribe<HttpMessage.HttpSendPart>(httpSendService);
bus.Subscribe<HttpMessage.HttpBeginSend>(httpSendService);
bus.Subscribe<HttpMessage.HttpEndSend>(httpSendService);
bus.Subscribe<HttpMessage.SendOverHttp>(httpSendService);
});
_httpService = new HttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(),
_workersHandler, vNodeSettings.HttpPrefixes);
_httpService.SetupController(new AdminController(_mainQueue));
_httpService.SetupController(new PingController());
_httpService.SetupController(new StatController(monitoringQueue, _workersHandler));
_httpService.SetupController(new AtomController(httpSendService, _mainQueue, _workersHandler));
_httpService.SetupController(new GuidController(_mainQueue));
_httpService.SetupController(new UsersController(httpSendService, _mainQueue, _workersHandler));
_mainBus.Subscribe<SystemMessage.SystemInit>(_httpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(_httpService);
_mainBus.Subscribe<HttpMessage.PurgeTimedOutRequests>(_httpService);
SubscribeWorkers(bus =>
{
HttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders.ToArray());
});
// REQUEST MANAGEMENT
var requestManagement = new RequestManagementService(_mainQueue, 1, 1, vNodeSettings.PrepareTimeout, vNodeSettings.CommitTimeout);
_mainBus.Subscribe<SystemMessage.SystemInit>(requestManagement);
_mainBus.Subscribe<ClientMessage.WriteEvents>(requestManagement);
_mainBus.Subscribe<ClientMessage.TransactionStart>(requestManagement);
_mainBus.Subscribe<ClientMessage.TransactionWrite>(requestManagement);
_mainBus.Subscribe<ClientMessage.TransactionCommit>(requestManagement);
_mainBus.Subscribe<ClientMessage.DeleteStream>(requestManagement);
_mainBus.Subscribe<StorageMessage.RequestCompleted>(requestManagement);
_mainBus.Subscribe<StorageMessage.CheckStreamAccessCompleted>(requestManagement);
_mainBus.Subscribe<StorageMessage.AlreadyCommitted>(requestManagement);
_mainBus.Subscribe<StorageMessage.CommitAck>(requestManagement);
_mainBus.Subscribe<StorageMessage.PrepareAck>(requestManagement);
_mainBus.Subscribe<StorageMessage.WrongExpectedVersion>(requestManagement);
_mainBus.Subscribe<StorageMessage.InvalidTransaction>(requestManagement);
_mainBus.Subscribe<StorageMessage.StreamDeleted>(requestManagement);
_mainBus.Subscribe<StorageMessage.RequestManagerTimerTick>(requestManagement);
// SUBSCRIPTIONS
var subscrBus = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", false);
_mainBus.Subscribe(subscrQueue.WidenFrom<SystemMessage.SystemStart, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<TcpMessage.ConnectionClosed, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<ClientMessage.SubscribeToStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<ClientMessage.UnsubscribeFromStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<SubscriptionMessage.PollStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<SubscriptionMessage.CheckPollTimeout, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<StorageMessage.EventCommited, Message>());
var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex);
subscrBus.Subscribe<SystemMessage.SystemStart>(subscription);
subscrBus.Subscribe<SystemMessage.BecomeShuttingDown>(subscription);
subscrBus.Subscribe<TcpMessage.ConnectionClosed>(subscription);
subscrBus.Subscribe<ClientMessage.SubscribeToStream>(subscription);
subscrBus.Subscribe<ClientMessage.UnsubscribeFromStream>(subscription);
subscrBus.Subscribe<SubscriptionMessage.PollStream>(subscription);
subscrBus.Subscribe<SubscriptionMessage.CheckPollTimeout>(subscription);
subscrBus.Subscribe<StorageMessage.EventCommited>(subscription);
// USER MANAGEMENT
var ioDispatcher = new IODispatcher(_mainQueue, new PublishEnvelope(_mainQueue));
_mainBus.Subscribe(ioDispatcher.BackwardReader);
_mainBus.Subscribe(ioDispatcher.ForwardReader);
_mainBus.Subscribe(ioDispatcher.Writer);
_mainBus.Subscribe(ioDispatcher.StreamDeleter);
_mainBus.Subscribe(ioDispatcher);
var userManagement = new UserManagementService(
_mainQueue, ioDispatcher, passwordHashAlgorithm, vNodeSettings.SkipInitializeStandardUsersCheck);
_mainBus.Subscribe<UserManagementMessage.Create>(userManagement);
_mainBus.Subscribe<UserManagementMessage.Update>(userManagement);
_mainBus.Subscribe<UserManagementMessage.Enable>(userManagement);
_mainBus.Subscribe<UserManagementMessage.Disable>(userManagement);
_mainBus.Subscribe<UserManagementMessage.Delete>(userManagement);
_mainBus.Subscribe<UserManagementMessage.ResetPassword>(userManagement);
_mainBus.Subscribe<UserManagementMessage.ChangePassword>(userManagement);
_mainBus.Subscribe<UserManagementMessage.Get>(userManagement);
_mainBus.Subscribe<UserManagementMessage.GetAll>(userManagement);
_mainBus.Subscribe<SystemMessage.BecomeMaster>(userManagement);
// TIMER
_timeProvider = new RealTimeProvider();
_timerService = new TimerService(new ThreadBasedScheduler(_timeProvider));
_mainBus.Subscribe<SystemMessage.BecomeShutdown>(_timerService);
_mainBus.Subscribe<TimerMessage.Schedule>(_timerService);
_workersHandler.Start();
_mainQueue.Start();
monitoringQueue.Start();
subscrQueue.Start();
if (subsystems != null)
foreach (var subsystem in subsystems)
subsystem.Register(db, _mainQueue, _mainBus, _timerService, _timeProvider, httpSendService, new[]{_httpService}, _workersHandler);
}
public SingleVNode(TFChunkDb db,
SingleVNodeSettings vNodeSettings,
bool dbVerifyHashes,
int memTableEntryCount,
params ISubsystem[] subsystems)
{
Ensure.NotNull(db, "db");
Ensure.NotNull(vNodeSettings, "vNodeSettings");
_settings = vNodeSettings;
_mainBus = new InMemoryBus("MainBus");
_controller = new SingleVNodeController(_mainBus, _settings.ExternalHttpEndPoint, db, this);
_mainQueue = new QueuedHandler(_controller, "MainQueue");
_controller.SetMainQueue(_mainQueue);
_subsystems = subsystems;
// MONITORING
var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
var monitoringQueue = new QueuedHandler(monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100));
var monitoring = new MonitoringService(monitoringQueue,
monitoringRequestBus,
_mainQueue,
db.Config.WriterCheckpoint,
db.Config.Path,
vNodeSettings.StatsPeriod,
_settings.ExternalHttpEndPoint,
vNodeSettings.StatsStorage);
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.SystemInit, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.StateChangeMessage, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <ClientMessage.WriteEventsCompleted, Message>());
monitoringInnerBus.Subscribe <SystemMessage.SystemInit>(monitoring);
monitoringInnerBus.Subscribe <SystemMessage.StateChangeMessage>(monitoring);
monitoringInnerBus.Subscribe <SystemMessage.BecomeShuttingDown>(monitoring);
monitoringInnerBus.Subscribe <ClientMessage.WriteEventsCompleted>(monitoring);
monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshStats>(monitoring);
var truncPos = db.Config.TruncateCheckpoint.Read();
if (truncPos != -1)
{
Log.Info("Truncate checkpoint is present. Truncate: {0} (0x{0:X}), Writer: {1} (0x{1:X}), Chaser: {2} (0x{2:X}), Epoch: {3} (0x{3:X})",
truncPos, db.Config.WriterCheckpoint.Read(), db.Config.ChaserCheckpoint.Read(), db.Config.EpochCheckpoint.Read());
var truncator = new TFChunkDbTruncator(db.Config);
truncator.TruncateDb(truncPos);
}
db.Open(dbVerifyHashes);
// STORAGE SUBSYSTEM
var indexPath = Path.Combine(db.Config.Path, "index");
var tableIndex = new TableIndex(indexPath,
() => new HashListMemTable(maxSize: memTableEntryCount * 2),
maxSizeForMemory: memTableEntryCount,
maxTablesPerLevel: 2);
var readIndex = new ReadIndex(_mainQueue,
ESConsts.PTableInitialReaderCount,
ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(db, db.Config.WriterCheckpoint),
tableIndex,
new XXHashUnsafe(),
new LRUCache <string, StreamCacheInfo>(ESConsts.StreamMetadataCacheCapacity),
Application.IsDefined(Application.AdditionalCommitChecks),
Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1);
var writer = new TFChunkWriter(db);
var epochManager = new EpochManager(ESConsts.CachedEpochCount,
db.Config.EpochCheckpoint,
writer,
initialReaderCount: 1,
maxReaderCount: 5,
readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint));
epochManager.Init();
var storageWriter = new StorageWriterService(_mainQueue, _mainBus, _settings.MinFlushDelayMs,
db, writer, readIndex, epochManager); // subscribes internally
monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageWriter);
var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, ESConsts.StorageReaderThreadCount, db.Config.WriterCheckpoint);
_mainBus.Subscribe <SystemMessage.SystemInit>(storageReader);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageReader);
_mainBus.Subscribe <SystemMessage.BecomeShutdown>(storageReader);
monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageReader);
var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, readIndex, epochManager);
_mainBus.Subscribe <SystemMessage.SystemInit>(storageChaser);
_mainBus.Subscribe <SystemMessage.SystemStart>(storageChaser);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageChaser);
var storageScavenger = new StorageScavenger(db,
readIndex,
Application.IsDefined(Application.AlwaysKeepScavenged),
mergeChunks: false /*!Application.IsDefined(Application.DisableMergeChunks)*/);
// ReSharper disable RedundantTypeArgumentsOfMethod
_mainBus.Subscribe <SystemMessage.ScavengeDatabase>(storageScavenger);
// ReSharper restore RedundantTypeArgumentsOfMethod
// MISC WORKERS
_workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray();
_workersHandler = new MultiQueuedHandler(
vNodeSettings.WorkerThreads,
queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
string.Format("Worker #{0}", queueNum + 1),
groupName: "Workers",
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200)));
// AUTHENTICATION INFRASTRUCTURE
var passwordHashAlgorithm = new Rfc2898PasswordHashAlgorithm();
var dispatcher = new IODispatcher(_mainBus, new PublishEnvelope(_workersHandler, crossThread: true));
var internalAuthenticationProvider = new InternalAuthenticationProvider(dispatcher, passwordHashAlgorithm, ESConsts.CachedPrincipalCount);
var passwordChangeNotificationReader = new PasswordChangeNotificationReader(_mainQueue, dispatcher);
_mainBus.Subscribe <SystemMessage.SystemStart>(passwordChangeNotificationReader);
_mainBus.Subscribe <SystemMessage.BecomeShutdown>(passwordChangeNotificationReader);
_mainBus.Subscribe(internalAuthenticationProvider);
SubscribeWorkers(bus =>
{
bus.Subscribe(dispatcher.ForwardReader);
bus.Subscribe(dispatcher.BackwardReader);
bus.Subscribe(dispatcher.Writer);
bus.Subscribe(dispatcher.StreamDeleter);
bus.Subscribe(dispatcher);
});
// TCP
{
var tcpService = new TcpService(_mainQueue, _settings.ExternalTcpEndPoint, _workersHandler,
TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(),
ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
internalAuthenticationProvider, null);
_mainBus.Subscribe <SystemMessage.SystemInit>(tcpService);
_mainBus.Subscribe <SystemMessage.SystemStart>(tcpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(tcpService);
}
// SECURE TCP
if (vNodeSettings.ExternalSecureTcpEndPoint != null)
{
var secureTcpService = new TcpService(_mainQueue, _settings.ExternalSecureTcpEndPoint, _workersHandler,
TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(),
ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
internalAuthenticationProvider, _settings.Certificate);
_mainBus.Subscribe <SystemMessage.SystemInit>(secureTcpService);
_mainBus.Subscribe <SystemMessage.SystemStart>(secureTcpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(secureTcpService);
}
SubscribeWorkers(bus =>
{
var tcpSendService = new TcpSendService();
// ReSharper disable RedundantTypeArgumentsOfMethod
bus.Subscribe <TcpMessage.TcpSend>(tcpSendService);
// ReSharper restore RedundantTypeArgumentsOfMethod
});
// HTTP
var authenticationProviders = new List <AuthenticationProvider>
{
new BasicHttpAuthenticationProvider(internalAuthenticationProvider),
};
if (_settings.EnableTrustedAuth)
{
authenticationProviders.Add(new TrustedAuthenticationProvider());
}
authenticationProviders.Add(new AnonymousAuthenticationProvider());
var httpPipe = new HttpMessagePipe();
var httpSendService = new HttpSendService(httpPipe, forwardRequests: false);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(httpSendService);
_mainBus.Subscribe(new WideningHandler <HttpMessage.SendOverHttp, Message>(_workersHandler));
SubscribeWorkers(bus =>
{
bus.Subscribe <HttpMessage.HttpSend>(httpSendService);
bus.Subscribe <HttpMessage.HttpSendPart>(httpSendService);
bus.Subscribe <HttpMessage.HttpBeginSend>(httpSendService);
bus.Subscribe <HttpMessage.HttpEndSend>(httpSendService);
bus.Subscribe <HttpMessage.SendOverHttp>(httpSendService);
});
_httpService = new HttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(),
_workersHandler, vNodeSettings.HttpPrefixes);
_httpService.SetupController(new AdminController(_mainQueue));
_httpService.SetupController(new PingController());
_httpService.SetupController(new StatController(monitoringQueue, _workersHandler));
_httpService.SetupController(new AtomController(httpSendService, _mainQueue, _workersHandler));
_httpService.SetupController(new GuidController(_mainQueue));
_httpService.SetupController(new UsersController(httpSendService, _mainQueue, _workersHandler));
_mainBus.Subscribe <SystemMessage.SystemInit>(_httpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_httpService);
_mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_httpService);
SubscribeWorkers(bus =>
{
HttpService.CreateAndSubscribePipeline(bus, authenticationProviders.ToArray());
});
// REQUEST MANAGEMENT
var requestManagement = new RequestManagementService(_mainQueue, 1, 1, vNodeSettings.PrepareTimeout, vNodeSettings.CommitTimeout);
_mainBus.Subscribe <SystemMessage.SystemInit>(requestManagement);
_mainBus.Subscribe <ClientMessage.WriteEvents>(requestManagement);
_mainBus.Subscribe <ClientMessage.TransactionStart>(requestManagement);
_mainBus.Subscribe <ClientMessage.TransactionWrite>(requestManagement);
_mainBus.Subscribe <ClientMessage.TransactionCommit>(requestManagement);
_mainBus.Subscribe <ClientMessage.DeleteStream>(requestManagement);
_mainBus.Subscribe <StorageMessage.RequestCompleted>(requestManagement);
_mainBus.Subscribe <StorageMessage.CheckStreamAccessCompleted>(requestManagement);
_mainBus.Subscribe <StorageMessage.AlreadyCommitted>(requestManagement);
_mainBus.Subscribe <StorageMessage.CommitAck>(requestManagement);
_mainBus.Subscribe <StorageMessage.PrepareAck>(requestManagement);
_mainBus.Subscribe <StorageMessage.WrongExpectedVersion>(requestManagement);
_mainBus.Subscribe <StorageMessage.InvalidTransaction>(requestManagement);
_mainBus.Subscribe <StorageMessage.StreamDeleted>(requestManagement);
_mainBus.Subscribe <StorageMessage.RequestManagerTimerTick>(requestManagement);
// SUBSCRIPTIONS
var subscrBus = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
var subscription = new SubscriptionsService(readIndex);
subscrBus.Subscribe <TcpMessage.ConnectionClosed>(subscription);
subscrBus.Subscribe <ClientMessage.SubscribeToStream>(subscription);
subscrBus.Subscribe <ClientMessage.UnsubscribeFromStream>(subscription);
subscrBus.Subscribe <StorageMessage.EventCommited>(subscription);
var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", false);
_mainBus.Subscribe(subscrQueue.WidenFrom <TcpMessage.ConnectionClosed, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.SubscribeToStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.UnsubscribeFromStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <StorageMessage.EventCommited, Message>());
// USER MANAGEMENT
var ioDispatcher = new IODispatcher(_mainBus, new PublishEnvelope(_mainQueue));
_mainBus.Subscribe(ioDispatcher.BackwardReader);
_mainBus.Subscribe(ioDispatcher.ForwardReader);
_mainBus.Subscribe(ioDispatcher.Writer);
_mainBus.Subscribe(ioDispatcher.StreamDeleter);
_mainBus.Subscribe(ioDispatcher);
var userManagement = new UserManagementService(
_mainQueue, ioDispatcher, passwordHashAlgorithm, vNodeSettings.SkipInitializeStandardUsersCheck);
_mainBus.Subscribe <UserManagementMessage.Create>(userManagement);
_mainBus.Subscribe <UserManagementMessage.Update>(userManagement);
_mainBus.Subscribe <UserManagementMessage.Enable>(userManagement);
_mainBus.Subscribe <UserManagementMessage.Disable>(userManagement);
_mainBus.Subscribe <UserManagementMessage.Delete>(userManagement);
_mainBus.Subscribe <UserManagementMessage.ResetPassword>(userManagement);
_mainBus.Subscribe <UserManagementMessage.ChangePassword>(userManagement);
_mainBus.Subscribe <UserManagementMessage.Get>(userManagement);
_mainBus.Subscribe <UserManagementMessage.GetAll>(userManagement);
_mainBus.Subscribe <SystemMessage.BecomeMaster>(userManagement);
// TIMER
_timeProvider = new RealTimeProvider();
_timerService = new TimerService(new ThreadBasedScheduler(_timeProvider));
// ReSharper disable RedundantTypeArgumentsOfMethod
_mainBus.Subscribe <TimerMessage.Schedule>(_timerService);
// ReSharper restore RedundantTypeArgumentsOfMethod
_workersHandler.Start();
_mainQueue.Start();
monitoringQueue.Start();
subscrQueue.Start();
if (subsystems != null)
{
foreach (var subsystem in subsystems)
{
subsystem.Register(db, _mainQueue, _mainBus, _timerService, _timeProvider, httpSendService, new[] { _httpService }, _workersHandler);
}
}
}
public void of_http_requests_routing()
{
const int iterations = 100000;
IPublisher inputBus = new NoopPublisher();
var bus = InMemoryBus.CreateTest();
var queue = new QueuedHandlerThreadPool(bus, "Test", true, TimeSpan.FromMilliseconds(50));
var multiQueuedHandler = new MultiQueuedHandler(new IQueuedHandler[]{queue}, null);
var providers = new AuthenticationProvider[] {new AnonymousAuthenticationProvider()};
var httpService = new HttpService(ServiceAccessibility.Public, inputBus,
new TrieUriRouter(), multiQueuedHandler, "http://localhost:12345/");
HttpService.CreateAndSubscribePipeline(bus, providers);
var fakeController = new FakeController(iterations, null);
httpService.SetupController(fakeController);
httpService.Handle(new SystemMessage.SystemInit());
var rnd = new Random();
var sw = Stopwatch.StartNew();
var httpClient = new HttpAsyncClient();
for (int i = 0; i < iterations; ++i)
{
var route = fakeController.BoundRoutes[rnd.Next(0, fakeController.BoundRoutes.Count)];
switch (route.Item2)
{
case HttpMethod.Get:
httpClient.Get(route.Item1,TimeSpan.FromMilliseconds(10000), x => { }, x => { throw new Exception();});
break;
case HttpMethod.Post:
httpClient.Post(route.Item1, "abracadabra", ContentType.Json, TimeSpan.FromMilliseconds(10000), x => { }, x => { throw new Exception(); });
break;
case HttpMethod.Delete:
httpClient.Delete(route.Item1, TimeSpan.FromMilliseconds(10000), x => { }, x => { throw new Exception(); });
break;
case HttpMethod.Put:
httpClient.Put(route.Item1, "abracadabra", ContentType.Json, TimeSpan.FromMilliseconds(10000), x => { }, x => { throw new Exception(); });
break;
default:
throw new Exception();
}
}
fakeController.CountdownEvent.Wait();
sw.Stop();
Console.WriteLine("{0} request done in {1} ({2:0.00} per sec)", iterations, sw.Elapsed, 1000.0 * iterations / sw.ElapsedMilliseconds);
httpService.Shutdown();
multiQueuedHandler.Stop();
}
public PlaygroundVNode(PlaygroundVNodeSettings vNodeSettings)
{
_tcpEndPoint = vNodeSettings.ExternalTcpEndPoint;
_httpEndPoint = vNodeSettings.ExternalHttpEndPoint;
_mainBus = new InMemoryBus("MainBus");
_controller = new PlaygroundVNodeController(Bus, _httpEndPoint);
_mainQueue = new QueuedHandler(_controller, "MainQueue");
_controller.SetMainQueue(MainQueue);
// MONITORING
var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
var monitoringQueue = new QueuedHandler(
monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100));
Bus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.SystemInit, Message>());
Bus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.StateChangeMessage, Message>());
Bus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
// MISC WORKERS
_workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(50))).ToArray();
_workersHandler = new MultiQueuedHandler(
vNodeSettings.WorkerThreads,
queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
string.Format("Worker #{0}", queueNum + 1),
groupName: "Workers",
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(50)));
// AUTHENTICATION INFRASTRUCTURE
var dispatcher = new IODispatcher(_mainBus, new PublishEnvelope(_workersHandler, crossThread: true));
var passwordHashAlgorithm = new Rfc2898PasswordHashAlgorithm();
var internalAuthenticationProvider = new InternalAuthenticationProvider(dispatcher, passwordHashAlgorithm, 1000);
var passwordChangeNotificationReader = new PasswordChangeNotificationReader(_mainQueue, dispatcher);
_mainBus.Subscribe<SystemMessage.SystemStart>(passwordChangeNotificationReader);
_mainBus.Subscribe<SystemMessage.BecomeShutdown>(passwordChangeNotificationReader);
_mainBus.Subscribe(internalAuthenticationProvider);
_mainBus.Subscribe(dispatcher);
SubscribeWorkers(bus =>
{
bus.Subscribe(dispatcher.ForwardReader);
bus.Subscribe(dispatcher.BackwardReader);
bus.Subscribe(dispatcher.Writer);
bus.Subscribe(dispatcher.StreamDeleter);
});
// TCP
var tcpService = new TcpService(
MainQueue, _tcpEndPoint, _workersHandler, TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(),
ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout, internalAuthenticationProvider, null);
Bus.Subscribe<SystemMessage.SystemInit>(tcpService);
Bus.Subscribe<SystemMessage.SystemStart>(tcpService);
Bus.Subscribe<SystemMessage.BecomeShuttingDown>(tcpService);
// HTTP
{
var httpAuthenticationProviders = new HttpAuthenticationProvider[]
{
new BasicHttpAuthenticationProvider(internalAuthenticationProvider),
new TrustedHttpAuthenticationProvider(),
new AnonymousHttpAuthenticationProvider()
};
var httpPipe = new HttpMessagePipe();
var httpSendService = new HttpSendService(httpPipe, forwardRequests: false);
_mainBus.Subscribe<SystemMessage.StateChangeMessage>(httpSendService);
_mainBus.Subscribe(new WideningHandler<HttpMessage.SendOverHttp, Message>(_workersHandler));
SubscribeWorkers(bus =>
{
bus.Subscribe<HttpMessage.HttpSend>(httpSendService);
bus.Subscribe<HttpMessage.HttpSendPart>(httpSendService);
bus.Subscribe<HttpMessage.HttpBeginSend>(httpSendService);
bus.Subscribe<HttpMessage.HttpEndSend>(httpSendService);
bus.Subscribe<HttpMessage.SendOverHttp>(httpSendService);
});
_httpService = new HttpService(ServiceAccessibility.Private, _mainQueue, new TrieUriRouter(),
_workersHandler, vNodeSettings.HttpPrefixes);
_mainBus.Subscribe<SystemMessage.SystemInit>(_httpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(_httpService);
_mainBus.Subscribe<HttpMessage.PurgeTimedOutRequests>(_httpService);
HttpService.SetupController(new AdminController(_mainQueue));
HttpService.SetupController(new PingController());
HttpService.SetupController(new StatController(monitoringQueue, _workersHandler));
HttpService.SetupController(new AtomController(httpSendService, _mainQueue, _workersHandler));
HttpService.SetupController(new GuidController(_mainQueue));
HttpService.SetupController(new UsersController(httpSendService, _mainQueue, _workersHandler));
SubscribeWorkers(bus => HttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders));
}
// TIMER
_timerService = new TimerService(new ThreadBasedScheduler(new RealTimeProvider()));
Bus.Subscribe<TimerMessage.Schedule>(TimerService);
monitoringQueue.Start();
MainQueue.Start();
}
public ClusterVNode(TFChunkDb db,
ClusterVNodeSettings vNodeSettings,
IGossipSeedSource gossipSeedSource,
bool dbVerifyHashes,
int memTableEntryCount,
params ISubsystem[] subsystems)
{
Ensure.NotNull(db, "db");
Ensure.NotNull(vNodeSettings, "vNodeSettings");
Ensure.NotNull(gossipSeedSource, "gossipSeedSource");
_nodeInfo = vNodeSettings.NodeInfo;
_mainBus = new InMemoryBus("MainBus");
var forwardingProxy = new MessageForwardingProxy();
// MISC WORKERS
_workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray();
_workersHandler = new MultiQueuedHandler(
vNodeSettings.WorkerThreads,
queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
string.Format("Worker #{0}", queueNum + 1),
groupName: "Workers",
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200)));
_controller = new ClusterVNodeController(_mainBus, _nodeInfo, db, vNodeSettings, this, forwardingProxy);
_mainQueue = new QueuedHandler(_controller, "MainQueue");
_controller.SetMainQueue(_mainQueue);
_subsystems = subsystems;
// MONITORING
var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
var monitoringQueue = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100));
var monitoring = new MonitoringService(monitoringQueue,
monitoringRequestBus,
_mainQueue,
db.Config.WriterCheckpoint,
db.Config.Path,
vNodeSettings.StatsPeriod,
_nodeInfo.ExternalHttp,
vNodeSettings.StatsStorage);
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.SystemInit, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.StateChangeMessage, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShutdown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <ClientMessage.WriteEventsCompleted, Message>());
monitoringInnerBus.Subscribe <SystemMessage.SystemInit>(monitoring);
monitoringInnerBus.Subscribe <SystemMessage.StateChangeMessage>(monitoring);
monitoringInnerBus.Subscribe <SystemMessage.BecomeShuttingDown>(monitoring);
monitoringInnerBus.Subscribe <SystemMessage.BecomeShutdown>(monitoring);
monitoringInnerBus.Subscribe <ClientMessage.WriteEventsCompleted>(monitoring);
monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshStats>(monitoring);
var truncPos = db.Config.TruncateCheckpoint.Read();
if (truncPos != -1)
{
Log.Info("Truncate checkpoint is present. Truncate: {0} (0x{0:X}), Writer: {1} (0x{1:X}), Chaser: {2} (0x{2:X}), Epoch: {3} (0x{3:X})",
truncPos, db.Config.WriterCheckpoint.Read(), db.Config.ChaserCheckpoint.Read(), db.Config.EpochCheckpoint.Read());
var truncator = new TFChunkDbTruncator(db.Config);
truncator.TruncateDb(truncPos);
}
// STORAGE SUBSYSTEM
db.Open(dbVerifyHashes);
var indexPath = Path.Combine(db.Config.Path, "index");
var readerPool = new ObjectPool <ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(db, db.Config.WriterCheckpoint));
var tableIndex = new TableIndex(indexPath,
() => new HashListMemTable(maxSize: memTableEntryCount * 2),
() => new TFReaderLease(readerPool),
maxSizeForMemory: memTableEntryCount,
maxTablesPerLevel: 2,
inMem: db.Config.InMemDb);
var hash = new XXHashUnsafe();
var readIndex = new ReadIndex(_mainQueue,
readerPool,
tableIndex,
hash,
ESConsts.StreamInfoCacheCapacity,
Application.IsDefined(Application.AdditionalCommitChecks),
Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1);
var writer = new TFChunkWriter(db);
var epochManager = new EpochManager(ESConsts.CachedEpochCount,
db.Config.EpochCheckpoint,
writer,
initialReaderCount: 1,
maxReaderCount: 5,
readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint));
epochManager.Init();
var storageWriter = new ClusterStorageWriterService(_mainQueue, _mainBus, vNodeSettings.MinFlushDelay,
db, writer, readIndex.IndexWriter, epochManager,
() => readIndex.LastCommitPosition); // subscribes internally
monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageWriter);
var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, ESConsts.StorageReaderThreadCount, db.Config.WriterCheckpoint);
_mainBus.Subscribe <SystemMessage.SystemInit>(storageReader);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageReader);
_mainBus.Subscribe <SystemMessage.BecomeShutdown>(storageReader);
monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageReader);
var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, readIndex.IndexCommitter, epochManager);
#if DEBUG
QueueStatsCollector.InitializeCheckpoints(
_nodeInfo.DebugIndex, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
#endif
_mainBus.Subscribe <SystemMessage.SystemInit>(storageChaser);
_mainBus.Subscribe <SystemMessage.SystemStart>(storageChaser);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageChaser);
var storageScavenger = new StorageScavenger(db, tableIndex, hash, readIndex,
Application.IsDefined(Application.AlwaysKeepScavenged),
mergeChunks: !vNodeSettings.DisableScavengeMerging);
// ReSharper disable RedundantTypeArgumentsOfMethod
_mainBus.Subscribe <ClientMessage.ScavengeDatabase>(storageScavenger);
// ReSharper restore RedundantTypeArgumentsOfMethod
// AUTHENTICATION INFRASTRUCTURE - delegate to plugins
var authenticationProvider = vNodeSettings.AuthenticationProviderFactory.BuildAuthenticationProvider(_mainQueue, _mainBus, _workersHandler, _workerBuses);
Ensure.NotNull(authenticationProvider, "authenticationProvider");
{
// EXTERNAL TCP
var extTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalTcp, _workersHandler,
TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(),
ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
authenticationProvider, null);
_mainBus.Subscribe <SystemMessage.SystemInit>(extTcpService);
_mainBus.Subscribe <SystemMessage.SystemStart>(extTcpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(extTcpService);
// EXTERNAL SECURE TCP
if (_nodeInfo.ExternalSecureTcp != null)
{
var extSecTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalSecureTcp, _workersHandler,
TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(),
ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
authenticationProvider, vNodeSettings.Certificate);
_mainBus.Subscribe <SystemMessage.SystemInit>(extSecTcpService);
_mainBus.Subscribe <SystemMessage.SystemStart>(extSecTcpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(extSecTcpService);
}
// INTERNAL TCP
var intTcpService = new TcpService(_mainQueue, _nodeInfo.InternalTcp, _workersHandler,
TcpServiceType.Internal, TcpSecurityType.Normal,
new InternalTcpDispatcher(),
ESConsts.InternalHeartbeatInterval, ESConsts.InternalHeartbeatTimeout,
authenticationProvider, null);
_mainBus.Subscribe <SystemMessage.SystemInit>(intTcpService);
_mainBus.Subscribe <SystemMessage.SystemStart>(intTcpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(intTcpService);
// INTERNAL SECURE TCP
if (_nodeInfo.InternalSecureTcp != null)
{
var intSecTcpService = new TcpService(_mainQueue, _nodeInfo.InternalSecureTcp, _workersHandler,
TcpServiceType.Internal, TcpSecurityType.Secure,
new InternalTcpDispatcher(),
ESConsts.InternalHeartbeatInterval, ESConsts.InternalHeartbeatTimeout,
authenticationProvider, vNodeSettings.Certificate);
_mainBus.Subscribe <SystemMessage.SystemInit>(intSecTcpService);
_mainBus.Subscribe <SystemMessage.SystemStart>(intSecTcpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(intSecTcpService);
}
}
SubscribeWorkers(bus =>
{
var tcpSendService = new TcpSendService();
// ReSharper disable RedundantTypeArgumentsOfMethod
bus.Subscribe <TcpMessage.TcpSend>(tcpSendService);
// ReSharper restore RedundantTypeArgumentsOfMethod
});
var httpAuthenticationProviders = new List <HttpAuthenticationProvider>
{
new BasicHttpAuthenticationProvider(authenticationProvider),
};
if (vNodeSettings.EnableTrustedAuth)
{
httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider());
}
httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider());
var httpPipe = new HttpMessagePipe();
var httpSendService = new HttpSendService(httpPipe, forwardRequests: true);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(httpSendService);
_mainBus.Subscribe(new WideningHandler <HttpMessage.SendOverHttp, Message>(_workersHandler));
SubscribeWorkers(bus =>
{
bus.Subscribe <HttpMessage.HttpSend>(httpSendService);
bus.Subscribe <HttpMessage.HttpSendPart>(httpSendService);
bus.Subscribe <HttpMessage.HttpBeginSend>(httpSendService);
bus.Subscribe <HttpMessage.HttpEndSend>(httpSendService);
bus.Subscribe <HttpMessage.SendOverHttp>(httpSendService);
});
var adminController = new AdminController(_mainQueue);
var pingController = new PingController();
var statController = new StatController(monitoringQueue, _workersHandler);
var atomController = new AtomController(httpSendService, _mainQueue, _workersHandler);
var gossipController = new GossipController(_mainQueue, _workersHandler);
var electController = new ElectController(_mainQueue);
// HTTP SENDERS
gossipController.SubscribeSenders(httpPipe);
electController.SubscribeSenders(httpPipe);
// EXTERNAL HTTP
_externalHttpService = new HttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(),
_workersHandler, vNodeSettings.HttpPrefixes);
if (vNodeSettings.AdminOnPublic)
{
_externalHttpService.SetupController(adminController);
}
_externalHttpService.SetupController(pingController);
if (vNodeSettings.StatsOnPublic)
{
_externalHttpService.SetupController(statController);
}
_externalHttpService.SetupController(atomController);
if (vNodeSettings.GossipOnPublic)
{
_externalHttpService.SetupController(gossipController);
}
_mainBus.Subscribe <SystemMessage.SystemInit>(_externalHttpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_externalHttpService);
_mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_externalHttpService);
// INTERNAL HTTP
_internalHttpService = new HttpService(ServiceAccessibility.Private, _mainQueue, new TrieUriRouter(),
_workersHandler, _nodeInfo.InternalHttp.ToHttpUrl());
_internalHttpService.SetupController(adminController);
_internalHttpService.SetupController(pingController);
_internalHttpService.SetupController(statController);
_internalHttpService.SetupController(atomController);
_internalHttpService.SetupController(gossipController);
_internalHttpService.SetupController(electController);
// Authentication plugin HTTP
vNodeSettings.AuthenticationProviderFactory.RegisterHttpControllers(_externalHttpService, _internalHttpService, httpSendService, _mainQueue, _workersHandler);
_mainBus.Subscribe <SystemMessage.SystemInit>(_internalHttpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_internalHttpService);
_mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_internalHttpService);
SubscribeWorkers(bus =>
{
HttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders.ToArray());
});
// REQUEST FORWARDING
var forwardingService = new RequestForwardingService(_mainQueue, forwardingProxy, TimeSpan.FromSeconds(1));
_mainBus.Subscribe <SystemMessage.SystemStart>(forwardingService);
_mainBus.Subscribe <SystemMessage.RequestForwardingTimerTick>(forwardingService);
_mainBus.Subscribe <ClientMessage.NotHandled>(forwardingService);
_mainBus.Subscribe <ClientMessage.WriteEventsCompleted>(forwardingService);
_mainBus.Subscribe <ClientMessage.TransactionStartCompleted>(forwardingService);
_mainBus.Subscribe <ClientMessage.TransactionWriteCompleted>(forwardingService);
_mainBus.Subscribe <ClientMessage.TransactionCommitCompleted>(forwardingService);
_mainBus.Subscribe <ClientMessage.DeleteStreamCompleted>(forwardingService);
// REQUEST MANAGEMENT
var requestManagement = new RequestManagementService(_mainQueue,
vNodeSettings.PrepareAckCount,
vNodeSettings.CommitAckCount,
vNodeSettings.PrepareTimeout,
vNodeSettings.CommitTimeout);
_mainBus.Subscribe <SystemMessage.SystemInit>(requestManagement);
_mainBus.Subscribe <ClientMessage.WriteEvents>(requestManagement);
_mainBus.Subscribe <ClientMessage.TransactionStart>(requestManagement);
_mainBus.Subscribe <ClientMessage.TransactionWrite>(requestManagement);
_mainBus.Subscribe <ClientMessage.TransactionCommit>(requestManagement);
_mainBus.Subscribe <ClientMessage.DeleteStream>(requestManagement);
_mainBus.Subscribe <StorageMessage.RequestCompleted>(requestManagement);
_mainBus.Subscribe <StorageMessage.CheckStreamAccessCompleted>(requestManagement);
_mainBus.Subscribe <StorageMessage.AlreadyCommitted>(requestManagement);
_mainBus.Subscribe <StorageMessage.CommitAck>(requestManagement);
_mainBus.Subscribe <StorageMessage.PrepareAck>(requestManagement);
_mainBus.Subscribe <StorageMessage.WrongExpectedVersion>(requestManagement);
_mainBus.Subscribe <StorageMessage.InvalidTransaction>(requestManagement);
_mainBus.Subscribe <StorageMessage.StreamDeleted>(requestManagement);
_mainBus.Subscribe <StorageMessage.RequestManagerTimerTick>(requestManagement);
// SUBSCRIPTIONS
var subscrBus = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", false);
_mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.SystemStart, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <TcpMessage.ConnectionClosed, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.SubscribeToStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.UnsubscribeFromStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.PollStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.CheckPollTimeout, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <StorageMessage.EventCommitted, Message>());
var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex);
subscrBus.Subscribe <SystemMessage.SystemStart>(subscription);
subscrBus.Subscribe <SystemMessage.BecomeShuttingDown>(subscription);
subscrBus.Subscribe <TcpMessage.ConnectionClosed>(subscription);
subscrBus.Subscribe <ClientMessage.SubscribeToStream>(subscription);
subscrBus.Subscribe <ClientMessage.UnsubscribeFromStream>(subscription);
subscrBus.Subscribe <SubscriptionMessage.PollStream>(subscription);
subscrBus.Subscribe <SubscriptionMessage.CheckPollTimeout>(subscription);
subscrBus.Subscribe <StorageMessage.EventCommitted>(subscription);
// TIMER
_timeProvider = new RealTimeProvider();
_timerService = new TimerService(new ThreadBasedScheduler(_timeProvider));
_mainBus.Subscribe <SystemMessage.BecomeShutdown>(_timerService);
_mainBus.Subscribe <TimerMessage.Schedule>(_timerService);
// MASTER REPLICATION
var masterReplicationService = new MasterReplicationService(_mainQueue, _nodeInfo.InstanceId, db, _workersHandler,
epochManager, vNodeSettings.ClusterNodeCount);
_mainBus.Subscribe <SystemMessage.SystemStart>(masterReplicationService);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(masterReplicationService);
_mainBus.Subscribe <ReplicationMessage.ReplicaSubscriptionRequest>(masterReplicationService);
_mainBus.Subscribe <ReplicationMessage.ReplicaLogPositionAck>(masterReplicationService);
// REPLICA REPLICATION
var replicaService = new ReplicaService(_mainQueue, db, epochManager, _workersHandler, authenticationProvider,
_nodeInfo, vNodeSettings.UseSsl, vNodeSettings.SslTargetHost, vNodeSettings.SslValidateServer);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(replicaService);
_mainBus.Subscribe <ReplicationMessage.ReconnectToMaster>(replicaService);
_mainBus.Subscribe <ReplicationMessage.SubscribeToMaster>(replicaService);
_mainBus.Subscribe <ReplicationMessage.AckLogPosition>(replicaService);
_mainBus.Subscribe <StorageMessage.PrepareAck>(replicaService);
_mainBus.Subscribe <StorageMessage.CommitAck>(replicaService);
_mainBus.Subscribe <ClientMessage.TcpForwardMessage>(replicaService);
// ELECTIONS
var electionsService = new ElectionsService(_mainQueue, _nodeInfo, vNodeSettings.ClusterNodeCount,
db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint,
epochManager, () => readIndex.LastCommitPosition, vNodeSettings.NodePriority);
electionsService.SubscribeMessages(_mainBus);
// GOSSIP
var gossip = new NodeGossipService(_mainQueue, gossipSeedSource, _nodeInfo, db.Config.WriterCheckpoint,
db.Config.ChaserCheckpoint, epochManager, () => readIndex.LastCommitPosition,
vNodeSettings.NodePriority);
_mainBus.Subscribe <SystemMessage.SystemInit>(gossip);
_mainBus.Subscribe <GossipMessage.RetrieveGossipSeedSources>(gossip);
_mainBus.Subscribe <GossipMessage.GotGossipSeedSources>(gossip);
_mainBus.Subscribe <GossipMessage.Gossip>(gossip);
_mainBus.Subscribe <GossipMessage.GossipReceived>(gossip);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(gossip);
_mainBus.Subscribe <GossipMessage.GossipSendFailed>(gossip);
_mainBus.Subscribe <SystemMessage.VNodeConnectionEstablished>(gossip);
_mainBus.Subscribe <SystemMessage.VNodeConnectionLost>(gossip);
_workersHandler.Start();
_mainQueue.Start();
monitoringQueue.Start();
subscrQueue.Start();
if (subsystems != null)
{
foreach (var subsystem in subsystems)
{
subsystem.Register(db, _mainQueue, _mainBus, _timerService, _timeProvider,
httpSendService, new[] { _internalHttpService, _externalHttpService }, _workersHandler);
}
}
}
public PlaygroundVNode(PlaygroundVNodeSettings vNodeSettings)
{
_tcpEndPoint = vNodeSettings.ExternalTcpEndPoint;
_httpEndPoint = vNodeSettings.ExternalHttpEndPoint;
_mainBus = new InMemoryBus("MainBus");
_controller = new PlaygroundVNodeController(Bus, _httpEndPoint);
_mainQueue = new QueuedHandler(_controller, "MainQueue");
_controller.SetMainQueue(MainQueue);
// MONITORING
var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
var monitoringQueue = new QueuedHandler(
monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100));
Bus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.SystemInit, Message>());
Bus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.StateChangeMessage, Message>());
Bus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
// MISC WORKERS
_workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(50))).ToArray();
_workersHandler = new MultiQueuedHandler(
vNodeSettings.WorkerThreads,
queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
string.Format("Worker #{0}", queueNum + 1),
groupName: "Workers",
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(50)));
// AUTHENTICATION INFRASTRUCTURE
var dispatcher = new IODispatcher(_mainBus, new PublishEnvelope(_workersHandler, crossThread: true));
var passwordHashAlgorithm = new Rfc2898PasswordHashAlgorithm();
var internalAuthenticationProvider = new InternalAuthenticationProvider(dispatcher, passwordHashAlgorithm, 1000);
var passwordChangeNotificationReader = new PasswordChangeNotificationReader(_mainQueue, dispatcher);
_mainBus.Subscribe <SystemMessage.SystemStart>(passwordChangeNotificationReader);
_mainBus.Subscribe <SystemMessage.BecomeShutdown>(passwordChangeNotificationReader);
_mainBus.Subscribe(internalAuthenticationProvider);
_mainBus.Subscribe(dispatcher);
SubscribeWorkers(bus =>
{
bus.Subscribe(dispatcher.ForwardReader);
bus.Subscribe(dispatcher.BackwardReader);
bus.Subscribe(dispatcher.Writer);
bus.Subscribe(dispatcher.StreamDeleter);
});
// TCP
var tcpService = new TcpService(
MainQueue, _tcpEndPoint, _workersHandler, TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(),
ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout, internalAuthenticationProvider, null);
Bus.Subscribe <SystemMessage.SystemInit>(tcpService);
Bus.Subscribe <SystemMessage.SystemStart>(tcpService);
Bus.Subscribe <SystemMessage.BecomeShuttingDown>(tcpService);
// HTTP
{
var authenticationProviders = new AuthenticationProvider[]
{
new BasicHttpAuthenticationProvider(internalAuthenticationProvider),
new TrustedAuthenticationProvider(),
new AnonymousAuthenticationProvider()
};
var httpPipe = new HttpMessagePipe();
var httpSendService = new HttpSendService(httpPipe, forwardRequests: false);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(httpSendService);
_mainBus.Subscribe(new WideningHandler <HttpMessage.SendOverHttp, Message>(_workersHandler));
SubscribeWorkers(bus =>
{
bus.Subscribe <HttpMessage.HttpSend>(httpSendService);
bus.Subscribe <HttpMessage.HttpSendPart>(httpSendService);
bus.Subscribe <HttpMessage.HttpBeginSend>(httpSendService);
bus.Subscribe <HttpMessage.HttpEndSend>(httpSendService);
bus.Subscribe <HttpMessage.SendOverHttp>(httpSendService);
});
_httpService = new HttpService(ServiceAccessibility.Private, _mainQueue, new TrieUriRouter(),
_workersHandler, vNodeSettings.HttpPrefixes);
_mainBus.Subscribe <SystemMessage.SystemInit>(_httpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_httpService);
_mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_httpService);
HttpService.SetupController(new AdminController(_mainQueue));
HttpService.SetupController(new PingController());
HttpService.SetupController(new StatController(monitoringQueue, _workersHandler));
HttpService.SetupController(new AtomController(httpSendService, _mainQueue, _workersHandler));
HttpService.SetupController(new GuidController(_mainQueue));
HttpService.SetupController(new UsersController(httpSendService, _mainQueue, _workersHandler));
SubscribeWorkers(bus => HttpService.CreateAndSubscribePipeline(bus, authenticationProviders));
}
// TIMER
_timerService = new TimerService(new ThreadBasedScheduler(new RealTimeProvider()));
Bus.Subscribe(TimerService);
monitoringQueue.Start();
MainQueue.Start();
}
public ClusterVNode(TFChunkDb db,
ClusterVNodeSettings vNodeSettings,
IGossipSeedSource gossipSeedSource,
bool dbVerifyHashes,
int memTableEntryCount,
params ISubsystem[] subsystems)
{
Ensure.NotNull(db, "db");
Ensure.NotNull(vNodeSettings, "vNodeSettings");
Ensure.NotNull(gossipSeedSource, "gossipSeedSource");
_nodeInfo = vNodeSettings.NodeInfo;
_mainBus = new InMemoryBus("MainBus");
var forwardingProxy = new MessageForwardingProxy();
// MISC WORKERS
_workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray();
_workersHandler = new MultiQueuedHandler(
vNodeSettings.WorkerThreads,
queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
string.Format("Worker #{0}", queueNum + 1),
groupName: "Workers",
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200)));
_controller = new ClusterVNodeController(_mainBus, _nodeInfo, db, vNodeSettings, this, forwardingProxy);
_mainQueue = new QueuedHandler(_controller, "MainQueue");
_controller.SetMainQueue(_mainQueue);
_subsystems = subsystems;
// MONITORING
var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
var monitoringQueue = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100));
var monitoring = new MonitoringService(monitoringQueue,
monitoringRequestBus,
_mainQueue,
db.Config.WriterCheckpoint,
db.Config.Path,
vNodeSettings.StatsPeriod,
_nodeInfo.ExternalHttp,
vNodeSettings.StatsStorage);
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.SystemInit, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.StateChangeMessage, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShutdown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<ClientMessage.WriteEventsCompleted, Message>());
monitoringInnerBus.Subscribe<SystemMessage.SystemInit>(monitoring);
monitoringInnerBus.Subscribe<SystemMessage.StateChangeMessage>(monitoring);
monitoringInnerBus.Subscribe<SystemMessage.BecomeShuttingDown>(monitoring);
monitoringInnerBus.Subscribe<SystemMessage.BecomeShutdown>(monitoring);
monitoringInnerBus.Subscribe<ClientMessage.WriteEventsCompleted>(monitoring);
monitoringInnerBus.Subscribe<MonitoringMessage.GetFreshStats>(monitoring);
var truncPos = db.Config.TruncateCheckpoint.Read();
if (truncPos != -1)
{
Log.Info("Truncate checkpoint is present. Truncate: {0} (0x{0:X}), Writer: {1} (0x{1:X}), Chaser: {2} (0x{2:X}), Epoch: {3} (0x{3:X})",
truncPos, db.Config.WriterCheckpoint.Read(), db.Config.ChaserCheckpoint.Read(), db.Config.EpochCheckpoint.Read());
var truncator = new TFChunkDbTruncator(db.Config);
truncator.TruncateDb(truncPos);
}
// STORAGE SUBSYSTEM
db.Open(dbVerifyHashes);
var indexPath = Path.Combine(db.Config.Path, "index");
var readerPool = new ObjectPool<ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(db, db.Config.WriterCheckpoint));
var tableIndex = new TableIndex(indexPath,
() => new HashListMemTable(maxSize: memTableEntryCount * 2),
() => new TFReaderLease(readerPool),
maxSizeForMemory: memTableEntryCount,
maxTablesPerLevel: 2,
inMem: db.Config.InMemDb);
var hash = new XXHashUnsafe();
var readIndex = new ReadIndex(_mainQueue,
readerPool,
tableIndex,
hash,
ESConsts.StreamInfoCacheCapacity,
Application.IsDefined(Application.AdditionalCommitChecks),
Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1);
var writer = new TFChunkWriter(db);
var epochManager = new EpochManager(ESConsts.CachedEpochCount,
db.Config.EpochCheckpoint,
writer,
initialReaderCount: 1,
maxReaderCount: 5,
readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint));
epochManager.Init();
var storageWriter = new ClusterStorageWriterService(_mainQueue, _mainBus, vNodeSettings.MinFlushDelay,
db, writer, readIndex.IndexWriter, epochManager,
() => readIndex.LastCommitPosition); // subscribes internally
monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageWriter);
var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, ESConsts.StorageReaderThreadCount, db.Config.WriterCheckpoint);
_mainBus.Subscribe<SystemMessage.SystemInit>(storageReader);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageReader);
_mainBus.Subscribe<SystemMessage.BecomeShutdown>(storageReader);
monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageReader);
var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, readIndex.IndexCommitter, epochManager);
_mainBus.Subscribe<SystemMessage.SystemInit>(storageChaser);
_mainBus.Subscribe<SystemMessage.SystemStart>(storageChaser);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageChaser);
var storageScavenger = new StorageScavenger(db, tableIndex, hash, readIndex,
Application.IsDefined(Application.AlwaysKeepScavenged),
mergeChunks: !vNodeSettings.DisableScavengeMerging);
// ReSharper disable RedundantTypeArgumentsOfMethod
_mainBus.Subscribe<ClientMessage.ScavengeDatabase>(storageScavenger);
// ReSharper restore RedundantTypeArgumentsOfMethod
// AUTHENTICATION INFRASTRUCTURE - delegate to plugins
var authenticationProvider = vNodeSettings.AuthenticationProviderFactory.BuildAuthenticationProvider(_mainQueue, _mainBus, _workersHandler, _workerBuses);
Ensure.NotNull(authenticationProvider, "authenticationProvider");
{
// EXTERNAL TCP
var extTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalTcp, _workersHandler,
TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(),
ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
authenticationProvider, null);
_mainBus.Subscribe<SystemMessage.SystemInit>(extTcpService);
_mainBus.Subscribe<SystemMessage.SystemStart>(extTcpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(extTcpService);
// EXTERNAL SECURE TCP
if (_nodeInfo.ExternalSecureTcp != null)
{
var extSecTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalSecureTcp, _workersHandler,
TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(),
ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
authenticationProvider, vNodeSettings.Certificate);
_mainBus.Subscribe<SystemMessage.SystemInit>(extSecTcpService);
_mainBus.Subscribe<SystemMessage.SystemStart>(extSecTcpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(extSecTcpService);
}
// INTERNAL TCP
var intTcpService = new TcpService(_mainQueue, _nodeInfo.InternalTcp, _workersHandler,
TcpServiceType.Internal, TcpSecurityType.Normal,
new InternalTcpDispatcher(),
ESConsts.InternalHeartbeatInterval, ESConsts.InternalHeartbeatTimeout,
authenticationProvider, null);
_mainBus.Subscribe<SystemMessage.SystemInit>(intTcpService);
_mainBus.Subscribe<SystemMessage.SystemStart>(intTcpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(intTcpService);
// INTERNAL SECURE TCP
if (_nodeInfo.InternalSecureTcp != null)
{
var intSecTcpService = new TcpService(_mainQueue, _nodeInfo.InternalSecureTcp, _workersHandler,
TcpServiceType.Internal, TcpSecurityType.Secure,
new InternalTcpDispatcher(),
ESConsts.InternalHeartbeatInterval, ESConsts.InternalHeartbeatTimeout,
authenticationProvider, vNodeSettings.Certificate);
_mainBus.Subscribe<SystemMessage.SystemInit>(intSecTcpService);
_mainBus.Subscribe<SystemMessage.SystemStart>(intSecTcpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(intSecTcpService);
}
}
SubscribeWorkers(bus =>
{
var tcpSendService = new TcpSendService();
// ReSharper disable RedundantTypeArgumentsOfMethod
bus.Subscribe<TcpMessage.TcpSend>(tcpSendService);
// ReSharper restore RedundantTypeArgumentsOfMethod
});
var httpAuthenticationProviders = new List<HttpAuthenticationProvider>
{
new BasicHttpAuthenticationProvider(authenticationProvider),
};
if (vNodeSettings.EnableTrustedAuth)
httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider());
httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider());
var httpPipe = new HttpMessagePipe();
var httpSendService = new HttpSendService(httpPipe, forwardRequests: true);
_mainBus.Subscribe<SystemMessage.StateChangeMessage>(httpSendService);
_mainBus.Subscribe(new WideningHandler<HttpMessage.SendOverHttp, Message>(_workersHandler));
SubscribeWorkers(bus =>
{
bus.Subscribe<HttpMessage.HttpSend>(httpSendService);
bus.Subscribe<HttpMessage.HttpSendPart>(httpSendService);
bus.Subscribe<HttpMessage.HttpBeginSend>(httpSendService);
bus.Subscribe<HttpMessage.HttpEndSend>(httpSendService);
bus.Subscribe<HttpMessage.SendOverHttp>(httpSendService);
});
var adminController = new AdminController(_mainQueue);
var pingController = new PingController();
var statController = new StatController(monitoringQueue, _workersHandler);
var atomController = new AtomController(httpSendService, _mainQueue, _workersHandler);
var gossipController = new GossipController(_mainQueue, _workersHandler);
var electController = new ElectController(_mainQueue);
// HTTP SENDERS
gossipController.SubscribeSenders(httpPipe);
electController.SubscribeSenders(httpPipe);
// EXTERNAL HTTP
_externalHttpService = new HttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(),
_workersHandler, vNodeSettings.HttpPrefixes);
_externalHttpService.SetupController(adminController);
_externalHttpService.SetupController(pingController);
_externalHttpService.SetupController(statController);
_externalHttpService.SetupController(atomController);
_externalHttpService.SetupController(gossipController);
_mainBus.Subscribe<SystemMessage.SystemInit>(_externalHttpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(_externalHttpService);
_mainBus.Subscribe<HttpMessage.PurgeTimedOutRequests>(_externalHttpService);
// INTERNAL HTTP
_internalHttpService = new HttpService(ServiceAccessibility.Private, _mainQueue, new TrieUriRouter(),
_workersHandler, _nodeInfo.InternalHttp.ToHttpUrl());
_internalHttpService.SetupController(adminController);
_internalHttpService.SetupController(pingController);
_internalHttpService.SetupController(statController);
_internalHttpService.SetupController(atomController);
_internalHttpService.SetupController(gossipController);
_internalHttpService.SetupController(electController);
// Authentication plugin HTTP
vNodeSettings.AuthenticationProviderFactory.RegisterHttpControllers(_externalHttpService, _internalHttpService, httpSendService, _mainQueue, _workersHandler);
_mainBus.Subscribe<SystemMessage.SystemInit>(_internalHttpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(_internalHttpService);
_mainBus.Subscribe<HttpMessage.PurgeTimedOutRequests>(_internalHttpService);
SubscribeWorkers(bus =>
{
HttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders.ToArray());
});
// REQUEST FORWARDING
var forwardingService = new RequestForwardingService(_mainQueue, forwardingProxy, TimeSpan.FromSeconds(1));
_mainBus.Subscribe<SystemMessage.SystemStart>(forwardingService);
_mainBus.Subscribe<SystemMessage.RequestForwardingTimerTick>(forwardingService);
_mainBus.Subscribe<ClientMessage.NotHandled>(forwardingService);
_mainBus.Subscribe<ClientMessage.WriteEventsCompleted>(forwardingService);
_mainBus.Subscribe<ClientMessage.TransactionStartCompleted>(forwardingService);
_mainBus.Subscribe<ClientMessage.TransactionWriteCompleted>(forwardingService);
_mainBus.Subscribe<ClientMessage.TransactionCommitCompleted>(forwardingService);
_mainBus.Subscribe<ClientMessage.DeleteStreamCompleted>(forwardingService);
// REQUEST MANAGEMENT
var requestManagement = new RequestManagementService(_mainQueue,
vNodeSettings.PrepareAckCount,
vNodeSettings.CommitAckCount,
vNodeSettings.PrepareTimeout,
vNodeSettings.CommitTimeout);
_mainBus.Subscribe<SystemMessage.SystemInit>(requestManagement);
_mainBus.Subscribe<ClientMessage.WriteEvents>(requestManagement);
_mainBus.Subscribe<ClientMessage.TransactionStart>(requestManagement);
_mainBus.Subscribe<ClientMessage.TransactionWrite>(requestManagement);
_mainBus.Subscribe<ClientMessage.TransactionCommit>(requestManagement);
_mainBus.Subscribe<ClientMessage.DeleteStream>(requestManagement);
_mainBus.Subscribe<StorageMessage.RequestCompleted>(requestManagement);
_mainBus.Subscribe<StorageMessage.CheckStreamAccessCompleted>(requestManagement);
_mainBus.Subscribe<StorageMessage.AlreadyCommitted>(requestManagement);
_mainBus.Subscribe<StorageMessage.CommitAck>(requestManagement);
_mainBus.Subscribe<StorageMessage.PrepareAck>(requestManagement);
_mainBus.Subscribe<StorageMessage.WrongExpectedVersion>(requestManagement);
_mainBus.Subscribe<StorageMessage.InvalidTransaction>(requestManagement);
_mainBus.Subscribe<StorageMessage.StreamDeleted>(requestManagement);
_mainBus.Subscribe<StorageMessage.RequestManagerTimerTick>(requestManagement);
// SUBSCRIPTIONS
var subscrBus = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", false);
_mainBus.Subscribe(subscrQueue.WidenFrom<SystemMessage.SystemStart, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<TcpMessage.ConnectionClosed, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<ClientMessage.SubscribeToStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<ClientMessage.UnsubscribeFromStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<SubscriptionMessage.PollStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<SubscriptionMessage.CheckPollTimeout, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<StorageMessage.EventCommited, Message>());
var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex);
subscrBus.Subscribe<SystemMessage.SystemStart>(subscription);
subscrBus.Subscribe<SystemMessage.BecomeShuttingDown>(subscription);
subscrBus.Subscribe<TcpMessage.ConnectionClosed>(subscription);
subscrBus.Subscribe<ClientMessage.SubscribeToStream>(subscription);
subscrBus.Subscribe<ClientMessage.UnsubscribeFromStream>(subscription);
subscrBus.Subscribe<SubscriptionMessage.PollStream>(subscription);
subscrBus.Subscribe<SubscriptionMessage.CheckPollTimeout>(subscription);
subscrBus.Subscribe<StorageMessage.EventCommited>(subscription);
// TIMER
_timeProvider = new RealTimeProvider();
_timerService = new TimerService(new ThreadBasedScheduler(_timeProvider));
_mainBus.Subscribe<SystemMessage.BecomeShutdown>(_timerService);
_mainBus.Subscribe<TimerMessage.Schedule>(_timerService);
// MASTER REPLICATION
var masterReplicationService = new MasterReplicationService(_mainQueue, _nodeInfo.InstanceId, db, _workersHandler,
epochManager, vNodeSettings.ClusterNodeCount);
_mainBus.Subscribe<SystemMessage.SystemStart>(masterReplicationService);
_mainBus.Subscribe<SystemMessage.StateChangeMessage>(masterReplicationService);
_mainBus.Subscribe<ReplicationMessage.ReplicaSubscriptionRequest>(masterReplicationService);
_mainBus.Subscribe<ReplicationMessage.ReplicaLogPositionAck>(masterReplicationService);
// REPLICA REPLICATION
var replicaService = new ReplicaService(_mainQueue, db, epochManager, _workersHandler, authenticationProvider,
_nodeInfo, vNodeSettings.UseSsl, vNodeSettings.SslTargetHost, vNodeSettings.SslValidateServer);
_mainBus.Subscribe<SystemMessage.StateChangeMessage>(replicaService);
_mainBus.Subscribe<ReplicationMessage.ReconnectToMaster>(replicaService);
_mainBus.Subscribe<ReplicationMessage.SubscribeToMaster>(replicaService);
_mainBus.Subscribe<ReplicationMessage.AckLogPosition>(replicaService);
_mainBus.Subscribe<StorageMessage.PrepareAck>(replicaService);
_mainBus.Subscribe<StorageMessage.CommitAck>(replicaService);
_mainBus.Subscribe<ClientMessage.TcpForwardMessage>(replicaService);
// ELECTIONS
var electionsService = new ElectionsService(_mainQueue, _nodeInfo, vNodeSettings.ClusterNodeCount,
db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint,
epochManager, () => readIndex.LastCommitPosition, vNodeSettings.NodePriority);
electionsService.SubscribeMessages(_mainBus);
// GOSSIP
var gossip = new NodeGossipService(_mainQueue, gossipSeedSource, _nodeInfo, db.Config.WriterCheckpoint,
db.Config.ChaserCheckpoint, epochManager, () => readIndex.LastCommitPosition,
vNodeSettings.NodePriority);
_mainBus.Subscribe<SystemMessage.SystemInit>(gossip);
_mainBus.Subscribe<GossipMessage.RetrieveGossipSeedSources>(gossip);
_mainBus.Subscribe<GossipMessage.GotGossipSeedSources>(gossip);
_mainBus.Subscribe<GossipMessage.Gossip>(gossip);
_mainBus.Subscribe<GossipMessage.GossipReceived>(gossip);
_mainBus.Subscribe<SystemMessage.StateChangeMessage>(gossip);
_mainBus.Subscribe<GossipMessage.GossipSendFailed>(gossip);
_mainBus.Subscribe<SystemMessage.VNodeConnectionEstablished>(gossip);
_mainBus.Subscribe<SystemMessage.VNodeConnectionLost>(gossip);
_workersHandler.Start();
_mainQueue.Start();
monitoringQueue.Start();
subscrQueue.Start();
if (subsystems != null)
{
foreach (var subsystem in subsystems)
{
subsystem.Register(db, _mainQueue, _mainBus, _timerService, _timeProvider,
httpSendService, new[] { _internalHttpService, _externalHttpService }, _workersHandler);
}
}
}
public ClusterVNode(TFChunkDb db,
ClusterVNodeSettings vNodeSettings,
IGossipSeedSource gossipSeedSource,
InfoController infoController,
params ISubsystem[] subsystems)
{
Ensure.NotNull(db, "db");
Ensure.NotNull(vNodeSettings, "vNodeSettings");
Ensure.NotNull(gossipSeedSource, "gossipSeedSource");
#if DEBUG
AddTask(_taskAddedTrigger.Task);
#endif
var isSingleNode = vNodeSettings.ClusterNodeCount == 1;
_nodeInfo = vNodeSettings.NodeInfo;
_mainBus = new InMemoryBus("MainBus");
var forwardingProxy = new MessageForwardingProxy();
if (vNodeSettings.EnableHistograms)
{
HistogramService.CreateHistograms();
//start watching jitter
HistogramService.StartJitterMonitor();
}
// MISC WORKERS
_workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray();
_workersHandler = new MultiQueuedHandler(
vNodeSettings.WorkerThreads,
queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
string.Format("Worker #{0}", queueNum + 1),
groupName: "Workers",
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200)));
_subsystems = subsystems;
_controller = new ClusterVNodeController((IPublisher)_mainBus, _nodeInfo, db, vNodeSettings, this, forwardingProxy, _subsystems);
_mainQueue = QueuedHandler.CreateQueuedHandler(_controller, "MainQueue");
_controller.SetMainQueue(_mainQueue);
//SELF
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(this);
_mainBus.Subscribe <SystemMessage.BecomeShutdown>(this);
// MONITORING
var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
var monitoringQueue = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100));
var monitoring = new MonitoringService(monitoringQueue,
monitoringRequestBus,
_mainQueue,
db.Config.WriterCheckpoint,
db.Config.Path,
vNodeSettings.StatsPeriod,
_nodeInfo.ExternalHttp,
vNodeSettings.StatsStorage,
_nodeInfo.ExternalTcp,
_nodeInfo.ExternalSecureTcp);
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.SystemInit, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.StateChangeMessage, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShutdown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <ClientMessage.WriteEventsCompleted, Message>());
monitoringInnerBus.Subscribe <SystemMessage.SystemInit>(monitoring);
monitoringInnerBus.Subscribe <SystemMessage.StateChangeMessage>(monitoring);
monitoringInnerBus.Subscribe <SystemMessage.BecomeShuttingDown>(monitoring);
monitoringInnerBus.Subscribe <SystemMessage.BecomeShutdown>(monitoring);
monitoringInnerBus.Subscribe <ClientMessage.WriteEventsCompleted>(monitoring);
monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshStats>(monitoring);
monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshTcpConnectionStats>(monitoring);
var truncPos = db.Config.TruncateCheckpoint.Read();
if (truncPos != -1)
{
Log.Info("Truncate checkpoint is present. Truncate: {0} (0x{0:X}), Writer: {1} (0x{1:X}), Chaser: {2} (0x{2:X}), Epoch: {3} (0x{3:X})",
truncPos, db.Config.WriterCheckpoint.Read(), db.Config.ChaserCheckpoint.Read(), db.Config.EpochCheckpoint.Read());
var truncator = new TFChunkDbTruncator(db.Config);
truncator.TruncateDb(truncPos);
}
// STORAGE SUBSYSTEM
db.Open(vNodeSettings.VerifyDbHash);
var indexPath = vNodeSettings.Index ?? Path.Combine(db.Config.Path, "index");
var readerPool = new ObjectPool <ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(db, db.Config.WriterCheckpoint, optimizeReadSideCache: db.Config.OptimizeReadSideCache));
var tableIndex = new TableIndex(indexPath,
new XXHashUnsafe(),
new Murmur3AUnsafe(),
() => new HashListMemTable(vNodeSettings.IndexBitnessVersion, maxSize: vNodeSettings.MaxMemtableEntryCount * 2),
() => new TFReaderLease(readerPool),
vNodeSettings.IndexBitnessVersion,
maxSizeForMemory: vNodeSettings.MaxMemtableEntryCount,
maxTablesPerLevel: 2,
inMem: db.Config.InMemDb,
skipIndexVerify: vNodeSettings.SkipIndexVerify,
indexCacheDepth: vNodeSettings.IndexCacheDepth);
var readIndex = new ReadIndex(_mainQueue,
readerPool,
tableIndex,
ESConsts.StreamInfoCacheCapacity,
Application.IsDefined(Application.AdditionalCommitChecks),
Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1,
vNodeSettings.HashCollisionReadLimit,
vNodeSettings.SkipIndexScanOnReads,
db.Config.ReplicationCheckpoint);
var writer = new TFChunkWriter(db);
var epochManager = new EpochManager(_mainQueue,
ESConsts.CachedEpochCount,
db.Config.EpochCheckpoint,
writer,
initialReaderCount: 1,
maxReaderCount: 5,
readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint, optimizeReadSideCache: db.Config.OptimizeReadSideCache));
epochManager.Init();
var storageWriter = new ClusterStorageWriterService(_mainQueue, _mainBus, vNodeSettings.MinFlushDelay,
db, writer, readIndex.IndexWriter, epochManager,
() => readIndex.LastCommitPosition); // subscribes internally
AddTasks(storageWriter.Tasks);
monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageWriter);
var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, vNodeSettings.ReaderThreadsCount, db.Config.WriterCheckpoint);
_mainBus.Subscribe <SystemMessage.SystemInit>(storageReader);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageReader);
_mainBus.Subscribe <SystemMessage.BecomeShutdown>(storageReader);
monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageReader);
var indexCommitterService = new IndexCommitterService(readIndex.IndexCommitter, _mainQueue, db.Config.ReplicationCheckpoint, db.Config.WriterCheckpoint, vNodeSettings.CommitAckCount);
AddTask(indexCommitterService.Task);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(indexCommitterService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(indexCommitterService);
_mainBus.Subscribe <StorageMessage.CommitAck>(indexCommitterService);
var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint, db.Config.OptimizeReadSideCache);
var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, indexCommitterService, epochManager);
AddTask(storageChaser.Task);
#if DEBUG
QueueStatsCollector.InitializeCheckpoints(
_nodeInfo.DebugIndex, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
#endif
_mainBus.Subscribe <SystemMessage.SystemInit>(storageChaser);
_mainBus.Subscribe <SystemMessage.SystemStart>(storageChaser);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageChaser);
// AUTHENTICATION INFRASTRUCTURE - delegate to plugins
_internalAuthenticationProvider = vNodeSettings.AuthenticationProviderFactory.BuildAuthenticationProvider(_mainQueue, _mainBus, _workersHandler, _workerBuses);
Ensure.NotNull(_internalAuthenticationProvider, "authenticationProvider");
{
// EXTERNAL TCP
if (!vNodeSettings.DisableInsecureTCP)
{
var extTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalTcp, _workersHandler,
TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(),
vNodeSettings.ExtTcpHeartbeatInterval, vNodeSettings.ExtTcpHeartbeatTimeout,
_internalAuthenticationProvider, null, vNodeSettings.ConnectionPendingSendBytesThreshold);
_mainBus.Subscribe <SystemMessage.SystemInit>(extTcpService);
_mainBus.Subscribe <SystemMessage.SystemStart>(extTcpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(extTcpService);
}
// EXTERNAL SECURE TCP
if (_nodeInfo.ExternalSecureTcp != null)
{
var extSecTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalSecureTcp, _workersHandler,
TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(),
vNodeSettings.ExtTcpHeartbeatInterval, vNodeSettings.ExtTcpHeartbeatTimeout,
_internalAuthenticationProvider, vNodeSettings.Certificate, vNodeSettings.ConnectionPendingSendBytesThreshold);
_mainBus.Subscribe <SystemMessage.SystemInit>(extSecTcpService);
_mainBus.Subscribe <SystemMessage.SystemStart>(extSecTcpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(extSecTcpService);
}
if (!isSingleNode)
{
// INTERNAL TCP
if (!vNodeSettings.DisableInsecureTCP)
{
var intTcpService = new TcpService(_mainQueue, _nodeInfo.InternalTcp, _workersHandler,
TcpServiceType.Internal, TcpSecurityType.Normal,
new InternalTcpDispatcher(),
vNodeSettings.IntTcpHeartbeatInterval, vNodeSettings.IntTcpHeartbeatTimeout,
_internalAuthenticationProvider, null, ESConsts.UnrestrictedPendingSendBytes);
_mainBus.Subscribe <SystemMessage.SystemInit>(intTcpService);
_mainBus.Subscribe <SystemMessage.SystemStart>(intTcpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(intTcpService);
}
// INTERNAL SECURE TCP
if (_nodeInfo.InternalSecureTcp != null)
{
var intSecTcpService = new TcpService(_mainQueue, _nodeInfo.InternalSecureTcp, _workersHandler,
TcpServiceType.Internal, TcpSecurityType.Secure,
new InternalTcpDispatcher(),
vNodeSettings.IntTcpHeartbeatInterval, vNodeSettings.IntTcpHeartbeatTimeout,
_internalAuthenticationProvider, vNodeSettings.Certificate, ESConsts.UnrestrictedPendingSendBytes);
_mainBus.Subscribe <SystemMessage.SystemInit>(intSecTcpService);
_mainBus.Subscribe <SystemMessage.SystemStart>(intSecTcpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(intSecTcpService);
}
}
}
SubscribeWorkers(bus =>
{
var tcpSendService = new TcpSendService();
// ReSharper disable RedundantTypeArgumentsOfMethod
bus.Subscribe <TcpMessage.TcpSend>(tcpSendService);
// ReSharper restore RedundantTypeArgumentsOfMethod
});
var httpAuthenticationProviders = new List <HttpAuthenticationProvider>
{
new BasicHttpAuthenticationProvider(_internalAuthenticationProvider),
};
if (vNodeSettings.EnableTrustedAuth)
{
httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider());
}
httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider());
var httpPipe = new HttpMessagePipe();
var httpSendService = new HttpSendService(httpPipe, forwardRequests: true);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(httpSendService);
_mainBus.Subscribe(new WideningHandler <HttpMessage.SendOverHttp, Message>(_workersHandler));
SubscribeWorkers(bus =>
{
bus.Subscribe <HttpMessage.HttpSend>(httpSendService);
bus.Subscribe <HttpMessage.HttpSendPart>(httpSendService);
bus.Subscribe <HttpMessage.HttpBeginSend>(httpSendService);
bus.Subscribe <HttpMessage.HttpEndSend>(httpSendService);
bus.Subscribe <HttpMessage.SendOverHttp>(httpSendService);
});
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(infoController);
var adminController = new AdminController(_mainQueue, _workersHandler);
var pingController = new PingController();
var histogramController = new HistogramController();
var statController = new StatController(monitoringQueue, _workersHandler);
var atomController = new AtomController(httpSendService, _mainQueue, _workersHandler, vNodeSettings.DisableHTTPCaching);
var gossipController = new GossipController(_mainQueue, _workersHandler, vNodeSettings.GossipTimeout);
var persistentSubscriptionController = new PersistentSubscriptionController(httpSendService, _mainQueue, _workersHandler);
var electController = new ElectController(_mainQueue);
// HTTP SENDERS
gossipController.SubscribeSenders(httpPipe);
electController.SubscribeSenders(httpPipe);
// EXTERNAL HTTP
_externalHttpService = new HttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(),
_workersHandler, vNodeSettings.LogHttpRequests, vNodeSettings.GossipAdvertiseInfo.AdvertiseExternalIPAs,
vNodeSettings.GossipAdvertiseInfo.AdvertiseExternalHttpPortAs, vNodeSettings.ExtHttpPrefixes);
_externalHttpService.SetupController(persistentSubscriptionController);
if (vNodeSettings.AdminOnPublic)
{
_externalHttpService.SetupController(adminController);
}
_externalHttpService.SetupController(pingController);
_externalHttpService.SetupController(infoController);
if (vNodeSettings.StatsOnPublic)
{
_externalHttpService.SetupController(statController);
}
_externalHttpService.SetupController(atomController);
if (vNodeSettings.GossipOnPublic)
{
_externalHttpService.SetupController(gossipController);
}
_externalHttpService.SetupController(histogramController);
_mainBus.Subscribe <SystemMessage.SystemInit>(_externalHttpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_externalHttpService);
_mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_externalHttpService);
// INTERNAL HTTP
if (!isSingleNode)
{
_internalHttpService = new HttpService(ServiceAccessibility.Private, _mainQueue, new TrieUriRouter(),
_workersHandler, vNodeSettings.LogHttpRequests, vNodeSettings.GossipAdvertiseInfo.AdvertiseInternalIPAs,
vNodeSettings.GossipAdvertiseInfo.AdvertiseInternalHttpPortAs, vNodeSettings.IntHttpPrefixes);
_internalHttpService.SetupController(adminController);
_internalHttpService.SetupController(pingController);
_internalHttpService.SetupController(infoController);
_internalHttpService.SetupController(statController);
_internalHttpService.SetupController(atomController);
_internalHttpService.SetupController(gossipController);
_internalHttpService.SetupController(electController);
_internalHttpService.SetupController(histogramController);
_internalHttpService.SetupController(persistentSubscriptionController);
}
// Authentication plugin HTTP
vNodeSettings.AuthenticationProviderFactory.RegisterHttpControllers(_externalHttpService, _internalHttpService, httpSendService, _mainQueue, _workersHandler);
if (_internalHttpService != null)
{
_mainBus.Subscribe <SystemMessage.SystemInit>(_internalHttpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_internalHttpService);
_mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_internalHttpService);
}
SubscribeWorkers(bus =>
{
HttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders.ToArray());
});
// REQUEST FORWARDING
var forwardingService = new RequestForwardingService(_mainQueue, forwardingProxy, TimeSpan.FromSeconds(1));
_mainBus.Subscribe <SystemMessage.SystemStart>(forwardingService);
_mainBus.Subscribe <SystemMessage.RequestForwardingTimerTick>(forwardingService);
_mainBus.Subscribe <ClientMessage.NotHandled>(forwardingService);
_mainBus.Subscribe <ClientMessage.WriteEventsCompleted>(forwardingService);
_mainBus.Subscribe <ClientMessage.TransactionStartCompleted>(forwardingService);
_mainBus.Subscribe <ClientMessage.TransactionWriteCompleted>(forwardingService);
_mainBus.Subscribe <ClientMessage.TransactionCommitCompleted>(forwardingService);
_mainBus.Subscribe <ClientMessage.DeleteStreamCompleted>(forwardingService);
// REQUEST MANAGEMENT
var requestManagement = new RequestManagementService(_mainQueue,
vNodeSettings.PrepareAckCount,
vNodeSettings.PrepareTimeout,
vNodeSettings.CommitTimeout,
vNodeSettings.BetterOrdering);
_mainBus.Subscribe <SystemMessage.SystemInit>(requestManagement);
_mainBus.Subscribe <ClientMessage.WriteEvents>(requestManagement);
_mainBus.Subscribe <ClientMessage.TransactionStart>(requestManagement);
_mainBus.Subscribe <ClientMessage.TransactionWrite>(requestManagement);
_mainBus.Subscribe <ClientMessage.TransactionCommit>(requestManagement);
_mainBus.Subscribe <ClientMessage.DeleteStream>(requestManagement);
_mainBus.Subscribe <StorageMessage.RequestCompleted>(requestManagement);
_mainBus.Subscribe <StorageMessage.CheckStreamAccessCompleted>(requestManagement);
_mainBus.Subscribe <StorageMessage.AlreadyCommitted>(requestManagement);
_mainBus.Subscribe <StorageMessage.CommitReplicated>(requestManagement);
_mainBus.Subscribe <StorageMessage.PrepareAck>(requestManagement);
_mainBus.Subscribe <StorageMessage.WrongExpectedVersion>(requestManagement);
_mainBus.Subscribe <StorageMessage.InvalidTransaction>(requestManagement);
_mainBus.Subscribe <StorageMessage.StreamDeleted>(requestManagement);
_mainBus.Subscribe <StorageMessage.RequestManagerTimerTick>(requestManagement);
// SUBSCRIPTIONS
var subscrBus = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", false);
_mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.SystemStart, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <TcpMessage.ConnectionClosed, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.SubscribeToStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.UnsubscribeFromStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.PollStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.CheckPollTimeout, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <StorageMessage.EventCommitted, Message>());
var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex);
subscrBus.Subscribe <SystemMessage.SystemStart>(subscription);
subscrBus.Subscribe <SystemMessage.BecomeShuttingDown>(subscription);
subscrBus.Subscribe <TcpMessage.ConnectionClosed>(subscription);
subscrBus.Subscribe <ClientMessage.SubscribeToStream>(subscription);
subscrBus.Subscribe <ClientMessage.UnsubscribeFromStream>(subscription);
subscrBus.Subscribe <SubscriptionMessage.PollStream>(subscription);
subscrBus.Subscribe <SubscriptionMessage.CheckPollTimeout>(subscription);
subscrBus.Subscribe <StorageMessage.EventCommitted>(subscription);
// PERSISTENT SUBSCRIPTIONS
// IO DISPATCHER
var ioDispatcher = new IODispatcher(_mainQueue, new PublishEnvelope(_mainQueue));
_mainBus.Subscribe <ClientMessage.ReadStreamEventsBackwardCompleted>(ioDispatcher.BackwardReader);
_mainBus.Subscribe <ClientMessage.WriteEventsCompleted>(ioDispatcher.Writer);
_mainBus.Subscribe <ClientMessage.ReadStreamEventsForwardCompleted>(ioDispatcher.ForwardReader);
_mainBus.Subscribe <ClientMessage.DeleteStreamCompleted>(ioDispatcher.StreamDeleter);
_mainBus.Subscribe(ioDispatcher);
var perSubscrBus = new InMemoryBus("PersistentSubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
var perSubscrQueue = new QueuedHandlerThreadPool(perSubscrBus, "PersistentSubscriptions", false);
_mainBus.Subscribe(perSubscrQueue.WidenFrom <SystemMessage.StateChangeMessage, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <TcpMessage.ConnectionClosed, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.CreatePersistentSubscription, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.UpdatePersistentSubscription, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.DeletePersistentSubscription, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ConnectToPersistentSubscription, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.UnsubscribeFromStream, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.PersistentSubscriptionAckEvents, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.PersistentSubscriptionNackEvents, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ReplayAllParkedMessages, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ReplayParkedMessage, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ReadNextNPersistentMessages, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <StorageMessage.EventCommitted, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <MonitoringMessage.GetAllPersistentSubscriptionStats, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <MonitoringMessage.GetStreamPersistentSubscriptionStats, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <MonitoringMessage.GetPersistentSubscriptionStats, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <SubscriptionMessage.PersistentSubscriptionTimerTick, Message>());
//TODO CC can have multiple threads working on subscription if partition
var consumerStrategyRegistry = new PersistentSubscriptionConsumerStrategyRegistry(_mainQueue, _mainBus, vNodeSettings.AdditionalConsumerStrategies);
var persistentSubscription = new PersistentSubscriptionService(perSubscrQueue, readIndex, ioDispatcher, _mainQueue, consumerStrategyRegistry);
perSubscrBus.Subscribe <SystemMessage.BecomeShuttingDown>(persistentSubscription);
perSubscrBus.Subscribe <SystemMessage.BecomeMaster>(persistentSubscription);
perSubscrBus.Subscribe <SystemMessage.StateChangeMessage>(persistentSubscription);
perSubscrBus.Subscribe <TcpMessage.ConnectionClosed>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.ConnectToPersistentSubscription>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.UnsubscribeFromStream>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.PersistentSubscriptionAckEvents>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.PersistentSubscriptionNackEvents>(persistentSubscription);
perSubscrBus.Subscribe <StorageMessage.EventCommitted>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.DeletePersistentSubscription>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.CreatePersistentSubscription>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.UpdatePersistentSubscription>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.ReplayAllParkedMessages>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.ReplayParkedMessage>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.ReadNextNPersistentMessages>(persistentSubscription);
perSubscrBus.Subscribe <MonitoringMessage.GetAllPersistentSubscriptionStats>(persistentSubscription);
perSubscrBus.Subscribe <MonitoringMessage.GetStreamPersistentSubscriptionStats>(persistentSubscription);
perSubscrBus.Subscribe <MonitoringMessage.GetPersistentSubscriptionStats>(persistentSubscription);
perSubscrBus.Subscribe <SubscriptionMessage.PersistentSubscriptionTimerTick>(persistentSubscription);
// STORAGE SCAVENGER
var scavengerLogManager = new TFChunkScavengerLogManager(_nodeInfo.ExternalHttp.ToString(), TimeSpan.FromDays(vNodeSettings.ScavengeHistoryMaxAge), ioDispatcher);
var storageScavenger = new StorageScavenger(db,
tableIndex,
readIndex,
scavengerLogManager,
vNodeSettings.AlwaysKeepScavenged,
!vNodeSettings.DisableScavengeMerging,
unsafeIgnoreHardDeletes: vNodeSettings.UnsafeIgnoreHardDeletes);
// ReSharper disable RedundantTypeArgumentsOfMethod
_mainBus.Subscribe <ClientMessage.ScavengeDatabase>(storageScavenger);
_mainBus.Subscribe <ClientMessage.StopDatabaseScavenge>(storageScavenger);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(storageScavenger);
// ReSharper restore RedundantTypeArgumentsOfMethod
// TIMER
_timeProvider = new RealTimeProvider();
var threadBasedScheduler = new ThreadBasedScheduler(_timeProvider);
AddTask(threadBasedScheduler.Task);
_timerService = new TimerService(threadBasedScheduler);
_mainBus.Subscribe <SystemMessage.BecomeShutdown>(_timerService);
_mainBus.Subscribe <TimerMessage.Schedule>(_timerService);
var gossipInfo = new VNodeInfo(_nodeInfo.InstanceId, _nodeInfo.DebugIndex,
vNodeSettings.GossipAdvertiseInfo.InternalTcp,
vNodeSettings.GossipAdvertiseInfo.InternalSecureTcp,
vNodeSettings.GossipAdvertiseInfo.ExternalTcp,
vNodeSettings.GossipAdvertiseInfo.ExternalSecureTcp,
vNodeSettings.GossipAdvertiseInfo.InternalHttp,
vNodeSettings.GossipAdvertiseInfo.ExternalHttp);
if (!isSingleNode)
{
// MASTER REPLICATION
var masterReplicationService = new MasterReplicationService(_mainQueue, gossipInfo.InstanceId, db, _workersHandler,
epochManager, vNodeSettings.ClusterNodeCount);
AddTask(masterReplicationService.Task);
_mainBus.Subscribe <SystemMessage.SystemStart>(masterReplicationService);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(masterReplicationService);
_mainBus.Subscribe <ReplicationMessage.ReplicaSubscriptionRequest>(masterReplicationService);
_mainBus.Subscribe <ReplicationMessage.ReplicaLogPositionAck>(masterReplicationService);
monitoringInnerBus.Subscribe <ReplicationMessage.GetReplicationStats>(masterReplicationService);
// REPLICA REPLICATION
var replicaService = new ReplicaService(_mainQueue, db, epochManager, _workersHandler, _internalAuthenticationProvider,
gossipInfo, vNodeSettings.UseSsl, vNodeSettings.SslTargetHost, vNodeSettings.SslValidateServer,
vNodeSettings.IntTcpHeartbeatTimeout, vNodeSettings.ExtTcpHeartbeatInterval);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(replicaService);
_mainBus.Subscribe <ReplicationMessage.ReconnectToMaster>(replicaService);
_mainBus.Subscribe <ReplicationMessage.SubscribeToMaster>(replicaService);
_mainBus.Subscribe <ReplicationMessage.AckLogPosition>(replicaService);
_mainBus.Subscribe <StorageMessage.PrepareAck>(replicaService);
_mainBus.Subscribe <StorageMessage.CommitAck>(replicaService);
_mainBus.Subscribe <ClientMessage.TcpForwardMessage>(replicaService);
}
// ELECTIONS
var electionsService = new ElectionsService(_mainQueue, gossipInfo, vNodeSettings.ClusterNodeCount,
db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint,
epochManager, () => readIndex.LastCommitPosition, vNodeSettings.NodePriority);
electionsService.SubscribeMessages(_mainBus);
if (!isSingleNode || vNodeSettings.GossipOnSingleNode)
{
// GOSSIP
var gossip = new NodeGossipService(_mainQueue, gossipSeedSource, gossipInfo, db.Config.WriterCheckpoint,
db.Config.ChaserCheckpoint, epochManager, () => readIndex.LastCommitPosition,
vNodeSettings.NodePriority, vNodeSettings.GossipInterval, vNodeSettings.GossipAllowedTimeDifference);
_mainBus.Subscribe <SystemMessage.SystemInit>(gossip);
_mainBus.Subscribe <GossipMessage.RetrieveGossipSeedSources>(gossip);
_mainBus.Subscribe <GossipMessage.GotGossipSeedSources>(gossip);
_mainBus.Subscribe <GossipMessage.Gossip>(gossip);
_mainBus.Subscribe <GossipMessage.GossipReceived>(gossip);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(gossip);
_mainBus.Subscribe <GossipMessage.GossipSendFailed>(gossip);
_mainBus.Subscribe <SystemMessage.VNodeConnectionEstablished>(gossip);
_mainBus.Subscribe <SystemMessage.VNodeConnectionLost>(gossip);
}
AddTasks(_workersHandler.Start());
AddTask(_mainQueue.Start());
AddTask(monitoringQueue.Start());
AddTask(subscrQueue.Start());
AddTask(perSubscrQueue.Start());
if (subsystems != null)
{
foreach (var subsystem in subsystems)
{
var http = isSingleNode ? new [] { _externalHttpService } : new [] { _internalHttpService, _externalHttpService };
subsystem.Register(new StandardComponents(db, _mainQueue, _mainBus, _timerService, _timeProvider, httpSendService, http, _workersHandler));
}
}
}
public InternalDispatcherEndpoint(IPublisher inputBus, MultiQueuedHandler requestsMultiHandler)
{
_inputBus = inputBus;
_requestsMultiHandler = requestsMultiHandler;
_publishEnvelope = new PublishEnvelope(inputBus);
}