public void Register(StandardComponents standardComponents) { _leaderInputQueue = QueuedHandler.CreateQueuedHandler(_leaderMainBus, "Projections Leader", standardComponents.QueueStatsManager); _leaderOutputBus = new InMemoryBus("ProjectionManagerAndCoreCoordinatorOutput"); _leaderMainBus.Subscribe <ProjectionSubsystemMessage.RestartSubsystem>(this); _leaderMainBus.Subscribe <ProjectionSubsystemMessage.ComponentStarted>(this); _leaderMainBus.Subscribe <ProjectionSubsystemMessage.ComponentStopped>(this); _leaderMainBus.Subscribe <ProjectionSubsystemMessage.IODispatcherDrained>(this); _leaderMainBus.Subscribe <SystemMessage.SystemCoreReady>(this); _leaderMainBus.Subscribe <SystemMessage.StateChangeMessage>(this); var projectionsStandardComponents = new ProjectionsStandardComponents( _projectionWorkerThreadCount, _runProjections, _leaderOutputBus, _leaderInputQueue, _leaderMainBus, _faultOutOfOrderProjections); CreateAwakerService(standardComponents); _coreQueues = ProjectionCoreWorkersNode.CreateCoreWorkers(standardComponents, projectionsStandardComponents); _queueMap = _coreQueues.ToDictionary(v => v.Key, v => (IPublisher)v.Value); ProjectionManagerNode.CreateManagerService(standardComponents, projectionsStandardComponents, _queueMap, _projectionsQueryExpiry); projectionsStandardComponents.LeaderMainBus.Subscribe <CoreProjectionStatusMessage.Stopped>(this); projectionsStandardComponents.LeaderMainBus.Subscribe <CoreProjectionStatusMessage.Started>(this); }
private static void SubscribeOutputBus( StandardComponents standardComponents, ProjectionsStandardComponents projectionsStandardComponents, RequestResponseQueueForwarder forwarder) { var managerOutput = projectionsStandardComponents.MasterOutputBus; managerOutput.Subscribe <ClientMessage.ReadEvent>(forwarder); managerOutput.Subscribe <ClientMessage.ReadStreamEventsBackward>(forwarder); managerOutput.Subscribe <ClientMessage.ReadStreamEventsForward>(forwarder); managerOutput.Subscribe <ClientMessage.WriteEvents>(forwarder); managerOutput.Subscribe <ClientMessage.DeleteStream>(forwarder); managerOutput.Subscribe( Forwarder.Create <ProjectionManagementMessage.RequestSystemProjections>(standardComponents.MainQueue)); managerOutput.Subscribe(Forwarder.Create <Message>(projectionsStandardComponents.MasterInputQueue)); managerOutput.Subscribe <TimerMessage.Schedule>(standardComponents.TimerService); managerOutput.Subscribe(Forwarder.Create <AwakeServiceMessage.SubscribeAwake>(standardComponents.MainQueue)); managerOutput.Subscribe( Forwarder.Create <AwakeServiceMessage.UnsubscribeAwake>(standardComponents.MainQueue)); // self forward all standardComponents.MainBus.Subscribe( Forwarder.Create <SystemMessage.StateChangeMessage>(projectionsStandardComponents.MasterInputQueue)); standardComponents.MainBus.Subscribe( Forwarder.Create <SystemMessage.SystemReady>(projectionsStandardComponents.MasterInputQueue)); projectionsStandardComponents.MasterMainBus.Subscribe(new UnwrapEnvelopeHandler()); }
public static void CreateManagerService( StandardComponents standardComponents, ProjectionsStandardComponents projectionsStandardComponents, IDictionary <Guid, IPublisher> queues, TimeSpan projectionQueryExpiry) { IQueuedHandler inputQueue = projectionsStandardComponents.MasterInputQueue; InMemoryBus outputBus = projectionsStandardComponents.MasterOutputBus; var ioDispatcher = new IODispatcher(outputBus, new PublishEnvelope(inputQueue)); var projectionsController = new ProjectionsController( standardComponents.HttpForwarder, inputQueue, standardComponents.NetworkSendService); var forwarder = new RequestResponseQueueForwarder( inputQueue: projectionsStandardComponents.MasterInputQueue, externalRequestQueue: standardComponents.MainQueue); if (projectionsStandardComponents.RunProjections != ProjectionType.None) { foreach (var httpService in standardComponents.HttpServices) { httpService.SetupController(projectionsController); } } var commandWriter = new MultiStreamMessageWriter(ioDispatcher); var projectionManagerCommandWriter = new ProjectionManagerCommandWriter(commandWriter); var projectionManagerResponseReader = new ProjectionManagerResponseReader(outputBus, ioDispatcher, queues.Count); var projectionManager = new ProjectionManager( inputQueue, outputBus, queues, new RealTimeProvider(), projectionsStandardComponents.RunProjections, ioDispatcher, projectionQueryExpiry); SubscribeMainBus( projectionsStandardComponents.MasterMainBus, projectionManager, projectionsStandardComponents.RunProjections, projectionManagerResponseReader, ioDispatcher, projectionManagerCommandWriter); SubscribeOutputBus(standardComponents, projectionsStandardComponents, forwarder); }
public static void CreateManagerService( StandardComponents standardComponents, ProjectionsStandardComponents projectionsStandardComponents, IDictionary<Guid, IPublisher> queues) { QueuedHandler inputQueue = projectionsStandardComponents.MasterInputQueue; InMemoryBus outputBus = projectionsStandardComponents.MasterOutputBus; var ioDispatcher = new IODispatcher(outputBus, new PublishEnvelope(inputQueue)); var projectionsController = new ProjectionsController( standardComponents.HttpForwarder, inputQueue, standardComponents.NetworkSendService); var forwarder = new RequestResponseQueueForwarder( inputQueue: projectionsStandardComponents.MasterInputQueue, externalRequestQueue: standardComponents.MainQueue); if (projectionsStandardComponents.RunProjections != ProjectionType.None) { foreach (var httpService in standardComponents.HttpServices) { httpService.SetupController(projectionsController); } } var commandWriter = new MultiStreamMessageWriter(ioDispatcher); var projectionManagerCommandWriter = new ProjectionManagerCommandWriter(commandWriter); var projectionManagerResponseReader = new ProjectionManagerResponseReader(outputBus, ioDispatcher, queues.Count); var projectionManager = new ProjectionManager( inputQueue, outputBus, queues, new RealTimeProvider(), projectionsStandardComponents.RunProjections, ioDispatcher); SubscribeMainBus( projectionsStandardComponents.MasterMainBus, projectionManager, projectionsStandardComponents.RunProjections, projectionManagerResponseReader, ioDispatcher, projectionManagerCommandWriter); SubscribeOutputBus(standardComponents, projectionsStandardComponents, forwarder); }
public ProjectionWorkerNode( Guid workerId, TFChunkDb db, IQueuedHandler inputQueue, ITimeProvider timeProvider, ISingletonTimeoutScheduler timeoutScheduler, ProjectionType runProjections, bool faultOutOfOrderProjections, IPublisher leaderOutputBus, ProjectionsStandardComponents configuration) { _runProjections = runProjections; Ensure.NotNull(db, "db"); _coreOutput = new InMemoryBus("Core Output"); _leaderOutputBus = leaderOutputBus; IPublisher publisher = CoreOutput; _subscriptionDispatcher = new ReaderSubscriptionDispatcher(publisher); _ioDispatcher = new IODispatcher(publisher, new PublishEnvelope(inputQueue), true); _eventReaderCoreService = new EventReaderCoreService( publisher, _ioDispatcher, 10, db.Config.WriterCheckpoint, runHeadingReader: runProjections >= ProjectionType.System, faultOutOfOrderProjections: faultOutOfOrderProjections); _feedReaderService = new FeedReaderService(_subscriptionDispatcher, timeProvider); if (runProjections >= ProjectionType.System) { _projectionCoreService = new ProjectionCoreService( workerId, inputQueue, publisher, _subscriptionDispatcher, timeProvider, _ioDispatcher, timeoutScheduler, configuration); } }
public void Register(StandardComponents standardComponents) { _masterMainBus = new InMemoryBus("manager input bus"); _masterInputQueue = new QueuedHandler(_masterMainBus, "Projections Master"); _masterOutputBus = new InMemoryBus("ProjectionManagerAndCoreCoordinatorOutput"); var projectionsStandardComponents = new ProjectionsStandardComponents( _projectionWorkerThreadCount, _runProjections, _masterOutputBus, _masterInputQueue, _masterMainBus); CreateAwakerService(standardComponents); _coreQueues = ProjectionCoreWorkersNode.CreateCoreWorkers(standardComponents, projectionsStandardComponents); _queueMap = _coreQueues.ToDictionary(v => v.Key, v => (IPublisher)v.Value); ProjectionManagerNode.CreateManagerService(standardComponents, projectionsStandardComponents, _queueMap); }
public void Register(StandardComponents standardComponents) { _masterMainBus = new InMemoryBus("manager input bus"); _masterInputQueue = new QueuedHandler(_masterMainBus, "Projections Master"); _masterOutputBus = new InMemoryBus("ProjectionManagerAndCoreCoordinatorOutput"); var projectionsStandardComponents = new ProjectionsStandardComponents( _projectionWorkerThreadCount, _runProjections, _masterOutputBus, _masterInputQueue, _masterMainBus); CreateAwakerService(standardComponents); _coreQueues = ProjectionCoreWorkersNode.CreateCoreWorkers(standardComponents, projectionsStandardComponents); _queueMap = _coreQueues.ToDictionary(v => v.Key, v => (IPublisher)v.Value); ProjectionManagerNode.CreateManagerService(standardComponents, projectionsStandardComponents, _queueMap); }
public void Register(StandardComponents standardComponents) { _masterMainBus = new InMemoryBus("manager input bus"); _masterInputQueue = QueuedHandler.CreateQueuedHandler(_masterMainBus, "Projections Master"); _masterOutputBus = new InMemoryBus("ProjectionManagerAndCoreCoordinatorOutput"); var projectionsStandardComponents = new ProjectionsStandardComponents( _projectionWorkerThreadCount, _runProjections, _masterOutputBus, _masterInputQueue, _masterMainBus, _faultOutOfOrderProjections); CreateAwakerService(standardComponents); _coreQueues = ProjectionCoreWorkersNode.CreateCoreWorkers(standardComponents, projectionsStandardComponents); _queueMap = _coreQueues.ToDictionary(v => v.Key, v => (IPublisher)v.Value); ProjectionManagerNode.CreateManagerService(standardComponents, projectionsStandardComponents, _queueMap, _projectionsQueryExpiry); projectionsStandardComponents.MasterMainBus.Subscribe <CoreProjectionStatusMessage.Stopped>(this); }
private static void SubscribeOutputBus( StandardComponents standardComponents, ProjectionsStandardComponents projectionsStandardComponents, RequestResponseQueueForwarder forwarder, IODispatcher ioDispatcher) { var managerOutput = projectionsStandardComponents.LeaderOutputBus; managerOutput.Subscribe <ClientMessage.ReadEvent>(forwarder); managerOutput.Subscribe <ClientMessage.ReadStreamEventsBackward>(forwarder); managerOutput.Subscribe <ClientMessage.ReadStreamEventsForward>(forwarder); managerOutput.Subscribe <ClientMessage.WriteEvents>(forwarder); managerOutput.Subscribe <ClientMessage.DeleteStream>(forwarder); managerOutput.Subscribe(Forwarder.Create <Message>(projectionsStandardComponents.LeaderInputQueue)); managerOutput.Subscribe <ClientMessage.NotHandled>(ioDispatcher); managerOutput.Subscribe <TimerMessage.Schedule>(standardComponents.TimerService); managerOutput.Subscribe(Forwarder.Create <AwakeServiceMessage.SubscribeAwake>(standardComponents.MainQueue)); managerOutput.Subscribe( Forwarder.Create <AwakeServiceMessage.UnsubscribeAwake>(standardComponents.MainQueue)); managerOutput.Subscribe <SystemMessage.SubSystemInitialized>(forwarder); // self forward all standardComponents.MainBus.Subscribe( Forwarder.Create <SystemMessage.StateChangeMessage>(projectionsStandardComponents.LeaderInputQueue)); standardComponents.MainBus.Subscribe( Forwarder.Create <SystemMessage.SystemCoreReady>(projectionsStandardComponents.LeaderInputQueue)); standardComponents.MainBus.Subscribe( Forwarder.Create <SystemMessage.EpochWritten>(projectionsStandardComponents.LeaderInputQueue)); standardComponents.MainBus.Subscribe( Forwarder.Create <ProjectionCoreServiceMessage.SubComponentStarted>(projectionsStandardComponents .LeaderInputQueue)); standardComponents.MainBus.Subscribe( Forwarder.Create <ProjectionCoreServiceMessage.SubComponentStopped>(projectionsStandardComponents .LeaderInputQueue)); projectionsStandardComponents.LeaderMainBus.Subscribe(new UnwrapEnvelopeHandler()); }
public static Dictionary<Guid, QueuedHandler> CreateCoreWorkers( StandardComponents standardComponents, ProjectionsStandardComponents projectionsStandardComponents) { var coreTimeoutSchedulers = CreateTimeoutSchedulers(projectionsStandardComponents.ProjectionWorkerThreadCount); var coreQueues = new Dictionary<Guid, QueuedHandler>(); while (coreQueues.Count < projectionsStandardComponents.ProjectionWorkerThreadCount) { var coreInputBus = new InMemoryBus("bus"); var coreQueue = new QueuedHandler( coreInputBus, "Projection Core #" + coreQueues.Count, groupName: "Projection Core"); var workerId = Guid.NewGuid(); var projectionNode = new ProjectionWorkerNode( workerId, standardComponents.Db, coreQueue, standardComponents.TimeProvider, coreTimeoutSchedulers[coreQueues.Count], projectionsStandardComponents.RunProjections); projectionNode.SetupMessaging(coreInputBus); var forwarder = new RequestResponseQueueForwarder( inputQueue: coreQueue, externalRequestQueue: standardComponents.MainQueue); // forwarded messages var coreOutput = projectionNode.CoreOutput; coreOutput.Subscribe<ClientMessage.ReadEvent>(forwarder); coreOutput.Subscribe<ClientMessage.ReadStreamEventsBackward>(forwarder); coreOutput.Subscribe<ClientMessage.ReadStreamEventsForward>(forwarder); coreOutput.Subscribe<ClientMessage.ReadAllEventsForward>(forwarder); coreOutput.Subscribe<ClientMessage.WriteEvents>(forwarder); coreOutput.Subscribe<ClientMessage.DeleteStream>(forwarder); if (projectionsStandardComponents.RunProjections >= ProjectionType.System) { var slaveProjectionResponseWriter = projectionNode.SlaveProjectionResponseWriter; coreOutput.Subscribe<PartitionMeasuredOutput>(slaveProjectionResponseWriter); coreOutput.Subscribe<PartitionProcessingProgressOutput>(slaveProjectionResponseWriter); coreOutput.Subscribe<PartitionProcessingResultOutput>(slaveProjectionResponseWriter); coreOutput.Subscribe<ReaderSubscriptionManagement.SpoolStreamReading>(slaveProjectionResponseWriter); coreOutput.Subscribe( Forwarder.Create<AwakeServiceMessage.SubscribeAwake>(standardComponents.MainQueue)); coreOutput.Subscribe( Forwarder.Create<AwakeServiceMessage.UnsubscribeAwake>(standardComponents.MainQueue)); } coreOutput.Subscribe<TimerMessage.Schedule>(standardComponents.TimerService); coreOutput.Subscribe(Forwarder.Create<Message>(coreQueue)); // forward all coreInputBus.Subscribe(new UnwrapEnvelopeHandler()); coreQueues.Add(workerId, coreQueue); } var queues = coreQueues.Select(v => v.Value).Cast<IPublisher>().ToArray(); var coordinator = new ProjectionCoreCoordinator( projectionsStandardComponents.RunProjections, coreTimeoutSchedulers, queues, projectionsStandardComponents.MasterOutputBus, new PublishEnvelope(projectionsStandardComponents.MasterInputQueue, crossThread: true)); coordinator.SetupMessaging(projectionsStandardComponents.MasterMainBus); projectionsStandardComponents.MasterMainBus.Subscribe( Forwarder.CreateBalancing<FeedReaderMessage.ReadPage>(coreQueues.Values.Cast<IPublisher>().ToArray())); return coreQueues; }
public static Dictionary <Guid, IQueuedHandler> CreateCoreWorkers( StandardComponents standardComponents, ProjectionsStandardComponents projectionsStandardComponents) { var coreTimeoutSchedulers = CreateTimeoutSchedulers(projectionsStandardComponents.ProjectionWorkerThreadCount); var coreQueues = new Dictionary <Guid, IQueuedHandler>(); while (coreQueues.Count < projectionsStandardComponents.ProjectionWorkerThreadCount) { var coreInputBus = new InMemoryBus("bus"); var coreQueue = QueuedHandler.CreateQueuedHandler(coreInputBus, "Projection Core #" + coreQueues.Count, groupName: "Projection Core"); var workerId = Guid.NewGuid(); var projectionNode = new ProjectionWorkerNode( workerId, standardComponents.Db, coreQueue, standardComponents.TimeProvider, coreTimeoutSchedulers[coreQueues.Count], projectionsStandardComponents.RunProjections); projectionNode.SetupMessaging(coreInputBus); var forwarder = new RequestResponseQueueForwarder( inputQueue: coreQueue, externalRequestQueue: standardComponents.MainQueue); // forwarded messages var coreOutput = projectionNode.CoreOutput; coreOutput.Subscribe <ClientMessage.ReadEvent>(forwarder); coreOutput.Subscribe <ClientMessage.ReadStreamEventsBackward>(forwarder); coreOutput.Subscribe <ClientMessage.ReadStreamEventsForward>(forwarder); coreOutput.Subscribe <ClientMessage.ReadAllEventsForward>(forwarder); coreOutput.Subscribe <ClientMessage.WriteEvents>(forwarder); coreOutput.Subscribe <ClientMessage.DeleteStream>(forwarder); if (projectionsStandardComponents.RunProjections >= ProjectionType.System) { var slaveProjectionResponseWriter = projectionNode.SlaveProjectionResponseWriter; coreOutput.Subscribe <PartitionMeasuredOutput>(slaveProjectionResponseWriter); coreOutput.Subscribe <PartitionProcessingProgressOutput>(slaveProjectionResponseWriter); coreOutput.Subscribe <PartitionProcessingResultOutput>(slaveProjectionResponseWriter); coreOutput.Subscribe <ReaderSubscriptionManagement.SpoolStreamReading>(slaveProjectionResponseWriter); coreOutput.Subscribe( Forwarder.Create <AwakeServiceMessage.SubscribeAwake>(standardComponents.MainQueue)); coreOutput.Subscribe( Forwarder.Create <AwakeServiceMessage.UnsubscribeAwake>(standardComponents.MainQueue)); } coreOutput.Subscribe <TimerMessage.Schedule>(standardComponents.TimerService); coreOutput.Subscribe(Forwarder.Create <Message>(coreQueue)); // forward all coreInputBus.Subscribe(new UnwrapEnvelopeHandler()); coreQueues.Add(workerId, coreQueue); } var queues = coreQueues.Select(v => v.Value).Cast <IPublisher>().ToArray(); var coordinator = new ProjectionCoreCoordinator( projectionsStandardComponents.RunProjections, coreTimeoutSchedulers, queues, projectionsStandardComponents.MasterOutputBus, new PublishEnvelope(projectionsStandardComponents.MasterInputQueue, crossThread: true)); coordinator.SetupMessaging(projectionsStandardComponents.MasterMainBus); projectionsStandardComponents.MasterMainBus.Subscribe( Forwarder.CreateBalancing <FeedReaderMessage.ReadPage>(coreQueues.Values.Cast <IPublisher>().ToArray())); return(coreQueues); }
private static void SubscribeOutputBus( StandardComponents standardComponents, ProjectionsStandardComponents projectionsStandardComponents, RequestResponseQueueForwarder forwarder) { var managerOutput = projectionsStandardComponents.MasterOutputBus; managerOutput.Subscribe<ClientMessage.ReadEvent>(forwarder); managerOutput.Subscribe<ClientMessage.ReadStreamEventsBackward>(forwarder); managerOutput.Subscribe<ClientMessage.ReadStreamEventsForward>(forwarder); managerOutput.Subscribe<ClientMessage.WriteEvents>(forwarder); managerOutput.Subscribe<ClientMessage.DeleteStream>(forwarder); managerOutput.Subscribe( Forwarder.Create<ProjectionManagementMessage.RequestSystemProjections>(standardComponents.MainQueue)); managerOutput.Subscribe(Forwarder.Create<Message>(projectionsStandardComponents.MasterInputQueue)); managerOutput.Subscribe<TimerMessage.Schedule>(standardComponents.TimerService); managerOutput.Subscribe(Forwarder.Create<AwakeServiceMessage.SubscribeAwake>(standardComponents.MainQueue)); managerOutput.Subscribe( Forwarder.Create<AwakeServiceMessage.UnsubscribeAwake>(standardComponents.MainQueue)); managerOutput.Subscribe<SystemMessage.SubSystemInitialized>(forwarder); // self forward all standardComponents.MainBus.Subscribe( Forwarder.Create<SystemMessage.StateChangeMessage>(projectionsStandardComponents.MasterInputQueue)); standardComponents.MainBus.Subscribe( Forwarder.Create<SystemMessage.SystemCoreReady>(projectionsStandardComponents.MasterInputQueue)); projectionsStandardComponents.MasterMainBus.Subscribe(new UnwrapEnvelopeHandler()); }