public void setup()
 {
     _bus = new InMemoryBus("bus");
     _listEventsHandler = new TestHandler<ClientMessage.ReadStreamEventsBackward>();
     _bus.Subscribe(_listEventsHandler);
     _ioDispatcher = new IODispatcher(_bus, new PublishEnvelope(_bus));
     _subscriptionDispatcher =
         new ReaderSubscriptionDispatcher
             (_bus);
     _bus.Subscribe(
         _subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.CommittedEventReceived>());
     _bus.Subscribe(
         _subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.CheckpointSuggested>());
     _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.EofReached>());
     _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.PartitionEofReached>());
     _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.PartitionMeasured>());
     _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.ProgressChanged>());
     _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.SubscriptionStarted>());
     _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.NotAuthorized>());
     _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.ReaderAssignedReader>());
     _bus.Subscribe(_ioDispatcher.BackwardReader);
     _bus.Subscribe(_ioDispatcher.ForwardReader);
     _bus.Subscribe(_ioDispatcher.Writer);
     _bus.Subscribe(_ioDispatcher);
     IProjectionStateHandler projectionStateHandler = new FakeProjectionStateHandler();
     _projectionConfig = new ProjectionConfig(null, 5, 10, 1000, 250, true, true, false, false, false);
     var version = new ProjectionVersion(1, 0, 0);
     var projectionProcessingStrategy = new ContinuousProjectionProcessingStrategy(
         "projection", version, projectionStateHandler, _projectionConfig,
         projectionStateHandler.GetSourceDefinition(), null, _subscriptionDispatcher);
     _coreProjection = projectionProcessingStrategy.Create(
         Guid.NewGuid(), _bus, SystemAccount.Principal, _bus, _ioDispatcher, _subscriptionDispatcher,
         new RealTimeProvider());
     _coreProjection.Start();
 }
 public IEventReader CreatePausedEventReader(
     Guid eventReaderId, IPublisher publisher, IODispatcher ioDispatcher, CheckpointTag checkpointTag,
     bool stopOnEof, int? stopAfterNEvents)
 {
     return new AllStreamsCatalogEventReader(
         ioDispatcher, publisher, eventReaderId, _runAs, checkpointTag.CatalogPosition + 1, _timeProvider, stopOnEof: stopOnEof);
 }
        protected override IProjectionProcessingPhase[] CreateProjectionProcessingPhases(
            IPublisher publisher, Guid projectionCorrelationId, ProjectionNamesBuilder namingBuilder,
            PartitionStateCache partitionStateCache, CoreProjection coreProjection, IODispatcher ioDispatcher,
            IProjectionProcessingPhase firstPhase)
        {

            var coreProjectionCheckpointWriter =
                new CoreProjectionCheckpointWriter(
                    namingBuilder.MakeCheckpointStreamName(), ioDispatcher, _projectionVersion, _name);
            var checkpointManager2 = new DefaultCheckpointManager(
                publisher, projectionCorrelationId, _projectionVersion, _projectionConfig.RunAs, ioDispatcher,
                _projectionConfig, _name, new PhasePositionTagger(1), namingBuilder, GetUseCheckpoints(), false,
                _sourceDefinition.DefinesFold, coreProjectionCheckpointWriter);

            IProjectionProcessingPhase writeResultsPhase;
            if (GetProducesRunningResults()
                || !string.IsNullOrEmpty(_sourceDefinition.CatalogStream) && _sourceDefinition.ByStreams)
                writeResultsPhase = new WriteQueryEofProjectionProcessingPhase(
                    1, namingBuilder.GetResultStreamName(), coreProjection, partitionStateCache, checkpointManager2,
                    checkpointManager2);
            else
                writeResultsPhase = new WriteQueryResultProjectionProcessingPhase(
                    1, namingBuilder.GetResultStreamName(), coreProjection, partitionStateCache, checkpointManager2,
                    checkpointManager2);

            return new[] {firstPhase, writeResultsPhase};
        }
        public override sealed IProjectionProcessingPhase[] CreateProcessingPhases(
            IPublisher publisher, Guid projectionCorrelationId, PartitionStateCache partitionStateCache,
            Action updateStatistics, CoreProjection coreProjection, ProjectionNamesBuilder namingBuilder,
            ITimeProvider timeProvider, IODispatcher ioDispatcher,
            CoreProjectionCheckpointWriter coreProjectionCheckpointWriter)
        {
            var definesFold = _sourceDefinition.DefinesFold;

            var readerStrategy = CreateReaderStrategy(timeProvider);

            var zeroCheckpointTag = readerStrategy.PositionTagger.MakeZeroCheckpointTag();

            var checkpointManager = CreateCheckpointManager(
                projectionCorrelationId, publisher, ioDispatcher, namingBuilder, coreProjectionCheckpointWriter,
                definesFold, readerStrategy);


            var resultWriter = CreateFirstPhaseResultWriter(
                checkpointManager as IEmittedEventWriter, zeroCheckpointTag, namingBuilder);

            var firstPhase = CreateFirstProcessingPhase(
                publisher, projectionCorrelationId, partitionStateCache, updateStatistics, coreProjection,
                _subscriptionDispatcher, zeroCheckpointTag, checkpointManager, readerStrategy, resultWriter);

            return CreateProjectionProcessingPhases(
                publisher, projectionCorrelationId, namingBuilder, partitionStateCache, coreProjection, ioDispatcher,
                firstPhase);
        }
 public void Setup()
 {
     _consumer = new TestHandler<Message>();
     _bus = new InMemoryBus("temp");
     _bus.Subscribe(_consumer);
     ICheckpoint writerCheckpoint = new InMemoryCheckpoint(1000);
     var ioDispatcher = new IODispatcher(_bus, new PublishEnvelope(_bus));
     _readerService = new EventReaderCoreService(_bus, ioDispatcher, 10, writerCheckpoint, runHeadingReader: true);
     _subscriptionDispatcher =
         new ReaderSubscriptionDispatcher(_bus);
     _spoolProcessingResponseDispatcher = new SpooledStreamReadingDispatcher(_bus);
     _timeoutScheduler = new TimeoutScheduler();
     _workerId = Guid.NewGuid();
     _service = new ProjectionCoreService(
         _workerId, _bus, _bus, _subscriptionDispatcher, new RealTimeProvider(), ioDispatcher,
         _spoolProcessingResponseDispatcher, _timeoutScheduler);
     _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.CheckpointSuggested>());
     _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.CommittedEventReceived>());
     _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.EofReached>());
     _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.PartitionEofReached>());
     _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.PartitionMeasured>());
     _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.PartitionDeleted>());
     _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.ProgressChanged>());
     _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.SubscriptionStarted>());
     _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.NotAuthorized>());
     _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.ReaderAssignedReader>());
     _bus.Subscribe(_spoolProcessingResponseDispatcher.CreateSubscriber<PartitionProcessingResult>());
     _readerService.Handle(new Messages.ReaderCoreServiceMessage.StartReader());
     _service.Handle(new ProjectionCoreServiceMessage.StartCore());
 }
 protected override IProjectionProcessingPhase[] CreateProjectionProcessingPhases(
     IPublisher publisher, Guid projectionCorrelationId, ProjectionNamesBuilder namingBuilder,
     PartitionStateCache partitionStateCache, CoreProjection coreProjection, IODispatcher ioDispatcher,
     IProjectionProcessingPhase firstPhase)
 {
     return new[] {firstPhase};
 }
		public IAuthenticationProvider BuildAuthenticationProvider(IPublisher mainQueue, IBus mainBus, IPublisher workersQueue, InMemoryBus[] workerBusses)
		{
			var passwordHashAlgorithm = new Rfc2898PasswordHashAlgorithm();
			var dispatcher = new IODispatcher(mainQueue, new PublishEnvelope(workersQueue, crossThread: true));
			
			foreach (var bus in workerBusses) {
				bus.Subscribe(dispatcher.ForwardReader);
				bus.Subscribe(dispatcher.BackwardReader);
				bus.Subscribe(dispatcher.Writer);
				bus.Subscribe(dispatcher.StreamDeleter);
				bus.Subscribe(dispatcher);
			}

			// USER MANAGEMENT
			var ioDispatcher = new IODispatcher(mainQueue, new PublishEnvelope(mainQueue));
			mainBus.Subscribe(ioDispatcher.BackwardReader);
			mainBus.Subscribe(ioDispatcher.ForwardReader);
			mainBus.Subscribe(ioDispatcher.Writer);
			mainBus.Subscribe(ioDispatcher.StreamDeleter);
			mainBus.Subscribe(ioDispatcher);

			var userManagement = new UserManagementService(mainQueue, ioDispatcher, passwordHashAlgorithm, skipInitializeStandardUsersCheck: false);
			mainBus.Subscribe<UserManagementMessage.Create>(userManagement);
			mainBus.Subscribe<UserManagementMessage.Update>(userManagement);
			mainBus.Subscribe<UserManagementMessage.Enable>(userManagement);
			mainBus.Subscribe<UserManagementMessage.Disable>(userManagement);
			mainBus.Subscribe<UserManagementMessage.Delete>(userManagement);
			mainBus.Subscribe<UserManagementMessage.ResetPassword>(userManagement);
			mainBus.Subscribe<UserManagementMessage.ChangePassword>(userManagement);
			mainBus.Subscribe<UserManagementMessage.Get>(userManagement);
			mainBus.Subscribe<UserManagementMessage.GetAll>(userManagement);
			mainBus.Subscribe<SystemMessage.BecomeMaster>(userManagement);
			
			return new InternalAuthenticationProvider(dispatcher, passwordHashAlgorithm, ESConsts.CachedPrincipalCount);
		}
        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            WriterCheckpoint = new InMemoryCheckpoint(0);
            ChaserCheckpoint = new InMemoryCheckpoint(0);

            Bus = new InMemoryBus("bus");
            IODispatcher = new IODispatcher(Bus, new PublishEnvelope(Bus));

            Db = new TFChunkDb(new TFChunkDbConfig(PathName,
                                                   new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
                                                   10000,
                                                   0,
                                                   WriterCheckpoint,
                                                   ChaserCheckpoint,
                                                   new InMemoryCheckpoint(-1),
                                                   new InMemoryCheckpoint(-1)));

            Db.Open();
            // create db
            Writer = new TFChunkWriter(Db);
            Writer.Open();
            WriteTestScenario();
            Writer.Close();
            Writer = null;

            WriterCheckpoint.Flush();
            ChaserCheckpoint.Write(WriterCheckpoint.Read());
            ChaserCheckpoint.Flush();

            var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
            var lowHasher = new XXHashUnsafe();
            var highHasher = new Murmur3AUnsafe();
            TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
                                        () => new HashListMemTable(PTableVersions.Index64Bit, MaxEntriesInMemTable * 2),
                                        () => new TFReaderLease(readers),
                                        PTableVersions.Index64Bit,
                                        MaxEntriesInMemTable);

            ReadIndex = new ReadIndex(new NoopPublisher(),
                                      readers,
                                      TableIndex,
                                      0,
                                      additionalCommitChecks: true,
                                      metastreamMaxCount: MetastreamMaxCount,
                                      hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault);

            ReadIndex.Init(ChaserCheckpoint.Read());

            // scavenge must run after readIndex is built
            if (_scavenge)
            {
                if (_completeLastChunkOnScavenge)
                    Db.Manager.GetChunk(Db.Manager.ChunksCount - 1).Complete();
                _scavenger = new TFChunkScavenger(Db, IODispatcher, TableIndex, ReadIndex, Guid.NewGuid(), "fakeNodeIp");
                _scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: _mergeChunks);
            }
        }
 public IEventReader CreatePausedEventReader(
     Guid eventReaderId, IPublisher publisher, IODispatcher ioDispatcher, CheckpointTag checkpointTag,
     bool stopOnEof, int? stopAfterNEvents)
 {
     return new ExternallyFedByStreamEventReader(
         publisher, eventReaderId, SystemAccount.Principal, ioDispatcher, checkpointTag.CommitPosition,
         _timeProvider, resolveLinkTos: true);
 }
 public IEventReader CreatePausedEventReader(
     Guid eventReaderId, IPublisher publisher, IODispatcher ioDispatcher, CheckpointTag checkpointTag,
     bool stopOnEof, int? stopAfterNEvents)
 {
     return new StreamEventReader(
         publisher, eventReaderId, _runAs, _catalogStream, checkpointTag.CatalogPosition + 1, _timeProvider,
         resolveLinkTos: true, stopOnEof: stopOnEof, stopAfterNEvents: stopAfterNEvents);
 }
 public override IProjectionProcessingPhase[] CreateProcessingPhases(
     IPublisher publisher, Guid projectionCorrelationId, PartitionStateCache partitionStateCache,
     Action updateStatistics, CoreProjection coreProjection, ProjectionNamesBuilder namingBuilder,
     ITimeProvider timeProvider, IODispatcher ioDispatcher,
     CoreProjectionCheckpointWriter coreProjectionCheckpointWriter)
 {
     return new IProjectionProcessingPhase[] {_phase1, _phase2};
 }
 protected override ICoreProjectionCheckpointManager CreateCheckpointManager(
     Guid projectionCorrelationId, IPublisher publisher, IODispatcher ioDispatcher, ProjectionNamesBuilder namingBuilder,
     CoreProjectionCheckpointWriter coreProjectionCheckpointWriter, bool definesFold, IReaderStrategy readerStrategy)
 {
     return new NoopCheckpointManager(
         publisher, projectionCorrelationId, _projectionConfig, _name, readerStrategy.PositionTagger,
         namingBuilder);
 }
        public void Setup()
        {
            var fakePublisher = new FakePublisher();
            _ioDispatcher = new IODispatcher(fakePublisher, new PublishEnvelope(fakePublisher));

            _subscriptionDispatcher =
                new ReaderSubscriptionDispatcher(new FakePublisher());
        }
        public void Setup()
        {
            //TODO: this became an integration test - proper ProjectionCoreService and ProjectionManager testing is required as well
            _bus.Subscribe(_consumer);

            _manager = new ProjectionManager(
                GetInputQueue(), GetInputQueue(), GivenCoreQueues(), _timeProvider, RunProjections.All,
                _initializeSystemProjections);

            IPublisher inputQueue = GetInputQueue();
            IPublisher publisher = GetInputQueue();
            var ioDispatcher = new IODispatcher(publisher, new PublishEnvelope(inputQueue));
            _bus.Subscribe<ProjectionManagementMessage.Internal.CleanupExpired>(_manager);
            _bus.Subscribe<ProjectionManagementMessage.Internal.Deleted>(_manager);
            _bus.Subscribe<CoreProjectionManagementMessage.Started>(_manager);
            _bus.Subscribe<CoreProjectionManagementMessage.Stopped>(_manager);
            _bus.Subscribe<CoreProjectionManagementMessage.Prepared>(_manager);
            _bus.Subscribe<CoreProjectionManagementMessage.Faulted>(_manager);
            _bus.Subscribe<CoreProjectionManagementMessage.StateReport>(_manager);
            _bus.Subscribe<CoreProjectionManagementMessage.ResultReport>(_manager);
            _bus.Subscribe<CoreProjectionManagementMessage.StatisticsReport>(_manager);
            _bus.Subscribe<CoreProjectionManagementMessage.SlaveProjectionReaderAssigned>(_manager);
            
            _bus.Subscribe<ProjectionManagementMessage.Post>(_manager);
            _bus.Subscribe<ProjectionManagementMessage.UpdateQuery>(_manager);
            _bus.Subscribe<ProjectionManagementMessage.GetQuery>(_manager);
            _bus.Subscribe<ProjectionManagementMessage.Delete>(_manager);
            _bus.Subscribe<ProjectionManagementMessage.GetStatistics>(_manager);
            _bus.Subscribe<ProjectionManagementMessage.GetState>(_manager);
            _bus.Subscribe<ProjectionManagementMessage.GetResult>(_manager);
            _bus.Subscribe<ProjectionManagementMessage.Disable>(_manager);
            _bus.Subscribe<ProjectionManagementMessage.Enable>(_manager);
            _bus.Subscribe<ProjectionManagementMessage.Abort>(_manager);
            _bus.Subscribe<ProjectionManagementMessage.SetRunAs>(_manager);
            _bus.Subscribe<ProjectionManagementMessage.Reset>(_manager);
            _bus.Subscribe<ProjectionManagementMessage.StartSlaveProjections>(_manager);
            _bus.Subscribe<ClientMessage.WriteEventsCompleted>(_manager);
            _bus.Subscribe<ClientMessage.ReadStreamEventsBackwardCompleted>(_manager);
            _bus.Subscribe<ClientMessage.WriteEventsCompleted>(_manager);
            _bus.Subscribe<SystemMessage.StateChangeMessage>(_manager);

            _bus.Subscribe<ClientMessage.ReadStreamEventsForwardCompleted>(ioDispatcher.ForwardReader);
            _bus.Subscribe<ClientMessage.ReadStreamEventsBackwardCompleted>(ioDispatcher.BackwardReader);
            _bus.Subscribe<ClientMessage.WriteEventsCompleted>(ioDispatcher.Writer);
            _bus.Subscribe<ClientMessage.DeleteStreamCompleted>(ioDispatcher.StreamDeleter);
            _bus.Subscribe<IODispatcherDelayedMessage>(ioDispatcher);

            _awakeReaderService = new AwakeReaderService();
            _bus.Subscribe<StorageMessage.EventCommitted>(_awakeReaderService);
            _bus.Subscribe<StorageMessage.TfEofAtNonCommitRecord>(_awakeReaderService);
            _bus.Subscribe<AwakeReaderServiceMessage.SubscribeAwake>(_awakeReaderService);
            _bus.Subscribe<AwakeReaderServiceMessage.UnsubscribeAwake>(_awakeReaderService);


            Given();
            WhenLoop();
        }
 public CoreProjectionCheckpointReader(
     IPublisher publisher, Guid projectionCorrelationId, IODispatcher ioDispatcher, string projectionCheckpointStreamId, ProjectionVersion projectionVersion, bool useCheckpoints)
 {
     _publisher = publisher;
     _projectionCorrelationId = projectionCorrelationId;
     _ioDispatcher = ioDispatcher;
     _projectionCheckpointStreamId = projectionCheckpointStreamId;
     _projectionVersion = projectionVersion;
     _useCheckpoints = useCheckpoints;
 }
 public CoreProjectionCheckpointWriter(
     string projectionCheckpointStreamId, IODispatcher ioDispatcher, ProjectionVersion projectionVersion,
     string name)
 {
     _projectionCheckpointStreamId = projectionCheckpointStreamId;
     _logger = LogManager.GetLoggerFor<CoreProjectionCheckpointWriter>();
     _ioDispatcher = ioDispatcher;
     _projectionVersion = projectionVersion;
     _name = name;
 }
        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            _db = new TFChunkDb(new TFChunkDbConfig(PathName,
                                                    new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
                                                    16 * 1024,
                                                    0,
                                                    new InMemoryCheckpoint(),
                                                    new InMemoryCheckpoint(),
                                                    new InMemoryCheckpoint(-1),
                                                    new InMemoryCheckpoint(-1)));
            _db.Open();
            
            var chunk = _db.Manager.GetChunkFor(0);

            _p1 = LogRecord.SingleWrite(0, Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1",
                                          new byte[] { 0, 1, 2 }, new byte[] { 5, 7 });
            _res1 = chunk.TryAppend(_p1);

            _c1 = LogRecord.Commit(_res1.NewPosition, Guid.NewGuid(), _p1.LogPosition, 0);
            _cres1 = chunk.TryAppend(_c1);

            _p2 = LogRecord.SingleWrite(_cres1.NewPosition,
                                        Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1",
                                        new byte[] { 0, 1, 2 }, new byte[] { 5, 7 });
            _res2 = chunk.TryAppend(_p2);

            _c2 = LogRecord.Commit(_res2.NewPosition, Guid.NewGuid(), _p2.LogPosition, 1);
            _cres2 = chunk.TryAppend(_c2);
            
            _p3 = LogRecord.SingleWrite(_cres2.NewPosition,
                                        Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1",
                                        new byte[] { 0, 1, 2 }, new byte[] { 5, 7 });
            _res3 = chunk.TryAppend(_p3);

            _c3 = LogRecord.Commit(_res3.NewPosition, Guid.NewGuid(), _p3.LogPosition, 2);
            _cres3 = chunk.TryAppend(_c3);

            chunk.Complete();

            _db.Config.WriterCheckpoint.Write(chunk.ChunkHeader.ChunkEndPosition);
            _db.Config.WriterCheckpoint.Flush();
            _db.Config.ChaserCheckpoint.Write(chunk.ChunkHeader.ChunkEndPosition);
            _db.Config.ChaserCheckpoint.Flush();

            var bus = new InMemoryBus("Bus");
            var ioDispatcher = new IODispatcher(bus, new PublishEnvelope(bus));
            var scavenger = new TFChunkScavenger(_db, ioDispatcher, new FakeTableIndex(),
                                                 new FakeReadIndex(x => x == "es-to-scavenge"), Guid.NewGuid(), "fakeNodeIp");
            scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false);

            _scavengedChunk = _db.Manager.GetChunk(0);
        }
 public MultiStreamMultiOutputCheckpointManager(
     IPublisher publisher, Guid projectionCorrelationId, ProjectionVersion projectionVersion, IPrincipal runAs,
     IODispatcher ioDispatcher, ProjectionConfig projectionConfig, string name, PositionTagger positionTagger,
     ProjectionNamesBuilder namingBuilder, bool useCheckpoints, bool producesRunningResults, bool definesFold,
     CoreProjectionCheckpointWriter coreProjectionCheckpointWriter)
     : base(
         publisher, projectionCorrelationId, projectionVersion, runAs, ioDispatcher, projectionConfig, name,
         positionTagger, namingBuilder, useCheckpoints, producesRunningResults, definesFold,
         coreProjectionCheckpointWriter)
 {
     _positionTagger = positionTagger;
 }
 protected override void Given()
 {
     _ioDispatcher = new IODispatcher(_node.Node.MainQueue, new PublishEnvelope(_node.Node.MainQueue));
     _node.Node.MainBus.Subscribe(_ioDispatcher.BackwardReader);
     _node.Node.MainBus.Subscribe(_ioDispatcher.ForwardReader);
     _node.Node.MainBus.Subscribe(_ioDispatcher.Writer);
     _node.Node.MainBus.Subscribe(_ioDispatcher.StreamDeleter);
     _node.Node.MainBus.Subscribe(_ioDispatcher.Awaker);
     _node.Node.MainBus.Subscribe(_ioDispatcher);
     _projectionNamesBuilder = ProjectionNamesBuilder.CreateForTest(_projectionName);
     _emittedStreamsTracker = new EmittedStreamsTracker(_ioDispatcher, new ProjectionConfig(null, 1000, 1000 * 1000, 100, 500, true, true, false, false, false, _trackEmittedStreams), _projectionNamesBuilder);
     _emittedStreamsDeleter = new EmittedStreamsDeleter(_ioDispatcher, _projectionNamesBuilder.GetEmittedStreamsName(), _projectionNamesBuilder.GetEmittedStreamsCheckpointName());
 }
Exemple #20
0
 protected EventReader(
     IODispatcher ioDispatcher, IPublisher publisher, Guid eventReaderCorrelationId, IPrincipal readAs,
     bool stopOnEof, int? stopAfterNEvents)
 {
     if (publisher == null) throw new ArgumentNullException("publisher");
     if (eventReaderCorrelationId == Guid.Empty)
         throw new ArgumentException("eventReaderCorrelationId");
     _publisher = publisher;
     EventReaderCorrelationId = eventReaderCorrelationId;
     _readAs = readAs;
     _stopOnEof = stopOnEof;
     _stopAfterNEvents = stopAfterNEvents;
 }
        protected void SetUpProvider()
        {
            _ioDispatcher = new IODispatcher(_bus, new PublishEnvelope(_bus));
            _bus.Subscribe(_ioDispatcher.BackwardReader);
            _bus.Subscribe(_ioDispatcher.ForwardReader);
            _bus.Subscribe(_ioDispatcher.Writer);
            _bus.Subscribe(_ioDispatcher.StreamDeleter);
            _bus.Subscribe(_ioDispatcher);

            PasswordHashAlgorithm passwordHashAlgorithm = new StubPasswordHashAlgorithm();
            _internalAuthenticationProvider = new InternalAuthenticationProvider(_ioDispatcher, passwordHashAlgorithm, 1000);
            _bus.Subscribe(_internalAuthenticationProvider);
        }
        public ProjectionWorkerNode(
            Guid workerId,
            TFChunkDb db,
            QueuedHandler inputQueue,
            ITimeProvider timeProvider,
            ISingletonTimeoutScheduler timeoutScheduler,
            ProjectionType runProjections)
        {
            _runProjections = runProjections;
            Ensure.NotNull(db, "db");

            _coreOutput = new InMemoryBus("Core Output");

            IPublisher publisher = CoreOutput;
            _subscriptionDispatcher = new ReaderSubscriptionDispatcher(publisher);
            _spoolProcessingResponseDispatcher = new SpooledStreamReadingDispatcher(publisher);

            _ioDispatcher = new IODispatcher(publisher, new PublishEnvelope(inputQueue));
            _eventReaderCoreService = new EventReaderCoreService(
                publisher,
                _ioDispatcher,
                10,
                db.Config.WriterCheckpoint,
                runHeadingReader: runProjections >= ProjectionType.System);

            _feedReaderService = new FeedReaderService(_subscriptionDispatcher, timeProvider);
            if (runProjections >= ProjectionType.System)
            {
                _projectionCoreServiceCommandReader = new ProjectionCoreServiceCommandReader(
                    publisher,
                    _ioDispatcher,
                    workerId.ToString("N"));

                var multiStreamWriter = new MultiStreamMessageWriter(_ioDispatcher);
                _slaveProjectionResponseWriter = new SlaveProjectionResponseWriter(multiStreamWriter);

                _projectionCoreService = new ProjectionCoreService(
                    workerId,
                    inputQueue,
                    publisher,
                    _subscriptionDispatcher,
                    timeProvider,
                    _ioDispatcher,
                    _spoolProcessingResponseDispatcher,
                    timeoutScheduler);

                var responseWriter = new ResponseWriter(_ioDispatcher);
                _coreResponseWriter = new ProjectionCoreResponseWriter(responseWriter);
            }
        }
 public void setup()
 {
     _timeProvider = new FakeTimeProvider();
     _queues = new Dictionary<Guid, IPublisher> {{Guid.NewGuid(), new FakePublisher()}};
     _timeoutSchedulers = ProjectionCoreWorkersNode.CreateTimeoutSchedulers(_queues.Count);
     var fakePublisher = new FakePublisher();
     new ProjectionCoreCoordinator(
         ProjectionType.All, 
         _timeoutSchedulers,
         _queues.Values.ToArray(),
         fakePublisher,
         new NoopEnvelope());
     _ioDispatcher = new IODispatcher(fakePublisher, new PublishEnvelope(fakePublisher));
 }
        public static void CreateManagerService(
            StandardComponents standardComponents,
            ProjectionsStandardComponents projectionsStandardComponents,
            IDictionary<Guid, IPublisher> queues)
        {
            QueuedHandler inputQueue = projectionsStandardComponents.MasterInputQueue;
            InMemoryBus outputBus = projectionsStandardComponents.MasterOutputBus;
            var ioDispatcher = new IODispatcher(outputBus, new PublishEnvelope(inputQueue));

            var projectionsController = new ProjectionsController(
                standardComponents.HttpForwarder,
                inputQueue,
                standardComponents.NetworkSendService);

            var forwarder = new RequestResponseQueueForwarder(
                inputQueue: projectionsStandardComponents.MasterInputQueue,
                externalRequestQueue: standardComponents.MainQueue);

            if (projectionsStandardComponents.RunProjections != ProjectionType.None)
            {
                foreach (var httpService in standardComponents.HttpServices)
                {
                    httpService.SetupController(projectionsController);
                }
            }

            var commandWriter = new MultiStreamMessageWriter(ioDispatcher);
            var projectionManagerCommandWriter = new ProjectionManagerCommandWriter(commandWriter);
            var projectionManagerResponseReader = new ProjectionManagerResponseReader(outputBus, ioDispatcher, queues.Count);

            var projectionManager = new ProjectionManager(
                inputQueue,
                outputBus,
                queues,
                new RealTimeProvider(),
                projectionsStandardComponents.RunProjections,
                ioDispatcher);

            SubscribeMainBus(
                projectionsStandardComponents.MasterMainBus,
                projectionManager,
                projectionsStandardComponents.RunProjections,
                projectionManagerResponseReader,
                ioDispatcher,
                projectionManagerCommandWriter);


            SubscribeOutputBus(standardComponents, projectionsStandardComponents, forwarder);
        }
        public MasterCoreProjectionResponseReader(
            IPublisher publisher,
            IODispatcher ioDispatcher,
            Guid workerId,
            Guid masterProjectionId)
        {
            if (publisher == null) throw new ArgumentNullException("publisher");
            if (ioDispatcher == null) throw new ArgumentNullException("ioDispatcher");

            _publisher = publisher;
            _ioDispatcher = ioDispatcher;
            _workerId = workerId;
            _masterProjectionId = masterProjectionId;
            _streamId = "$projections-$" + masterProjectionId.ToString("N");
        }
        public TFChunkScavenger(TFChunkDb db, IODispatcher ioDispatcher, ITableIndex tableIndex, IReadIndex readIndex,
                                Guid scavengeId, string nodeEndpoint, long? maxChunkDataSize = null, bool unsafeIgnoreHardDeletes=false)
        {
            Ensure.NotNull(db, "db");
            Ensure.NotNull(ioDispatcher, "ioDispatcher");
            Ensure.NotNull(tableIndex, "tableIndex");
            Ensure.NotNull(nodeEndpoint, "nodeEndpoint");
            Ensure.NotNull(readIndex, "readIndex");

            _db = db;
            _ioDispatcher = ioDispatcher;
            _tableIndex = tableIndex;
            _scavengeId = scavengeId;
            _nodeEndpoint = nodeEndpoint;
            _readIndex = readIndex;
            _maxChunkDataSize = maxChunkDataSize ?? db.Config.ChunkSize;
            _unsafeIgnoreHardDeletes = unsafeIgnoreHardDeletes;
        }
        public ByStreamCatalogEventReader(
            IPublisher publisher, Guid eventReaderCorrelationId, IPrincipal readAs, IODispatcher ioDispatcher,
            string catalogCatalogStreamName, int catalogNextSequenceNumber, string dataStreamName,
            int dataNextSequenceNumber, long? limitingCommitPosition, ITimeProvider timeProvider, bool resolveLinkTos,
            int? stopAfterNEvents = null)
            : base(ioDispatcher, publisher, eventReaderCorrelationId, readAs, true, stopAfterNEvents)
        {

            _ioDispatcher = ioDispatcher;
            _catalogStreamName = catalogCatalogStreamName;
            _catalogCurrentSequenceNumber = catalogNextSequenceNumber - 1;
            _catalogNextSequenceNumber = catalogNextSequenceNumber;
            _dataStreamName = dataStreamName;
            _dataNextSequenceNumber = dataNextSequenceNumber;
            _limitingCommitPosition = limitingCommitPosition;
            _timeProvider = timeProvider;
            _resolveLinkTos = resolveLinkTos;
        }
        public CoreProjection Create(
            Guid projectionCorrelationId,
            IPublisher inputQueue,
            Guid workerId,
            IPrincipal runAs,
            IPublisher publisher,
            IODispatcher ioDispatcher,
            ReaderSubscriptionDispatcher subscriptionDispatcher,
            ITimeProvider timeProvider)
        {
            if (inputQueue == null) throw new ArgumentNullException("inputQueue");
            //if (runAs == null) throw new ArgumentNullException("runAs");
            if (publisher == null) throw new ArgumentNullException("publisher");
            if (ioDispatcher == null) throw new ArgumentNullException("ioDispatcher");
            if (timeProvider == null) throw new ArgumentNullException("timeProvider");

            var namingBuilder = new ProjectionNamesBuilder(_name, GetSourceDefinition());

            var coreProjectionCheckpointWriter =
                new CoreProjectionCheckpointWriter(
                    namingBuilder.MakeCheckpointStreamName(),
                    ioDispatcher,
                    _projectionVersion,
                    namingBuilder.EffectiveProjectionName);

            var partitionStateCache = new PartitionStateCache();

            return new CoreProjection(
                this,
                _projectionVersion,
                projectionCorrelationId,
                inputQueue,
                workerId,
                runAs,
                publisher,
                ioDispatcher,
                subscriptionDispatcher,
                _logger,
                namingBuilder,
                coreProjectionCheckpointWriter,
                partitionStateCache,
                namingBuilder.EffectiveProjectionName,
                timeProvider);
        }
        protected override IProjectionProcessingPhase[] CreateProjectionProcessingPhases(
            IPublisher publisher,
            IPublisher inputQueue,
            Guid projectionCorrelationId,
            ProjectionNamesBuilder namingBuilder,
            PartitionStateCache partitionStateCache,
            CoreProjection coreProjection,
            IODispatcher ioDispatcher,
            IProjectionProcessingPhase firstPhase)
        {
            var coreProjectionCheckpointWriter =
                new CoreProjectionCheckpointWriter(
                    namingBuilder.MakeCheckpointStreamName(),
                    ioDispatcher,
                    _projectionVersion,
                    _name);
            var checkpointManager2 = new DefaultCheckpointManager(
                publisher,
                projectionCorrelationId,
                _projectionVersion,
                _projectionConfig.RunAs,
                ioDispatcher,
                _projectionConfig,
                _name,
                new PhasePositionTagger(1),
                namingBuilder,
                GetUseCheckpoints(),
                false,
                _sourceDefinition.DefinesFold,
                coreProjectionCheckpointWriter);

            var writeResultsPhase = new WriteQueryEofProjectionProcessingPhase(
                publisher,
                1,
                namingBuilder.GetResultStreamName(),
                coreProjection,
                partitionStateCache,
                checkpointManager2,
                checkpointManager2,
                firstPhase.EmittedStreamsTracker);

            return new[] {firstPhase, writeResultsPhase};
        }
        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            var dbConfig = new TFChunkDbConfig(PathName,
                                               new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
                                               1024*1024,
                                               0,
                                               new InMemoryCheckpoint(0),
                                               new InMemoryCheckpoint(0),
                                               new InMemoryCheckpoint(-1),
                                               new InMemoryCheckpoint(-1));
            var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
            _dbResult = CreateDb(dbCreationHelper);
            _keptRecords = KeptRecords(_dbResult);

            _dbResult.Db.Config.WriterCheckpoint.Flush();
            _dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read());
            _dbResult.Db.Config.ChaserCheckpoint.Flush();

            var indexPath = Path.Combine(PathName, "index");
            var readerPool = new ObjectPool<ITransactionFileReader>(
                "ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
                () => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint));
            var lowHasher = new XXHashUnsafe();
            var highHasher = new Murmur3AUnsafe();
            var tableIndex = new TableIndex(indexPath, lowHasher, highHasher,
                                            () => new HashListMemTable(PTableVersions.Index64Bit, maxSize: 200),
                                            () => new TFReaderLease(readerPool),
                                            PTableVersions.Index64Bit,
                                            maxSizeForMemory: 100,
                                            maxTablesPerLevel: 2);
            ReadIndex = new ReadIndex(new NoopPublisher(), readerPool, tableIndex, 100, true, _metastreamMaxCount, Opts.HashCollisionReadLimitDefault);
            ReadIndex.Init(_dbResult.Db.Config.WriterCheckpoint.Read());

            //var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount);
            var bus = new InMemoryBus("Bus");
            var ioDispatcher = new IODispatcher(bus, new PublishEnvelope(bus));
            var scavenger = new TFChunkScavenger(_dbResult.Db, ioDispatcher, tableIndex, ReadIndex, Guid.NewGuid(), "fakeNodeIp",
                                            unsafeIgnoreHardDeletes: UnsafeIgnoreHardDelete());
            scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false);
        }