Пример #1
0
 public void indicate_that_existing_stream_with_different_hash_is_not_deleted()
 {
     Assert.That(ReadIndex.IsStreamDeleted("ES"), Is.False);
 }
 public void indicate_that_streams_are_deleted()
 {
     Assert.That(ReadIndex.IsStreamDeleted("ES1"));
     Assert.That(ReadIndex.IsStreamDeleted("ES"));
 }
Пример #3
0
        public void read_all_events_backward_returns_no_events()
        {
            var records = ReadIndex.ReadAllEventsBackward(GetBackwardReadPos(), 10).EventRecords();

            Assert.AreEqual(0, records.Count);
        }
Пример #4
0
        public ClusterVNode(TFChunkDb db,
                            ClusterVNodeSettings vNodeSettings,
                            IGossipSeedSource gossipSeedSource,
                            InfoController infoController,
                            params ISubsystem[] subsystems)
        {
            Ensure.NotNull(db, "db");
            Ensure.NotNull(vNodeSettings, "vNodeSettings");
            Ensure.NotNull(gossipSeedSource, "gossipSeedSource");

#if DEBUG
            AddTask(_taskAddedTrigger.Task);
#endif

            var isSingleNode = vNodeSettings.ClusterNodeCount == 1;
            _vNodeSettings      = vNodeSettings;
            _nodeInfo           = vNodeSettings.NodeInfo;
            _certificate        = vNodeSettings.Certificate;
            _mainBus            = new InMemoryBus("MainBus");
            _httpMessageHandler = vNodeSettings.CreateHttpMessageHandler?.Invoke();
            _queueStatsManager  = new QueueStatsManager();

            var forwardingProxy = new MessageForwardingProxy();
            if (vNodeSettings.EnableHistograms)
            {
                HistogramService.CreateHistograms();
                //start watching jitter
                HistogramService.StartJitterMonitor();
            }

            // MISC WORKERS
            _workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
                                                                                   new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
                                                                                                   watchSlowMsg: true,
                                                                                                   slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray();
            _workersHandler = new MultiQueuedHandler(
                vNodeSettings.WorkerThreads,
                queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
                                                        string.Format("Worker #{0}", queueNum + 1),
                                                        _queueStatsManager,
                                                        groupName: "Workers",
                                                        watchSlowMsg: true,
                                                        slowMsgThreshold: TimeSpan.FromMilliseconds(200)));

            _subsystems = subsystems;

            _controller = new ClusterVNodeController((IPublisher)_mainBus, _nodeInfo, db, vNodeSettings, this,
                                                     forwardingProxy, _subsystems);
            _mainQueue = QueuedHandler.CreateQueuedHandler(_controller, "MainQueue", _queueStatsManager);

            _controller.SetMainQueue(_mainQueue);

            //SELF
            _mainBus.Subscribe <SystemMessage.StateChangeMessage>(this);
            _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(this);
            _mainBus.Subscribe <SystemMessage.BecomeShutdown>(this);
            // MONITORING
            var monitoringInnerBus   = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
            var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
            var monitoringQueue      = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", _queueStatsManager, true,
                                                                   TimeSpan.FromMilliseconds(800));
            var monitoring = new MonitoringService(monitoringQueue,
                                                   monitoringRequestBus,
                                                   _mainQueue,
                                                   db.Config.WriterCheckpoint,
                                                   db.Config.Path,
                                                   vNodeSettings.StatsPeriod,
                                                   _nodeInfo.ExternalHttp,
                                                   vNodeSettings.StatsStorage,
                                                   _nodeInfo.ExternalTcp,
                                                   _nodeInfo.ExternalSecureTcp);
            _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.SystemInit, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.StateChangeMessage, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShutdown, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom <ClientMessage.WriteEventsCompleted, Message>());
            monitoringInnerBus.Subscribe <SystemMessage.SystemInit>(monitoring);
            monitoringInnerBus.Subscribe <SystemMessage.StateChangeMessage>(monitoring);
            monitoringInnerBus.Subscribe <SystemMessage.BecomeShuttingDown>(monitoring);
            monitoringInnerBus.Subscribe <SystemMessage.BecomeShutdown>(monitoring);
            monitoringInnerBus.Subscribe <ClientMessage.WriteEventsCompleted>(monitoring);
            monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshStats>(monitoring);
            monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshTcpConnectionStats>(monitoring);

            var truncPos         = db.Config.TruncateCheckpoint.Read();
            var writerCheckpoint = db.Config.WriterCheckpoint.Read();
            var chaserCheckpoint = db.Config.ChaserCheckpoint.Read();
            var epochCheckpoint  = db.Config.EpochCheckpoint.Read();
            if (truncPos != -1)
            {
                Log.Info(
                    "Truncate checkpoint is present. Truncate: {truncatePosition} (0x{truncatePosition:X}), Writer: {writerCheckpoint} (0x{writerCheckpoint:X}), Chaser: {chaserCheckpoint} (0x{chaserCheckpoint:X}), Epoch: {epochCheckpoint} (0x{epochCheckpoint:X})",
                    truncPos, truncPos, writerCheckpoint, writerCheckpoint, chaserCheckpoint, chaserCheckpoint,
                    epochCheckpoint, epochCheckpoint);
                var truncator = new TFChunkDbTruncator(db.Config);
                truncator.TruncateDb(truncPos);
            }

            // STORAGE SUBSYSTEM
            db.Open(vNodeSettings.VerifyDbHash, threads: vNodeSettings.InitializationThreads);
            var indexPath  = vNodeSettings.Index ?? Path.Combine(db.Config.Path, "index");
            var readerPool = new ObjectPool <ITransactionFileReader>(
                "ReadIndex readers pool", ESConsts.PTableInitialReaderCount, vNodeSettings.PTableMaxReaderCount,
                () => new TFChunkReader(db, db.Config.WriterCheckpoint,
                                        optimizeReadSideCache: db.Config.OptimizeReadSideCache));
            var tableIndex = new TableIndex(indexPath,
                                            new XXHashUnsafe(),
                                            new Murmur3AUnsafe(),
                                            () => new HashListMemTable(vNodeSettings.IndexBitnessVersion,
                                                                       maxSize: vNodeSettings.MaxMemtableEntryCount * 2),
                                            () => new TFReaderLease(readerPool),
                                            vNodeSettings.IndexBitnessVersion,
                                            maxSizeForMemory: vNodeSettings.MaxMemtableEntryCount,
                                            maxTablesPerLevel: 2,
                                            inMem: db.Config.InMemDb,
                                            skipIndexVerify: vNodeSettings.SkipIndexVerify,
                                            indexCacheDepth: vNodeSettings.IndexCacheDepth,
                                            initializationThreads: vNodeSettings.InitializationThreads,
                                            additionalReclaim: false,
                                            maxAutoMergeIndexLevel: vNodeSettings.MaxAutoMergeIndexLevel,
                                            pTableMaxReaderCount: vNodeSettings.PTableMaxReaderCount);
            var readIndex = new ReadIndex(_mainQueue,
                                          readerPool,
                                          tableIndex,
                                          ESConsts.StreamInfoCacheCapacity,
                                          Application.IsDefined(Application.AdditionalCommitChecks),
                                          Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1,
                                          vNodeSettings.HashCollisionReadLimit,
                                          vNodeSettings.SkipIndexScanOnReads,
                                          db.Config.ReplicationCheckpoint);
            _readIndex = readIndex;
            var writer       = new TFChunkWriter(db);
            var epochManager = new EpochManager(_mainQueue,
                                                ESConsts.CachedEpochCount,
                                                db.Config.EpochCheckpoint,
                                                writer,
                                                initialReaderCount: 1,
                                                maxReaderCount: 5,
                                                readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint,
                                                                                       optimizeReadSideCache: db.Config.OptimizeReadSideCache));
            epochManager.Init();

            var storageWriter = new ClusterStorageWriterService(_mainQueue, _mainBus, vNodeSettings.MinFlushDelay,
                                                                db, writer, readIndex.IndexWriter, epochManager, _queueStatsManager,
                                                                () => readIndex.LastCommitPosition); // subscribes internally
            AddTasks(storageWriter.Tasks);

            monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageWriter);

            var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex,
                                                         vNodeSettings.ReaderThreadsCount, db.Config.WriterCheckpoint, _queueStatsManager);
            _mainBus.Subscribe <SystemMessage.SystemInit>(storageReader);
            _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageReader);
            _mainBus.Subscribe <SystemMessage.BecomeShutdown>(storageReader);
            monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageReader);

            var indexCommitterService = new IndexCommitterService(readIndex.IndexCommitter, _mainQueue,
                                                                  db.Config.ReplicationCheckpoint, db.Config.WriterCheckpoint, vNodeSettings.CommitAckCount, tableIndex, _queueStatsManager);
            AddTask(indexCommitterService.Task);

            _mainBus.Subscribe <SystemMessage.StateChangeMessage>(indexCommitterService);
            _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(indexCommitterService);
            _mainBus.Subscribe <StorageMessage.CommitAck>(indexCommitterService);
            _mainBus.Subscribe <ClientMessage.MergeIndexes>(indexCommitterService);

            var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint,
                                           db.Config.OptimizeReadSideCache);
            var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, indexCommitterService,
                                                  epochManager, _queueStatsManager);
            AddTask(storageChaser.Task);

#if DEBUG
            _queueStatsManager.InitializeCheckpoints(db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
#endif
            _mainBus.Subscribe <SystemMessage.SystemInit>(storageChaser);
            _mainBus.Subscribe <SystemMessage.SystemStart>(storageChaser);
            _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageChaser);

            // AUTHENTICATION INFRASTRUCTURE - delegate to plugins
            _internalAuthenticationProvider =
                vNodeSettings.AuthenticationProviderFactory.BuildAuthenticationProvider(_mainQueue, _mainBus,
                                                                                        _workersHandler, _workerBuses, vNodeSettings.LogFailedAuthenticationAttempts);

            Ensure.NotNull(_internalAuthenticationProvider, "authenticationProvider");

            {
                // EXTERNAL TCP
                if (!vNodeSettings.DisableInsecureTCP)
                {
                    var extTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalTcp, _workersHandler,
                                                       TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(),
                                                       vNodeSettings.ExtTcpHeartbeatInterval, vNodeSettings.ExtTcpHeartbeatTimeout,
                                                       _internalAuthenticationProvider, null, vNodeSettings.ConnectionPendingSendBytesThreshold,
                                                       vNodeSettings.ConnectionQueueSizeThreshold);
                    _mainBus.Subscribe <SystemMessage.SystemInit>(extTcpService);
                    _mainBus.Subscribe <SystemMessage.SystemStart>(extTcpService);
                    _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(extTcpService);
                }

                // EXTERNAL SECURE TCP
                if (_nodeInfo.ExternalSecureTcp != null)
                {
                    var extSecTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalSecureTcp, _workersHandler,
                                                          TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(),
                                                          vNodeSettings.ExtTcpHeartbeatInterval, vNodeSettings.ExtTcpHeartbeatTimeout,
                                                          _internalAuthenticationProvider, vNodeSettings.Certificate,
                                                          vNodeSettings.ConnectionPendingSendBytesThreshold, vNodeSettings.ConnectionQueueSizeThreshold);
                    _mainBus.Subscribe <SystemMessage.SystemInit>(extSecTcpService);
                    _mainBus.Subscribe <SystemMessage.SystemStart>(extSecTcpService);
                    _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(extSecTcpService);
                }

                if (!isSingleNode)
                {
                    // INTERNAL TCP
                    if (!vNodeSettings.DisableInsecureTCP)
                    {
                        var intTcpService = new TcpService(_mainQueue, _nodeInfo.InternalTcp, _workersHandler,
                                                           TcpServiceType.Internal, TcpSecurityType.Normal,
                                                           new InternalTcpDispatcher(),
                                                           vNodeSettings.IntTcpHeartbeatInterval, vNodeSettings.IntTcpHeartbeatTimeout,
                                                           _internalAuthenticationProvider, null, ESConsts.UnrestrictedPendingSendBytes,
                                                           ESConsts.MaxConnectionQueueSize);
                        _mainBus.Subscribe <SystemMessage.SystemInit>(intTcpService);
                        _mainBus.Subscribe <SystemMessage.SystemStart>(intTcpService);
                        _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(intTcpService);
                    }

                    // INTERNAL SECURE TCP
                    if (_nodeInfo.InternalSecureTcp != null)
                    {
                        var intSecTcpService = new TcpService(_mainQueue, _nodeInfo.InternalSecureTcp, _workersHandler,
                                                              TcpServiceType.Internal, TcpSecurityType.Secure,
                                                              new InternalTcpDispatcher(),
                                                              vNodeSettings.IntTcpHeartbeatInterval, vNodeSettings.IntTcpHeartbeatTimeout,
                                                              _internalAuthenticationProvider, vNodeSettings.Certificate,
                                                              ESConsts.UnrestrictedPendingSendBytes,
                                                              ESConsts.MaxConnectionQueueSize);
                        _mainBus.Subscribe <SystemMessage.SystemInit>(intSecTcpService);
                        _mainBus.Subscribe <SystemMessage.SystemStart>(intSecTcpService);
                        _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(intSecTcpService);
                    }
                }
            }

            SubscribeWorkers(bus => {
                var tcpSendService = new TcpSendService();
                // ReSharper disable RedundantTypeArgumentsOfMethod
                bus.Subscribe <TcpMessage.TcpSend>(tcpSendService);
                // ReSharper restore RedundantTypeArgumentsOfMethod
            });

            var httpAuthenticationProviders = new List <HttpAuthenticationProvider> {
                new BasicHttpAuthenticationProvider(_internalAuthenticationProvider),
            };
            if (vNodeSettings.EnableTrustedAuth)
            {
                httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider());
            }
            httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider());

            var httpPipe        = new HttpMessagePipe();
            var httpSendService = new HttpSendService(httpPipe, forwardRequests: true);
            _mainBus.Subscribe <SystemMessage.StateChangeMessage>(httpSendService);
            _mainBus.Subscribe(new WideningHandler <HttpMessage.SendOverHttp, Message>(_workersHandler));
            SubscribeWorkers(bus => {
                bus.Subscribe <HttpMessage.HttpSend>(httpSendService);
                bus.Subscribe <HttpMessage.HttpSendPart>(httpSendService);
                bus.Subscribe <HttpMessage.HttpBeginSend>(httpSendService);
                bus.Subscribe <HttpMessage.HttpEndSend>(httpSendService);
                bus.Subscribe <HttpMessage.SendOverHttp>(httpSendService);
            });

            _mainBus.Subscribe <SystemMessage.StateChangeMessage>(infoController);

            var adminController     = new AdminController(_mainQueue, _workersHandler);
            var pingController      = new PingController();
            var histogramController = new HistogramController();
            var statController      = new StatController(monitoringQueue, _workersHandler);
            var atomController      = new AtomController(httpSendService, _mainQueue, _workersHandler,
                                                         vNodeSettings.DisableHTTPCaching);
            var gossipController = new GossipController(_mainQueue, _workersHandler, vNodeSettings.GossipTimeout,
                                                        _httpMessageHandler);
            var persistentSubscriptionController =
                new PersistentSubscriptionController(httpSendService, _mainQueue, _workersHandler);
            var electController = new ElectController(_mainQueue, _httpMessageHandler);

            // HTTP SENDERS
            gossipController.SubscribeSenders(httpPipe);
            electController.SubscribeSenders(httpPipe);

            // EXTERNAL HTTP
            _externalHttpService = new KestrelHttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(),
                                                          _workersHandler, vNodeSettings.LogHttpRequests,
                                                          vNodeSettings.GossipAdvertiseInfo.AdvertiseExternalIPAs,
                                                          vNodeSettings.GossipAdvertiseInfo.AdvertiseExternalHttpPortAs,
                                                          vNodeSettings.DisableFirstLevelHttpAuthorization,
                                                          vNodeSettings.NodeInfo.ExternalHttp);
            _externalHttpService.SetupController(persistentSubscriptionController);
            if (vNodeSettings.AdminOnPublic)
            {
                _externalHttpService.SetupController(adminController);
            }
            _externalHttpService.SetupController(pingController);
            _externalHttpService.SetupController(infoController);
            if (vNodeSettings.StatsOnPublic)
            {
                _externalHttpService.SetupController(statController);
            }
            _externalHttpService.SetupController(atomController);
            if (vNodeSettings.GossipOnPublic)
            {
                _externalHttpService.SetupController(gossipController);
            }
            _externalHttpService.SetupController(histogramController);

            _mainBus.Subscribe <SystemMessage.SystemInit>(_externalHttpService);
            _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_externalHttpService);
            _mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_externalHttpService);
            // INTERNAL HTTP
            if (!isSingleNode)
            {
                _internalHttpService = new KestrelHttpService(ServiceAccessibility.Private, _mainQueue,
                                                              new TrieUriRouter(),
                                                              _workersHandler, vNodeSettings.LogHttpRequests,
                                                              vNodeSettings.GossipAdvertiseInfo.AdvertiseInternalIPAs,
                                                              vNodeSettings.GossipAdvertiseInfo.AdvertiseInternalHttpPortAs,
                                                              vNodeSettings.DisableFirstLevelHttpAuthorization,
                                                              vNodeSettings.NodeInfo.InternalHttp);
                _internalHttpService.SetupController(adminController);
                _internalHttpService.SetupController(pingController);
                _internalHttpService.SetupController(infoController);
                _internalHttpService.SetupController(statController);
                _internalHttpService.SetupController(atomController);
                _internalHttpService.SetupController(gossipController);
                _internalHttpService.SetupController(electController);
                _internalHttpService.SetupController(histogramController);
                _internalHttpService.SetupController(persistentSubscriptionController);
            }

            // Authentication plugin HTTP
            vNodeSettings.AuthenticationProviderFactory.RegisterHttpControllers(_externalHttpService,
                                                                                _internalHttpService, httpSendService, _mainQueue, _workersHandler);
            if (_internalHttpService != null)
            {
                _mainBus.Subscribe <SystemMessage.SystemInit>(_internalHttpService);
                _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_internalHttpService);
                _mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_internalHttpService);
            }

            SubscribeWorkers(bus => {
                KestrelHttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders.ToArray());
            });

            // REQUEST FORWARDING
            var forwardingService = new RequestForwardingService(_mainQueue, forwardingProxy, TimeSpan.FromSeconds(1));
            _mainBus.Subscribe <SystemMessage.SystemStart>(forwardingService);
            _mainBus.Subscribe <SystemMessage.RequestForwardingTimerTick>(forwardingService);
            _mainBus.Subscribe <ClientMessage.NotHandled>(forwardingService);
            _mainBus.Subscribe <ClientMessage.WriteEventsCompleted>(forwardingService);
            _mainBus.Subscribe <ClientMessage.TransactionStartCompleted>(forwardingService);
            _mainBus.Subscribe <ClientMessage.TransactionWriteCompleted>(forwardingService);
            _mainBus.Subscribe <ClientMessage.TransactionCommitCompleted>(forwardingService);
            _mainBus.Subscribe <ClientMessage.DeleteStreamCompleted>(forwardingService);

            // REQUEST MANAGEMENT
            var requestManagement = new RequestManagementService(_mainQueue,
                                                                 vNodeSettings.PrepareAckCount,
                                                                 vNodeSettings.PrepareTimeout,
                                                                 vNodeSettings.CommitTimeout,
                                                                 vNodeSettings.BetterOrdering);
            _mainBus.Subscribe <SystemMessage.SystemInit>(requestManagement);
            _mainBus.Subscribe <SystemMessage.StateChangeMessage>(requestManagement);
            _mainBus.Subscribe <ClientMessage.WriteEvents>(requestManagement);
            _mainBus.Subscribe <ClientMessage.TransactionStart>(requestManagement);
            _mainBus.Subscribe <ClientMessage.TransactionWrite>(requestManagement);
            _mainBus.Subscribe <ClientMessage.TransactionCommit>(requestManagement);
            _mainBus.Subscribe <ClientMessage.DeleteStream>(requestManagement);
            _mainBus.Subscribe <StorageMessage.RequestCompleted>(requestManagement);
            _mainBus.Subscribe <StorageMessage.CheckStreamAccessCompleted>(requestManagement);
            _mainBus.Subscribe <StorageMessage.AlreadyCommitted>(requestManagement);
            _mainBus.Subscribe <StorageMessage.CommitReplicated>(requestManagement);
            _mainBus.Subscribe <StorageMessage.PrepareAck>(requestManagement);
            _mainBus.Subscribe <StorageMessage.WrongExpectedVersion>(requestManagement);
            _mainBus.Subscribe <StorageMessage.InvalidTransaction>(requestManagement);
            _mainBus.Subscribe <StorageMessage.StreamDeleted>(requestManagement);
            _mainBus.Subscribe <StorageMessage.RequestManagerTimerTick>(requestManagement);

            // SUBSCRIPTIONS
            var subscrBus   = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
            var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", _queueStatsManager, false);
            _mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.SystemStart, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <TcpMessage.ConnectionClosed, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.SubscribeToStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.FilteredSubscribeToStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.UnsubscribeFromStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.PollStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.CheckPollTimeout, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <StorageMessage.EventCommitted, Message>());

            var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex);
            subscrBus.Subscribe <SystemMessage.SystemStart>(subscription);
            subscrBus.Subscribe <SystemMessage.BecomeShuttingDown>(subscription);
            subscrBus.Subscribe <TcpMessage.ConnectionClosed>(subscription);
            subscrBus.Subscribe <ClientMessage.SubscribeToStream>(subscription);
            subscrBus.Subscribe <ClientMessage.FilteredSubscribeToStream>(subscription);
            subscrBus.Subscribe <ClientMessage.UnsubscribeFromStream>(subscription);
            subscrBus.Subscribe <SubscriptionMessage.PollStream>(subscription);
            subscrBus.Subscribe <SubscriptionMessage.CheckPollTimeout>(subscription);
            subscrBus.Subscribe <StorageMessage.EventCommitted>(subscription);

            // PERSISTENT SUBSCRIPTIONS
            // IO DISPATCHER
            var ioDispatcher = new IODispatcher(_mainQueue, new PublishEnvelope(_mainQueue));
            _mainBus.Subscribe <ClientMessage.ReadStreamEventsBackwardCompleted>(ioDispatcher.BackwardReader);
            _mainBus.Subscribe <ClientMessage.WriteEventsCompleted>(ioDispatcher.Writer);
            _mainBus.Subscribe <ClientMessage.ReadStreamEventsForwardCompleted>(ioDispatcher.ForwardReader);
            _mainBus.Subscribe <ClientMessage.DeleteStreamCompleted>(ioDispatcher.StreamDeleter);
            _mainBus.Subscribe(ioDispatcher);
            var perSubscrBus   = new InMemoryBus("PersistentSubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
            var perSubscrQueue = new QueuedHandlerThreadPool(perSubscrBus, "PersistentSubscriptions", _queueStatsManager, false);
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <SystemMessage.StateChangeMessage, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <TcpMessage.ConnectionClosed, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.CreatePersistentSubscription, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.UpdatePersistentSubscription, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.DeletePersistentSubscription, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ConnectToPersistentSubscription, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.UnsubscribeFromStream, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.PersistentSubscriptionAckEvents, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.PersistentSubscriptionNackEvents, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ReplayAllParkedMessages, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ReplayParkedMessage, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ReadNextNPersistentMessages, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <StorageMessage.EventCommitted, Message>());
            _mainBus.Subscribe(perSubscrQueue
                               .WidenFrom <MonitoringMessage.GetAllPersistentSubscriptionStats, Message>());
            _mainBus.Subscribe(
                perSubscrQueue.WidenFrom <MonitoringMessage.GetStreamPersistentSubscriptionStats, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <MonitoringMessage.GetPersistentSubscriptionStats, Message>());
            _mainBus.Subscribe(perSubscrQueue
                               .WidenFrom <SubscriptionMessage.PersistentSubscriptionTimerTick, Message>());

            //TODO CC can have multiple threads working on subscription if partition
            var consumerStrategyRegistry =
                new PersistentSubscriptionConsumerStrategyRegistry(_mainQueue, _mainBus,
                                                                   vNodeSettings.AdditionalConsumerStrategies);
            var persistentSubscription = new PersistentSubscriptionService(perSubscrQueue, readIndex, ioDispatcher,
                                                                           _mainQueue, consumerStrategyRegistry);
            perSubscrBus.Subscribe <SystemMessage.BecomeShuttingDown>(persistentSubscription);
            perSubscrBus.Subscribe <SystemMessage.BecomeMaster>(persistentSubscription);
            perSubscrBus.Subscribe <SystemMessage.StateChangeMessage>(persistentSubscription);
            perSubscrBus.Subscribe <TcpMessage.ConnectionClosed>(persistentSubscription);
            perSubscrBus.Subscribe <ClientMessage.ConnectToPersistentSubscription>(persistentSubscription);
            perSubscrBus.Subscribe <ClientMessage.UnsubscribeFromStream>(persistentSubscription);
            perSubscrBus.Subscribe <ClientMessage.PersistentSubscriptionAckEvents>(persistentSubscription);
            perSubscrBus.Subscribe <ClientMessage.PersistentSubscriptionNackEvents>(persistentSubscription);
            perSubscrBus.Subscribe <StorageMessage.EventCommitted>(persistentSubscription);
            perSubscrBus.Subscribe <ClientMessage.DeletePersistentSubscription>(persistentSubscription);
            perSubscrBus.Subscribe <ClientMessage.CreatePersistentSubscription>(persistentSubscription);
            perSubscrBus.Subscribe <ClientMessage.UpdatePersistentSubscription>(persistentSubscription);
            perSubscrBus.Subscribe <ClientMessage.ReplayAllParkedMessages>(persistentSubscription);
            perSubscrBus.Subscribe <ClientMessage.ReplayParkedMessage>(persistentSubscription);
            perSubscrBus.Subscribe <ClientMessage.ReadNextNPersistentMessages>(persistentSubscription);
            perSubscrBus.Subscribe <MonitoringMessage.GetAllPersistentSubscriptionStats>(persistentSubscription);
            perSubscrBus.Subscribe <MonitoringMessage.GetStreamPersistentSubscriptionStats>(persistentSubscription);
            perSubscrBus.Subscribe <MonitoringMessage.GetPersistentSubscriptionStats>(persistentSubscription);
            perSubscrBus.Subscribe <SubscriptionMessage.PersistentSubscriptionTimerTick>(persistentSubscription);

            // STORAGE SCAVENGER
            var scavengerLogManager = new TFChunkScavengerLogManager(_nodeInfo.ExternalHttp.ToString(),
                                                                     TimeSpan.FromDays(vNodeSettings.ScavengeHistoryMaxAge), ioDispatcher);
            var storageScavenger = new StorageScavenger(db,
                                                        tableIndex,
                                                        readIndex,
                                                        scavengerLogManager,
                                                        vNodeSettings.AlwaysKeepScavenged,
                                                        !vNodeSettings.DisableScavengeMerging,
                                                        unsafeIgnoreHardDeletes: vNodeSettings.UnsafeIgnoreHardDeletes);

            // ReSharper disable RedundantTypeArgumentsOfMethod
            _mainBus.Subscribe <ClientMessage.ScavengeDatabase>(storageScavenger);
            _mainBus.Subscribe <ClientMessage.StopDatabaseScavenge>(storageScavenger);
            _mainBus.Subscribe <SystemMessage.StateChangeMessage>(storageScavenger);
            // ReSharper restore RedundantTypeArgumentsOfMethod


            // TIMER
            _timeProvider = new RealTimeProvider();
            var threadBasedScheduler = new ThreadBasedScheduler(_timeProvider, _queueStatsManager);
            AddTask(threadBasedScheduler.Task);
            _timerService = new TimerService(threadBasedScheduler);
            _mainBus.Subscribe <SystemMessage.BecomeShutdown>(_timerService);
            _mainBus.Subscribe <TimerMessage.Schedule>(_timerService);

            var gossipInfo = new VNodeInfo(_nodeInfo.InstanceId, _nodeInfo.DebugIndex,
                                           vNodeSettings.GossipAdvertiseInfo.InternalTcp,
                                           vNodeSettings.GossipAdvertiseInfo.InternalSecureTcp,
                                           vNodeSettings.GossipAdvertiseInfo.ExternalTcp,
                                           vNodeSettings.GossipAdvertiseInfo.ExternalSecureTcp,
                                           vNodeSettings.GossipAdvertiseInfo.InternalHttp,
                                           vNodeSettings.GossipAdvertiseInfo.ExternalHttp,
                                           vNodeSettings.ReadOnlyReplica);
            if (!isSingleNode)
            {
                // MASTER REPLICATION
                var masterReplicationService = new MasterReplicationService(_mainQueue, gossipInfo.InstanceId, db,
                                                                            _workersHandler,
                                                                            epochManager, vNodeSettings.ClusterNodeCount,
                                                                            vNodeSettings.UnsafeAllowSurplusNodes,
                                                                            _queueStatsManager);
                AddTask(masterReplicationService.Task);
                _mainBus.Subscribe <SystemMessage.SystemStart>(masterReplicationService);
                _mainBus.Subscribe <SystemMessage.StateChangeMessage>(masterReplicationService);
                _mainBus.Subscribe <ReplicationMessage.ReplicaSubscriptionRequest>(masterReplicationService);
                _mainBus.Subscribe <ReplicationMessage.ReplicaLogPositionAck>(masterReplicationService);
                monitoringInnerBus.Subscribe <ReplicationMessage.GetReplicationStats>(masterReplicationService);

                // REPLICA REPLICATION
                var replicaService = new ReplicaService(_mainQueue, db, epochManager, _workersHandler,
                                                        _internalAuthenticationProvider,
                                                        gossipInfo, vNodeSettings.UseSsl, vNodeSettings.SslTargetHost, vNodeSettings.SslValidateServer,
                                                        vNodeSettings.IntTcpHeartbeatTimeout, vNodeSettings.ExtTcpHeartbeatInterval);
                _mainBus.Subscribe <SystemMessage.StateChangeMessage>(replicaService);
                _mainBus.Subscribe <ReplicationMessage.ReconnectToMaster>(replicaService);
                _mainBus.Subscribe <ReplicationMessage.SubscribeToMaster>(replicaService);
                _mainBus.Subscribe <ReplicationMessage.AckLogPosition>(replicaService);
                _mainBus.Subscribe <StorageMessage.PrepareAck>(replicaService);
                _mainBus.Subscribe <StorageMessage.CommitAck>(replicaService);
                _mainBus.Subscribe <ClientMessage.TcpForwardMessage>(replicaService);
            }

            // ELECTIONS

            var electionsService = new ElectionsService(_mainQueue, gossipInfo, vNodeSettings.ClusterNodeCount,
                                                        db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint,
                                                        epochManager, () => readIndex.LastCommitPosition, vNodeSettings.NodePriority, _timeProvider);
            electionsService.SubscribeMessages(_mainBus);
            if (!isSingleNode || vNodeSettings.GossipOnSingleNode)
            {
                // GOSSIP

                var gossip = new NodeGossipService(_mainQueue, gossipSeedSource, gossipInfo, db.Config.WriterCheckpoint,
                                                   db.Config.ChaserCheckpoint, epochManager, () => readIndex.LastCommitPosition,
                                                   vNodeSettings.NodePriority, vNodeSettings.GossipInterval,
                                                   vNodeSettings.GossipAllowedTimeDifference,
                                                   _timeProvider);
                _mainBus.Subscribe <SystemMessage.SystemInit>(gossip);
                _mainBus.Subscribe <GossipMessage.RetrieveGossipSeedSources>(gossip);
                _mainBus.Subscribe <GossipMessage.GotGossipSeedSources>(gossip);
                _mainBus.Subscribe <GossipMessage.Gossip>(gossip);
                _mainBus.Subscribe <GossipMessage.GossipReceived>(gossip);
                _mainBus.Subscribe <SystemMessage.StateChangeMessage>(gossip);
                _mainBus.Subscribe <GossipMessage.GossipSendFailed>(gossip);
                _mainBus.Subscribe <GossipMessage.UpdateNodePriority>(gossip);
                _mainBus.Subscribe <SystemMessage.VNodeConnectionEstablished>(gossip);
                _mainBus.Subscribe <SystemMessage.VNodeConnectionLost>(gossip);
                _mainBus.Subscribe <GossipMessage.GetGossipFailed>(gossip);
                _mainBus.Subscribe <GossipMessage.GetGossipReceived>(gossip);
                _mainBus.Subscribe <ElectionMessage.ElectionsDone>(gossip);
            }

            _startup = new ClusterVNodeStartup(_subsystems, _mainQueue, _internalAuthenticationProvider, _readIndex,
                                               _vNodeSettings, _externalHttpService, _internalHttpService);

            _mainBus.Subscribe <SystemMessage.SystemReady>(_startup);
            _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_startup);

            // kestrel
            AddTasks(_workersHandler.Start());
            AddTask(_mainQueue.Start());
            AddTask(monitoringQueue.Start());
            AddTask(subscrQueue.Start());
            AddTask(perSubscrQueue.Start());

            if (subsystems != null)
            {
                foreach (var subsystem in subsystems)
                {
                    var http = isSingleNode
                                                ? new[] { _externalHttpService }
                                                : new[] { _internalHttpService, _externalHttpService };
                    subsystem.Register(new StandardComponents(db, _mainQueue, _mainBus, _timerService, _timeProvider,
                                                              httpSendService, http, _workersHandler, _queueStatsManager));
                }
            }
        }
Пример #5
0
 public void return_correct_last_event_version_for_stream()
 {
     Assert.AreEqual(2, ReadIndex.GetStreamLastEventNumber("ES"));
 }
Пример #6
0
 public void throw_on_empty_stream_argument()
 {
     Assert.Throws <ArgumentNullException>(() => ReadIndex.IsStreamDeleted(string.Empty));
 }
 public void read_stream_events_backward_reports_deleted_metastream()
 {
     Assert.AreEqual(ReadStreamResult.StreamDeleted, ReadIndex.ReadStreamEventsBackward("$$test", 0, 100).Result);
 }
Пример #8
0
        public void read_all_events_backwards_returns_nothing_when_prepare_position_is_smaller_than_first_prepare_in_commit()
        {
            var records = ReadIndex.ReadAllEventsBackward(new TFPos(_t2CommitPos, 0), 10).Records;

            Assert.AreEqual(0, records.Count);
        }
 public void get_last_event_number_reports_deleted_metastream()
 {
     Assert.AreEqual(EventNumber.DeletedStream, ReadIndex.GetStreamLastEventNumber("$$test"));
 }
 public void last_event_read_reports_deleted_metastream()
 {
     Assert.AreEqual(ReadEventResult.StreamDeleted, ReadIndex.ReadEvent("$$test", -1).Result);
 }
 public void the_metastream_is_deleted()
 {
     Assert.IsTrue(ReadIndex.IsStreamDeleted("$$test"));
 }
Пример #12
0
        public ClusterVNode(TFChunkDb db,
                            ClusterVNodeSettings vNodeSettings,
                            IGossipSeedSource gossipSeedSource,
                            params ISubsystem[] subsystems)
        {
            Ensure.NotNull(db, "db");
            Ensure.NotNull(vNodeSettings, "vNodeSettings");
            Ensure.NotNull(gossipSeedSource, "gossipSeedSource");

            var isSingleNode = vNodeSettings.ClusterNodeCount == 1;

            _nodeInfo = vNodeSettings.NodeInfo;
            _mainBus  = new InMemoryBus("MainBus");

            var forwardingProxy = new MessageForwardingProxy();

            // MISC WORKERS
            _workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
                                                                                   new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
                                                                                                   watchSlowMsg: true,
                                                                                                   slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray();
            _workersHandler = new MultiQueuedHandler(
                vNodeSettings.WorkerThreads,
                queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
                                                        string.Format("Worker #{0}", queueNum + 1),
                                                        groupName: "Workers",
                                                        watchSlowMsg: true,
                                                        slowMsgThreshold: TimeSpan.FromMilliseconds(200)));

            _controller = new ClusterVNodeController(_mainBus, _nodeInfo, db, vNodeSettings, this, forwardingProxy);
            _mainQueue  = new QueuedHandler(_controller, "MainQueue");

            _controller.SetMainQueue(_mainQueue);

            _subsystems = subsystems;
            //SELF
            _mainBus.Subscribe <SystemMessage.StateChangeMessage>(this);

            // MONITORING
            var monitoringInnerBus   = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
            var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
            var monitoringQueue      = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100));
            var monitoring           = new MonitoringService(monitoringQueue,
                                                             monitoringRequestBus,
                                                             _mainQueue,
                                                             db.Config.WriterCheckpoint,
                                                             db.Config.Path,
                                                             vNodeSettings.StatsPeriod,
                                                             _nodeInfo.ExternalHttp,
                                                             vNodeSettings.StatsStorage);

            _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.SystemInit, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.StateChangeMessage, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShutdown, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom <ClientMessage.WriteEventsCompleted, Message>());
            monitoringInnerBus.Subscribe <SystemMessage.SystemInit>(monitoring);
            monitoringInnerBus.Subscribe <SystemMessage.StateChangeMessage>(monitoring);
            monitoringInnerBus.Subscribe <SystemMessage.BecomeShuttingDown>(monitoring);
            monitoringInnerBus.Subscribe <SystemMessage.BecomeShutdown>(monitoring);
            monitoringInnerBus.Subscribe <ClientMessage.WriteEventsCompleted>(monitoring);
            monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshStats>(monitoring);

            var truncPos = db.Config.TruncateCheckpoint.Read();

            if (truncPos != -1)
            {
                Log.Info("Truncate checkpoint is present. Truncate: {0} (0x{0:X}), Writer: {1} (0x{1:X}), Chaser: {2} (0x{2:X}), Epoch: {3} (0x{3:X})",
                         truncPos, db.Config.WriterCheckpoint.Read(), db.Config.ChaserCheckpoint.Read(), db.Config.EpochCheckpoint.Read());
                var truncator = new TFChunkDbTruncator(db.Config);
                truncator.TruncateDb(truncPos);
            }

            // STORAGE SUBSYSTEM
            db.Open(vNodeSettings.VerifyDbHash);
            var indexPath  = Path.Combine(db.Config.Path, "index");
            var readerPool = new ObjectPool <ITransactionFileReader>(
                "ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
                () => new TFChunkReader(db, db.Config.WriterCheckpoint));
            var tableIndex = new TableIndex(indexPath,
                                            () => new HashListMemTable(maxSize: vNodeSettings.MaxMemtableEntryCount * 2),
                                            () => new TFReaderLease(readerPool),
                                            maxSizeForMemory: vNodeSettings.MaxMemtableEntryCount,
                                            maxTablesPerLevel: 2,
                                            inMem: db.Config.InMemDb);
            var hash      = new XXHashUnsafe();
            var readIndex = new ReadIndex(_mainQueue,
                                          readerPool,
                                          tableIndex,
                                          hash,
                                          ESConsts.StreamInfoCacheCapacity,
                                          Application.IsDefined(Application.AdditionalCommitChecks),
                                          Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1);
            var writer       = new TFChunkWriter(db);
            var epochManager = new EpochManager(ESConsts.CachedEpochCount,
                                                db.Config.EpochCheckpoint,
                                                writer,
                                                initialReaderCount: 1,
                                                maxReaderCount: 5,
                                                readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint));

            epochManager.Init();

            var storageWriter = new ClusterStorageWriterService(_mainQueue, _mainBus, vNodeSettings.MinFlushDelay,
                                                                db, writer, readIndex.IndexWriter, epochManager,
                                                                () => readIndex.LastCommitPosition); // subscribes internally

            monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageWriter);

            var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, ESConsts.StorageReaderThreadCount, db.Config.WriterCheckpoint);

            _mainBus.Subscribe <SystemMessage.SystemInit>(storageReader);
            _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageReader);
            _mainBus.Subscribe <SystemMessage.BecomeShutdown>(storageReader);
            monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageReader);

            var chaser        = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
            var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, readIndex.IndexCommitter, epochManager);

#if DEBUG
            QueueStatsCollector.InitializeCheckpoints(
                _nodeInfo.DebugIndex, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
#endif
            _mainBus.Subscribe <SystemMessage.SystemInit>(storageChaser);
            _mainBus.Subscribe <SystemMessage.SystemStart>(storageChaser);
            _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageChaser);

            var storageScavenger = new StorageScavenger(db, tableIndex, hash, readIndex,
                                                        Application.IsDefined(Application.AlwaysKeepScavenged),
                                                        mergeChunks: !vNodeSettings.DisableScavengeMerging);

            // ReSharper disable RedundantTypeArgumentsOfMethod
            _mainBus.Subscribe <ClientMessage.ScavengeDatabase>(storageScavenger);
            // ReSharper restore RedundantTypeArgumentsOfMethod

            // AUTHENTICATION INFRASTRUCTURE - delegate to plugins
            var authenticationProvider = vNodeSettings.AuthenticationProviderFactory.BuildAuthenticationProvider(_mainQueue, _mainBus, _workersHandler, _workerBuses);

            Ensure.NotNull(authenticationProvider, "authenticationProvider");

            {
                // EXTERNAL TCP
                var extTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalTcp, _workersHandler,
                                                   TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(),
                                                   vNodeSettings.IntTcpHeartbeatInterval, vNodeSettings.IntTcpHeartbeatTimeout,
                                                   authenticationProvider, null);
                _mainBus.Subscribe <SystemMessage.SystemInit>(extTcpService);
                _mainBus.Subscribe <SystemMessage.SystemStart>(extTcpService);
                _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(extTcpService);

                // EXTERNAL SECURE TCP
                if (_nodeInfo.ExternalSecureTcp != null)
                {
                    var extSecTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalSecureTcp, _workersHandler,
                                                          TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(),
                                                          vNodeSettings.ExtTcpHeartbeatInterval, vNodeSettings.ExtTcpHeartbeatTimeout,
                                                          authenticationProvider, vNodeSettings.Certificate);
                    _mainBus.Subscribe <SystemMessage.SystemInit>(extSecTcpService);
                    _mainBus.Subscribe <SystemMessage.SystemStart>(extSecTcpService);
                    _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(extSecTcpService);
                }
                if (!isSingleNode)
                {
                    // INTERNAL TCP
                    var intTcpService = new TcpService(_mainQueue, _nodeInfo.InternalTcp, _workersHandler,
                                                       TcpServiceType.Internal, TcpSecurityType.Normal,
                                                       new InternalTcpDispatcher(),
                                                       vNodeSettings.IntTcpHeartbeatInterval, vNodeSettings.IntTcpHeartbeatTimeout,
                                                       authenticationProvider, null);
                    _mainBus.Subscribe <SystemMessage.SystemInit>(intTcpService);
                    _mainBus.Subscribe <SystemMessage.SystemStart>(intTcpService);
                    _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(intTcpService);

                    // INTERNAL SECURE TCP
                    if (_nodeInfo.InternalSecureTcp != null)
                    {
                        var intSecTcpService = new TcpService(_mainQueue, _nodeInfo.InternalSecureTcp, _workersHandler,
                                                              TcpServiceType.Internal, TcpSecurityType.Secure,
                                                              new InternalTcpDispatcher(),
                                                              vNodeSettings.IntTcpHeartbeatInterval, vNodeSettings.IntTcpHeartbeatTimeout,
                                                              authenticationProvider, vNodeSettings.Certificate);
                        _mainBus.Subscribe <SystemMessage.SystemInit>(intSecTcpService);
                        _mainBus.Subscribe <SystemMessage.SystemStart>(intSecTcpService);
                        _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(intSecTcpService);
                    }
                }
            }

            SubscribeWorkers(bus =>
            {
                var tcpSendService = new TcpSendService();
                // ReSharper disable RedundantTypeArgumentsOfMethod
                bus.Subscribe <TcpMessage.TcpSend>(tcpSendService);
                // ReSharper restore RedundantTypeArgumentsOfMethod
            });

            var httpAuthenticationProviders = new List <HttpAuthenticationProvider>
            {
                new BasicHttpAuthenticationProvider(authenticationProvider),
            };
            if (vNodeSettings.EnableTrustedAuth)
            {
                httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider());
            }
            httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider());

            var httpPipe        = new HttpMessagePipe();
            var httpSendService = new HttpSendService(httpPipe, forwardRequests: true);
            _mainBus.Subscribe <SystemMessage.StateChangeMessage>(httpSendService);
            _mainBus.Subscribe(new WideningHandler <HttpMessage.SendOverHttp, Message>(_workersHandler));
            SubscribeWorkers(bus =>
            {
                bus.Subscribe <HttpMessage.HttpSend>(httpSendService);
                bus.Subscribe <HttpMessage.HttpSendPart>(httpSendService);
                bus.Subscribe <HttpMessage.HttpBeginSend>(httpSendService);
                bus.Subscribe <HttpMessage.HttpEndSend>(httpSendService);
                bus.Subscribe <HttpMessage.SendOverHttp>(httpSendService);
            });

            var adminController  = new AdminController(_mainQueue);
            var pingController   = new PingController();
            var infoController   = new InfoController();
            var statController   = new StatController(monitoringQueue, _workersHandler);
            var atomController   = new AtomController(httpSendService, _mainQueue, _workersHandler);
            var gossipController = new GossipController(_mainQueue, _workersHandler, vNodeSettings.GossipTimeout);
            var electController  = new ElectController(_mainQueue);

            // HTTP SENDERS
            gossipController.SubscribeSenders(httpPipe);
            electController.SubscribeSenders(httpPipe);

            // EXTERNAL HTTP
            _externalHttpService = new HttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(),
                                                   _workersHandler, vNodeSettings.HttpPrefixes);
            if (vNodeSettings.AdminOnPublic)
            {
                _externalHttpService.SetupController(adminController);
            }
            _externalHttpService.SetupController(pingController);
            _externalHttpService.SetupController(infoController);
            if (vNodeSettings.StatsOnPublic)
            {
                _externalHttpService.SetupController(statController);
            }
            _externalHttpService.SetupController(atomController);
            if (vNodeSettings.GossipOnPublic)
            {
                _externalHttpService.SetupController(gossipController);
            }

            _mainBus.Subscribe <SystemMessage.SystemInit>(_externalHttpService);
            _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_externalHttpService);
            _mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_externalHttpService);
            // INTERNAL HTTP
            if (!isSingleNode)
            {
                _internalHttpService = new HttpService(ServiceAccessibility.Private, _mainQueue, new TrieUriRouter(),
                                                       _workersHandler, _nodeInfo.InternalHttp.ToHttpUrl());
                _internalHttpService.SetupController(adminController);
                _internalHttpService.SetupController(pingController);
                _internalHttpService.SetupController(infoController);
                _internalHttpService.SetupController(statController);
                _internalHttpService.SetupController(atomController);
                _internalHttpService.SetupController(gossipController);
                _internalHttpService.SetupController(electController);
            }
            // Authentication plugin HTTP
            vNodeSettings.AuthenticationProviderFactory.RegisterHttpControllers(_externalHttpService, _internalHttpService, httpSendService, _mainQueue, _workersHandler);
            if (_internalHttpService != null)
            {
                _mainBus.Subscribe <SystemMessage.SystemInit>(_internalHttpService);
                _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_internalHttpService);
                _mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_internalHttpService);
            }

            SubscribeWorkers(bus =>
            {
                HttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders.ToArray());
            });

            // REQUEST FORWARDING
            var forwardingService = new RequestForwardingService(_mainQueue, forwardingProxy, TimeSpan.FromSeconds(1));
            _mainBus.Subscribe <SystemMessage.SystemStart>(forwardingService);
            _mainBus.Subscribe <SystemMessage.RequestForwardingTimerTick>(forwardingService);
            _mainBus.Subscribe <ClientMessage.NotHandled>(forwardingService);
            _mainBus.Subscribe <ClientMessage.WriteEventsCompleted>(forwardingService);
            _mainBus.Subscribe <ClientMessage.TransactionStartCompleted>(forwardingService);
            _mainBus.Subscribe <ClientMessage.TransactionWriteCompleted>(forwardingService);
            _mainBus.Subscribe <ClientMessage.TransactionCommitCompleted>(forwardingService);
            _mainBus.Subscribe <ClientMessage.DeleteStreamCompleted>(forwardingService);

            // REQUEST MANAGEMENT
            var requestManagement = new RequestManagementService(_mainQueue,
                                                                 vNodeSettings.PrepareAckCount,
                                                                 vNodeSettings.CommitAckCount,
                                                                 vNodeSettings.PrepareTimeout,
                                                                 vNodeSettings.CommitTimeout);
            _mainBus.Subscribe <SystemMessage.SystemInit>(requestManagement);
            _mainBus.Subscribe <ClientMessage.WriteEvents>(requestManagement);
            _mainBus.Subscribe <ClientMessage.TransactionStart>(requestManagement);
            _mainBus.Subscribe <ClientMessage.TransactionWrite>(requestManagement);
            _mainBus.Subscribe <ClientMessage.TransactionCommit>(requestManagement);
            _mainBus.Subscribe <ClientMessage.DeleteStream>(requestManagement);
            _mainBus.Subscribe <StorageMessage.RequestCompleted>(requestManagement);
            _mainBus.Subscribe <StorageMessage.CheckStreamAccessCompleted>(requestManagement);
            _mainBus.Subscribe <StorageMessage.AlreadyCommitted>(requestManagement);
            _mainBus.Subscribe <StorageMessage.CommitAck>(requestManagement);
            _mainBus.Subscribe <StorageMessage.PrepareAck>(requestManagement);
            _mainBus.Subscribe <StorageMessage.WrongExpectedVersion>(requestManagement);
            _mainBus.Subscribe <StorageMessage.InvalidTransaction>(requestManagement);
            _mainBus.Subscribe <StorageMessage.StreamDeleted>(requestManagement);
            _mainBus.Subscribe <StorageMessage.RequestManagerTimerTick>(requestManagement);

            // SUBSCRIPTIONS
            var subscrBus   = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
            var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", false);
            _mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.SystemStart, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <TcpMessage.ConnectionClosed, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.SubscribeToStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.UnsubscribeFromStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.PollStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.CheckPollTimeout, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <StorageMessage.EventCommitted, Message>());

            var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex);
            subscrBus.Subscribe <SystemMessage.SystemStart>(subscription);
            subscrBus.Subscribe <SystemMessage.BecomeShuttingDown>(subscription);
            subscrBus.Subscribe <TcpMessage.ConnectionClosed>(subscription);
            subscrBus.Subscribe <ClientMessage.SubscribeToStream>(subscription);
            subscrBus.Subscribe <ClientMessage.UnsubscribeFromStream>(subscription);
            subscrBus.Subscribe <SubscriptionMessage.PollStream>(subscription);
            subscrBus.Subscribe <SubscriptionMessage.CheckPollTimeout>(subscription);
            subscrBus.Subscribe <StorageMessage.EventCommitted>(subscription);

            // TIMER
            _timeProvider = new RealTimeProvider();
            _timerService = new TimerService(new ThreadBasedScheduler(_timeProvider));
            _mainBus.Subscribe <SystemMessage.BecomeShutdown>(_timerService);
            _mainBus.Subscribe <TimerMessage.Schedule>(_timerService);
            if (!isSingleNode)
            {
                // MASTER REPLICATION
                var masterReplicationService = new MasterReplicationService(_mainQueue, _nodeInfo.InstanceId, db, _workersHandler,
                                                                            epochManager, vNodeSettings.ClusterNodeCount);
                _mainBus.Subscribe <SystemMessage.SystemStart>(masterReplicationService);
                _mainBus.Subscribe <SystemMessage.StateChangeMessage>(masterReplicationService);
                _mainBus.Subscribe <ReplicationMessage.ReplicaSubscriptionRequest>(masterReplicationService);
                _mainBus.Subscribe <ReplicationMessage.ReplicaLogPositionAck>(masterReplicationService);

                // REPLICA REPLICATION
                var replicaService = new ReplicaService(_mainQueue, db, epochManager, _workersHandler, authenticationProvider,
                                                        _nodeInfo, vNodeSettings.UseSsl, vNodeSettings.SslTargetHost, vNodeSettings.SslValidateServer,
                                                        vNodeSettings.IntTcpHeartbeatTimeout, vNodeSettings.ExtTcpHeartbeatInterval);
                _mainBus.Subscribe <SystemMessage.StateChangeMessage>(replicaService);
                _mainBus.Subscribe <ReplicationMessage.ReconnectToMaster>(replicaService);
                _mainBus.Subscribe <ReplicationMessage.SubscribeToMaster>(replicaService);
                _mainBus.Subscribe <ReplicationMessage.AckLogPosition>(replicaService);
                _mainBus.Subscribe <StorageMessage.PrepareAck>(replicaService);
                _mainBus.Subscribe <StorageMessage.CommitAck>(replicaService);
                _mainBus.Subscribe <ClientMessage.TcpForwardMessage>(replicaService);
            }

            // ELECTIONS
            var electionsService = new ElectionsService(_mainQueue, _nodeInfo, vNodeSettings.ClusterNodeCount,
                                                        db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint,
                                                        epochManager, () => readIndex.LastCommitPosition, vNodeSettings.NodePriority);
            electionsService.SubscribeMessages(_mainBus);
            if (!isSingleNode)
            {
                // GOSSIP
                var gossip = new NodeGossipService(_mainQueue, gossipSeedSource, _nodeInfo, db.Config.WriterCheckpoint,
                                                   db.Config.ChaserCheckpoint, epochManager, () => readIndex.LastCommitPosition,
                                                   vNodeSettings.NodePriority, vNodeSettings.GossipInterval, vNodeSettings.GossipAllowedTimeDifference);
                _mainBus.Subscribe <SystemMessage.SystemInit>(gossip);
                _mainBus.Subscribe <GossipMessage.RetrieveGossipSeedSources>(gossip);
                _mainBus.Subscribe <GossipMessage.GotGossipSeedSources>(gossip);
                _mainBus.Subscribe <GossipMessage.Gossip>(gossip);
                _mainBus.Subscribe <GossipMessage.GossipReceived>(gossip);
                _mainBus.Subscribe <SystemMessage.StateChangeMessage>(gossip);
                _mainBus.Subscribe <GossipMessage.GossipSendFailed>(gossip);
                _mainBus.Subscribe <SystemMessage.VNodeConnectionEstablished>(gossip);
                _mainBus.Subscribe <SystemMessage.VNodeConnectionLost>(gossip);
            }
            _workersHandler.Start();
            _mainQueue.Start();
            monitoringQueue.Start();
            subscrQueue.Start();

            if (subsystems != null)
            {
                foreach (var subsystem in subsystems)
                {
                    var http = isSingleNode ? new [] { _externalHttpService } : new [] { _internalHttpService, _externalHttpService };
                    subsystem.Register(new StandardComponents(db, _mainQueue, _mainBus, _timerService, _timeProvider, httpSendService, http, _workersHandler));
                }
            }
        }
 public void last_event_number_returns_stream_deleted()
 {
     Assert.AreEqual(EventNumber.DeletedStream, ReadIndex.GetLastStreamEventNumber("ES"));
 }
 public void is_stream_deleted_returns_true()
 {
     Assert.That(ReadIndex.IsStreamDeleted("ES"));
 }
Пример #15
0
        public void should_be_able_to_read_stream_events_backward()
        {
            var result = ReadIndex.ReadStreamEventsBackward(_stream, -1, 10);

            Assert.AreEqual(2, result.Records.Length);
        }
Пример #16
0
        public void read_all_events_forward_returns_nothing_when_prepare_position_is_greater_than_last_prepare_in_commit()
        {
            var records = ReadIndex.ReadAllEventsForward(new TFPos(_t1CommitPos, _t1CommitPos), 10).Records;

            Assert.AreEqual(0, records.Count);
        }
Пример #17
0
 public void indicate_that_stream_is_deleted()
 {
     Assert.That(ReadIndex.IsStreamDeleted("S1"));
 }
Пример #18
0
        public void should_read_the_correct_last_event_number()
        {
            var result = ReadIndex.GetStreamLastEventNumber("duplicate_stream");

            Assert.AreEqual(2, result);
        }
Пример #19
0
 public void indicate_that_other_stream_with_same_hash_is_not_deleted()
 {
     Assert.That(ReadIndex.IsStreamDeleted("S2"), Is.False);
 }
Пример #20
0
 public void crash_on_null_stream_argument()
 {
     Assert.Throws <ArgumentNullException>(() => ReadIndex.IsStreamDeleted(null));
 }
 public void return_minus_one_for_nonexistent_stream_as_last_event_version()
 {
     Assert.AreEqual(-1, ReadIndex.GetStreamLastEventNumber("ES-NONEXISTENT"));
 }
Пример #22
0
        public void read_all_events_forward_returns_no_events()
        {
            var records = ReadIndex.ReadAllEventsForward(new TFPos(0, 0), 10).EventRecords();

            Assert.AreEqual(0, records.Count);
        }
 public void return_correct_last_event_version_for_existing_stream_with_single_event()
 {
     Assert.AreEqual(0, ReadIndex.GetStreamLastEventNumber("ES"));
 }
 public void indicate_that_nonexisting_stream_with_same_hash_is_not_deleted()
 {
     Assert.That(ReadIndex.IsStreamDeleted("ZZ"), Is.False);
 }
 public void return_correct_last_event_version_for_nonexistent_stream_with_same_hash_as_existing_one()
 {
     Assert.AreEqual(-1, ReadIndex.GetStreamLastEventNumber("AB"));
 }
Пример #26
0
        public SingleVNode(TFChunkDb db, SingleVNodeSettings vNodeSettings, SingleVNodeAppSettings appSettings)
        {
            Ensure.NotNull(db, "db");
            Ensure.NotNull(vNodeSettings, "vNodeSettings");

            db.OpenVerifyAndClean();

            _tcpEndPoint  = vNodeSettings.ExternalTcpEndPoint;
            _httpEndPoint = vNodeSettings.ExternalHttpEndPoint;

            _outputBus  = new InMemoryBus("OutputBus");
            _controller = new SingleVNodeController(Bus, _httpEndPoint);
            _mainQueue  = new QueuedHandler(_controller, "MainQueue");
            _controller.SetMainQueue(MainQueue);

            //MONITORING
            var monitoringInnerBus   = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
            var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
            var monitoringQueue      = new QueuedHandler(monitoringInnerBus, "MonitoringQueue", watchSlowMsg: true, slowMsgThresholdMs: 100);
            var monitoring           = new MonitoringService(monitoringQueue, monitoringRequestBus, db.Config.WriterCheckpoint, db.Config.Path, appSettings.StatsPeriod);

            Bus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.SystemInit, Message>());
            Bus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
            monitoringInnerBus.Subscribe <SystemMessage.SystemInit>(monitoring);
            monitoringInnerBus.Subscribe <SystemMessage.BecomeShuttingDown>(monitoring);
            monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshStats>(monitoring);

            //STORAGE SUBSYSTEM
            var indexPath  = Path.Combine(db.Config.Path, "index");
            var tableIndex = new TableIndex(indexPath,
                                            () => new HashListMemTable(),
                                            maxSizeForMemory: 1000000,
                                            maxTablesPerLevel: 2);

            var readIndex     = new ReadIndex(_mainQueue, TFConsts.ReadIndexReaderCount, () => new TFChunkSequentialReader(db, db.Config.WriterCheckpoint, 0), () => new TFChunkReader(db, db.Config.WriterCheckpoint), tableIndex, new XXHashUnsafe());
            var writer        = new TFChunkWriter(db);
            var storageWriter = new StorageWriter(_mainQueue, _outputBus, writer, readIndex);
            var storageReader = new StorageReader(_mainQueue, _outputBus, readIndex, TFConsts.StorageReaderHandlerCount, db.Config.WriterCheckpoint);

            monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageReader);

            var chaser = new TFChunkChaser(db,
                                           db.Config.WriterCheckpoint,
                                           db.Config.GetNamedCheckpoint(Checkpoint.Chaser));
            var storageChaser = new StorageChaser(_mainQueue, chaser);

            _outputBus.Subscribe <SystemMessage.SystemInit>(storageChaser);
            _outputBus.Subscribe <SystemMessage.SystemStart>(storageChaser);
            _outputBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageChaser);

            var storageScavenger = new StorageScavenger(db, readIndex);

            _outputBus.Subscribe <SystemMessage.ScavengeDatabase>(storageScavenger);

            //TCP
            var tcpService = new TcpService(MainQueue, _tcpEndPoint);

            Bus.Subscribe <SystemMessage.SystemInit>(tcpService);
            Bus.Subscribe <SystemMessage.SystemStart>(tcpService);
            Bus.Subscribe <SystemMessage.BecomeShuttingDown>(tcpService);

            //HTTP
            HttpService = new HttpService(ServiceAccessibility.Private, MainQueue, vNodeSettings.HttpPrefixes);
            Bus.Subscribe <SystemMessage.SystemInit>(HttpService);
            Bus.Subscribe <SystemMessage.BecomeShuttingDown>(HttpService);
            Bus.Subscribe <HttpMessage.SendOverHttp>(HttpService);
            Bus.Subscribe <HttpMessage.UpdatePendingRequests>(HttpService);
            HttpService.SetupController(new AdminController(MainQueue));
            HttpService.SetupController(new PingController());
            HttpService.SetupController(new StatController(monitoringQueue));
            HttpService.SetupController(new ReadEventDataController(MainQueue));
            HttpService.SetupController(new AtomController(MainQueue));
            HttpService.SetupController(new WebSiteController(MainQueue));

            //REQUEST MANAGEMENT
            var requestManagement = new RequestManagementService(MainQueue, 1, 1);

            Bus.Subscribe <ReplicationMessage.CreateStreamRequestCreated>(requestManagement);
            Bus.Subscribe <ReplicationMessage.WriteRequestCreated>(requestManagement);
            Bus.Subscribe <ReplicationMessage.TransactionStartRequestCreated>(requestManagement);
            Bus.Subscribe <ReplicationMessage.TransactionWriteRequestCreated>(requestManagement);
            Bus.Subscribe <ReplicationMessage.TransactionCommitRequestCreated>(requestManagement);
            Bus.Subscribe <ReplicationMessage.DeleteStreamRequestCreated>(requestManagement);
            Bus.Subscribe <ReplicationMessage.RequestCompleted>(requestManagement);
            Bus.Subscribe <ReplicationMessage.AlreadyCommitted>(requestManagement);
            Bus.Subscribe <ReplicationMessage.CommitAck>(requestManagement);
            Bus.Subscribe <ReplicationMessage.PrepareAck>(requestManagement);
            Bus.Subscribe <ReplicationMessage.WrongExpectedVersion>(requestManagement);
            Bus.Subscribe <ReplicationMessage.InvalidTransaction>(requestManagement);
            Bus.Subscribe <ReplicationMessage.StreamDeleted>(requestManagement);
            Bus.Subscribe <ReplicationMessage.PreparePhaseTimeout>(requestManagement);
            Bus.Subscribe <ReplicationMessage.CommitPhaseTimeout>(requestManagement);

            var clientService = new ClientService();

            Bus.Subscribe <TcpMessage.ConnectionClosed>(clientService);
            Bus.Subscribe <ClientMessage.SubscribeToStream>(clientService);
            Bus.Subscribe <ClientMessage.UnsubscribeFromStream>(clientService);
            Bus.Subscribe <ClientMessage.SubscribeToAllStreams>(clientService);
            Bus.Subscribe <ClientMessage.UnsubscribeFromAllStreams>(clientService);
            Bus.Subscribe <ReplicationMessage.EventCommited>(clientService);

            //TIMER
            //var timer = new TimerService(new TimerBasedScheduler(new RealTimer(), new RealTimeProvider()));
            TimerService = new TimerService(new ThreadBasedScheduler(new RealTimeProvider()));
            Bus.Subscribe <TimerMessage.Schedule>(TimerService);

            MainQueue.Start();
            monitoringQueue.Start();
        }
 public void the_kept_stream_is_present()
 {
     Assert.AreEqual(ReadEventResult.Success, ReadIndex.ReadEvent(_keptStream, 0).Result);
     Assert.AreEqual(ReadStreamResult.Success, ReadIndex.ReadStreamEventsForward(_keptStream, 0, 100).Result);
     Assert.AreEqual(ReadStreamResult.Success, ReadIndex.ReadStreamEventsBackward(_keptStream, -1, 100).Result);
 }
 public void indicate_that_any_stream_is_not_deleted()
 {
     Assert.That(ReadIndex.IsStreamDeleted("X"), Is.False);
     Assert.That(ReadIndex.IsStreamDeleted("YY"), Is.False);
     Assert.That(ReadIndex.IsStreamDeleted("ZZZ"), Is.False);
 }
Пример #29
0
 public void return_correct_last_event_version_for_smaller_stream()
 {
     Assert.AreEqual(1, ReadIndex.GetLastStreamEventNumber("ABC"));
 }
 public void indicate_that_nonexisting_streams_with_same_hashes_are_not_deleted()
 {
     Assert.That(ReadIndex.IsStreamDeleted("XXX"), Is.False);
     Assert.That(ReadIndex.IsStreamDeleted("XX"), Is.False);
 }
Пример #31
0
        /// <summary>
        /// Public method mapping Reads to Contigs.
        /// </summary>
        /// <param name="contigs">List of sequences of contigs.</param>
        /// <param name="reads">List of input reads.</param>
        /// <param name="kmerLength">Length of kmer.</param>
        /// <returns>Contig Read Map.</returns>
        public ReadContigMap Map(IList<ISequence> contigs, IEnumerable<ISequence> reads, int kmerLength)
        {
            KmerIndexerDictionary map = SequenceToKmerBuilder.BuildKmerDictionary(contigs, kmerLength);
            ReadContigMap maps = new ReadContigMap();
            Parallel.ForEach(reads, readSequence =>
            {
                IEnumerable<ISequence> kmers = SequenceToKmerBuilder.GetKmerSequences(readSequence, kmerLength);
                ReadIndex read = new ReadIndex(readSequence);
                foreach (ISequence kmer in kmers)
                {
                    IList<KmerIndexer> positions;
                    if (map.TryGetValue(kmer, out positions) ||
                        map.TryGetValue(kmer.GetReverseComplementedSequence(), out positions))
                    {
                        read.ContigReadMatchIndexes.Add(positions);
                    }
                }

                IList<Task<IList<ReadMap>>> tasks =
                    new List<Task<IList<ReadMap>>>();

                // Stores information about contigs for which tasks has been generated.
                IList<long> visitedContigs = new List<long>();

                // Creates Task for every read in nodes for a given contig.
                for (int index = 0; index < read.ContigReadMatchIndexes.Count; index++)
                {
                    int readPosition = index;
                    foreach (KmerIndexer kmer in read.ContigReadMatchIndexes[index])
                    {
                        long contigIndex = kmer.SequenceIndex;
                        if (!visitedContigs.Contains(contigIndex))
                        {
                            visitedContigs.Add(contigIndex);
                            tasks.Add(
                                Task<IList<ReadMap>>.Factory.StartNew(
                                t => MapRead(
                                    readPosition, 
                                    read.ContigReadMatchIndexes, 
                                    contigIndex, 
                                    read.ReadSequence.Count, 
                                    kmerLength),
                                TaskCreationOptions.AttachedToParent));
                        }
                    }
                }

                var overlapMaps = new Dictionary<ISequence, IList<ReadMap>>();
                for (int index = 0; index < visitedContigs.Count; index++)
                {
                    overlapMaps.Add(contigs.ElementAt(visitedContigs[index]), tasks[index].Result);
                }

                lock (maps)
                {
                    if (!maps.ContainsKey(read.ReadSequence.ID))
                    {
                        maps.Add(read.ReadSequence.ID, overlapMaps);
                    }
                    else
                    {
                        throw new ArgumentException(
                            string.Format(CultureInfo.CurrentCulture, Resource.DuplicatingReadIds, read.ReadSequence.ID));
                    }
                }
            });

            return maps;
        }