public ReadIndex(IPublisher bus, ObjectPool <ITransactionFileReader> readerPool, ITableIndex tableIndex, int streamInfoCacheCapacity, bool additionalCommitChecks, long metastreamMaxCount, int hashCollisionReadLimit, bool skipIndexScanOnReads, IReadOnlyCheckpoint replicationCheckpoint, ICheckpoint indexCheckpoint) { Ensure.NotNull(bus, "bus"); Ensure.NotNull(readerPool, "readerPool"); Ensure.NotNull(tableIndex, "tableIndex"); Ensure.Nonnegative(streamInfoCacheCapacity, "streamInfoCacheCapacity"); Ensure.Positive(metastreamMaxCount, "metastreamMaxCount"); Ensure.NotNull(replicationCheckpoint, "replicationCheckpoint"); Ensure.NotNull(indexCheckpoint, "indexCheckpoint"); var metastreamMetadata = new StreamMetadata(maxCount: metastreamMaxCount); IIndexBackend indexBackend = new IndexBackend(readerPool, streamInfoCacheCapacity, streamInfoCacheCapacity); _indexReader = new IndexReader(indexBackend, tableIndex, metastreamMetadata, hashCollisionReadLimit, skipIndexScanOnReads); _indexWriter = new IndexWriter(indexBackend, _indexReader); _indexCommitter = new IndexCommitter(bus, indexBackend, _indexReader, tableIndex, indexCheckpoint, additionalCommitChecks); _allReader = new AllReader(indexBackend, _indexCommitter); }
public NodeGossipService(IPublisher bus, IGossipSeedSource gossipSeedSource, MemberInfo memberInfo, IReadOnlyCheckpoint writerCheckpoint, IReadOnlyCheckpoint chaserCheckpoint, IEpochManager epochManager, Func <long> getLastCommitPosition, int nodePriority, TimeSpan gossipInterval, TimeSpan allowedTimeDifference, TimeSpan gossipTimeout, TimeSpan deadMemberRemovalPeriod, ITimeProvider timeProvider, Func <MemberInfo[], MemberInfo> getNodeToGossipTo = null) : base(bus, gossipSeedSource, memberInfo, gossipInterval, allowedTimeDifference, gossipTimeout, deadMemberRemovalPeriod, timeProvider, getNodeToGossipTo) { Ensure.NotNull(writerCheckpoint, nameof(writerCheckpoint)); Ensure.NotNull(chaserCheckpoint, nameof(chaserCheckpoint)); Ensure.NotNull(epochManager, nameof(epochManager)); Ensure.NotNull(getLastCommitPosition, nameof(getLastCommitPosition)); _writerCheckpoint = writerCheckpoint; _chaserCheckpoint = chaserCheckpoint; _epochManager = epochManager; _getLastCommitPosition = getLastCommitPosition; _nodePriority = nodePriority; _timeProvider = timeProvider; }
public MonitoringService(IQueuedHandler monitoringQueue, IPublisher statsCollectionBus, IPublisher mainBus, IReadOnlyCheckpoint writerCheckpoint, string dbPath, TimeSpan statsCollectionPeriod, EndPoint nodeEndpoint, StatsStorage statsStorage, IPEndPoint tcpEndpoint, IPEndPoint tcpSecureEndpoint) { Ensure.NotNull(monitoringQueue, "monitoringQueue"); Ensure.NotNull(statsCollectionBus, "statsCollectionBus"); Ensure.NotNull(mainBus, "mainBus"); Ensure.NotNull(writerCheckpoint, "writerCheckpoint"); Ensure.NotNullOrEmpty(dbPath, "dbPath"); Ensure.NotNull(nodeEndpoint, "nodeEndpoint"); _monitoringQueue = monitoringQueue; _statsCollectionBus = statsCollectionBus; _mainBus = mainBus; _writerCheckpoint = writerCheckpoint; _dbPath = dbPath; _statsStorage = statsStorage; _statsCollectionPeriodMs = statsCollectionPeriod > TimeSpan.Zero ? (long)statsCollectionPeriod.TotalMilliseconds : Timeout.Infinite; _nodeStatsStream = string.Format("{0}-{1}", SystemStreams.StatsStreamPrefix, nodeEndpoint); _tcpEndpoint = tcpEndpoint; _tcpSecureEndpoint = tcpSecureEndpoint; _timer = new Timer(OnTimerTick, null, Timeout.Infinite, Timeout.Infinite); _systemStats = new SystemStatsHelper(Log, _writerCheckpoint, _dbPath, _statsCollectionPeriodMs); }
public void InitializeCheckpoints( IReadOnlyCheckpoint writerCheckpoint, IReadOnlyCheckpoint chaserCheckpoint) { _writerCheckpoint = writerCheckpoint; _chaserCheckpoint = chaserCheckpoint; }
public StorageReaderService( IPublisher bus, ISubscriber subscriber, IReadIndex readIndex, int threadCount, IReadOnlyCheckpoint writerCheckpoint, QueueStatsManager queueStatsManager) { Ensure.NotNull(bus, "bus"); Ensure.NotNull(subscriber, "subscriber"); Ensure.NotNull(readIndex, "readIndex"); Ensure.Positive(threadCount, "threadCount"); Ensure.NotNull(writerCheckpoint, "writerCheckpoint"); _bus = bus; _readIndex = readIndex; _threadCount = threadCount; StorageReaderWorker[] readerWorkers = new StorageReaderWorker[threadCount]; InMemoryBus[] storageReaderBuses = new InMemoryBus[threadCount]; for (var i = 0; i < threadCount; i++) { readerWorkers[i] = new StorageReaderWorker(bus, readIndex, writerCheckpoint, i); storageReaderBuses[i] = new InMemoryBus("StorageReaderBus", watchSlowMsg: false); storageReaderBuses[i].Subscribe <ClientMessage.ReadEvent>(readerWorkers[i]); storageReaderBuses[i].Subscribe <ClientMessage.ReadStreamEventsBackward>(readerWorkers[i]); storageReaderBuses[i].Subscribe <ClientMessage.ReadStreamEventsForward>(readerWorkers[i]); storageReaderBuses[i].Subscribe <ClientMessage.ReadAllEventsForward>(readerWorkers[i]); storageReaderBuses[i].Subscribe <ClientMessage.ReadAllEventsBackward>(readerWorkers[i]); storageReaderBuses[i].Subscribe <ClientMessage.FilteredReadAllEventsForward>(readerWorkers[i]); storageReaderBuses[i].Subscribe <ClientMessage.FilteredReadAllEventsBackward>(readerWorkers[i]); storageReaderBuses[i].Subscribe <StorageMessage.BatchLogExpiredMessages>(readerWorkers[i]); storageReaderBuses[i].Subscribe <StorageMessage.EffectiveStreamAclRequest>(readerWorkers[i]); storageReaderBuses[i].Subscribe <StorageMessage.StreamIdFromTransactionIdRequest>(readerWorkers[i]); } _workersMultiHandler = new MultiQueuedHandler( _threadCount, queueNum => new QueuedHandlerThreadPool(storageReaderBuses[queueNum], string.Format("StorageReaderQueue #{0}", queueNum + 1), queueStatsManager, groupName: "StorageReaderQueue", watchSlowMsg: true, slowMsgThreshold: TimeSpan.FromMilliseconds(200))); _workersMultiHandler.Start(); subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadEvent, Message>()); subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadStreamEventsBackward, Message>()); subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadStreamEventsForward, Message>()); subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadAllEventsForward, Message>()); subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadAllEventsBackward, Message>()); subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.FilteredReadAllEventsForward, Message>()); subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.FilteredReadAllEventsBackward, Message>()); subscriber.Subscribe(_workersMultiHandler.WidenFrom <StorageMessage.BatchLogExpiredMessages, Message>()); subscriber.Subscribe(_workersMultiHandler.WidenFrom <StorageMessage.EffectiveStreamAclRequest, Message>()); subscriber.Subscribe(_workersMultiHandler.WidenFrom <StorageMessage.StreamIdFromTransactionIdRequest, Message>()); }
public TFChunkChaser(TFChunkDb db, IReadOnlyCheckpoint writerCheckpoint, ICheckpoint chaserCheckpoint, bool optimizeReadSideCache) { Ensure.NotNull(db, "dbConfig"); Ensure.NotNull(writerCheckpoint, "writerCheckpoint"); Ensure.NotNull(chaserCheckpoint, "chaserCheckpoint"); _chaserCheckpoint = chaserCheckpoint; _reader = new TFChunkReader(db, writerCheckpoint, _chaserCheckpoint.Read(), optimizeReadSideCache); }
public ReadIndex(IPublisher bus, ObjectPool <ITransactionFileReader> readerPool, ITableIndex <TStreamId> tableIndex, INameIndexConfirmer <TStreamId> streamNameIndex, IValueLookup <TStreamId> streamIds, IStreamNamesProvider <TStreamId> streamNamesProvider, TStreamId emptyStreamName, IValidator <TStreamId> streamIdValidator, ISizer <TStreamId> sizer, INameExistenceFilter streamExistenceFilter, IExistenceFilterReader <TStreamId> streamExistenceFilterReader, INameIndexConfirmer <TStreamId> eventTypeIndex, int streamInfoCacheCapacity, bool additionalCommitChecks, long metastreamMaxCount, int hashCollisionReadLimit, bool skipIndexScanOnReads, IReadOnlyCheckpoint replicationCheckpoint, ICheckpoint indexCheckpoint) { Ensure.NotNull(bus, "bus"); Ensure.NotNull(readerPool, "readerPool"); Ensure.NotNull(tableIndex, "tableIndex"); Ensure.NotNull(streamIds, nameof(streamIds)); Ensure.NotNull(streamNamesProvider, nameof(streamNamesProvider)); Ensure.NotNull(streamIdValidator, nameof(streamIdValidator)); Ensure.NotNull(sizer, nameof(sizer)); Ensure.NotNull(streamExistenceFilter, nameof(streamExistenceFilter)); Ensure.NotNull(streamExistenceFilterReader, nameof(streamExistenceFilterReader)); Ensure.Nonnegative(streamInfoCacheCapacity, "streamInfoCacheCapacity"); Ensure.Positive(metastreamMaxCount, "metastreamMaxCount"); Ensure.NotNull(replicationCheckpoint, "replicationCheckpoint"); Ensure.NotNull(indexCheckpoint, "indexCheckpoint"); var metastreamMetadata = new StreamMetadata(maxCount: metastreamMaxCount); var indexBackend = new IndexBackend <TStreamId>(readerPool, streamInfoCacheCapacity, streamInfoCacheCapacity); _indexReader = new IndexReader <TStreamId>(indexBackend, tableIndex, streamNamesProvider, streamIdValidator, streamExistenceFilterReader, metastreamMetadata, hashCollisionReadLimit, skipIndexScanOnReads); _streamIds = streamIds; _streamNames = streamNamesProvider.StreamNames; var systemStreams = streamNamesProvider.SystemStreams; var eventTypeNames = streamNamesProvider.EventTypes; var streamExistenceFilterInitializer = streamNamesProvider.StreamExistenceFilterInitializer; _indexWriter = new IndexWriter <TStreamId>(indexBackend, _indexReader, _streamIds, _streamNames, systemStreams, emptyStreamName, sizer); _indexCommitter = new IndexCommitter <TStreamId>(bus, indexBackend, _indexReader, tableIndex, streamNameIndex, _streamNames, eventTypeIndex, eventTypeNames, systemStreams, streamExistenceFilter, streamExistenceFilterInitializer, indexCheckpoint, additionalCommitChecks); _allReader = new AllReader <TStreamId>(indexBackend, _indexCommitter, _streamNames, eventTypeNames); }
public SystemStatsHelper(ILogger log, IReadOnlyCheckpoint writerCheckpoint, string dbPath, long collectIntervalMs) { Ensure.NotNull(log, "log"); Ensure.NotNull(writerCheckpoint, "writerCheckpoint"); _log = log; _writerCheckpoint = writerCheckpoint; _perfCounter = new PerfCounterHelper(_log); _eventCountersHelper = new EventCountersHelper(collectIntervalMs); _hostStat = new HostStat.HostStat(); _dbPath = dbPath; }
public StorageReaderWorker(IPublisher publisher, IReadIndex readIndex, IReadOnlyCheckpoint writerCheckpoint, int queueId) { Ensure.NotNull(publisher, "publisher"); Ensure.NotNull(readIndex, "readIndex"); Ensure.NotNull(writerCheckpoint, "writerCheckpoint"); _publisher = publisher; _readIndex = readIndex; _writerCheckpoint = writerCheckpoint; _queueId = queueId; }
public ReplicationTrackingService( IPublisher publisher, int clusterNodeCount, ICheckpoint replicationCheckpoint, IReadOnlyCheckpoint writerCheckpoint) { Ensure.NotNull(publisher, nameof(publisher)); Ensure.NotNull(replicationCheckpoint, nameof(replicationCheckpoint)); Ensure.NotNull(writerCheckpoint, nameof(writerCheckpoint)); Ensure.Positive(clusterNodeCount, nameof(clusterNodeCount)); _publisher = publisher; _replicationCheckpoint = replicationCheckpoint; _writerCheckpoint = writerCheckpoint; _quorumSize = clusterNodeCount / 2 + 1; }
public InaugurationManager( IPublisher publisher, IReadOnlyCheckpoint replicationCheckpoint, IReadOnlyCheckpoint indexCheckpoint) { _log.Information("Using {name}", nameof(InaugurationManager)); _publisher = publisher; _replicationCheckpoint = replicationCheckpoint; _indexCheckpoint = indexCheckpoint; _scheduleCheckInaugurationConditions = TimerMessage.Schedule.Create( triggerAfter: TimeSpan.FromSeconds(1), envelope: new PublishEnvelope(_publisher), replyMessage: new SystemMessage.CheckInaugurationConditions()); }
public TFChunkReader(TFChunkDb db, IReadOnlyCheckpoint writerCheckpoint, long initialPosition = 0, bool optimizeReadSideCache = false) { Ensure.NotNull(db, "dbConfig"); Ensure.NotNull(writerCheckpoint, "writerCheckpoint"); Ensure.Nonnegative(initialPosition, "initialPosition"); _db = db; _writerCheckpoint = writerCheckpoint; _curPos = initialPosition; _optimizeReadSideCache = optimizeReadSideCache; if (_optimizeReadSideCache) { _existsAtOptimizer = TFChunkReaderExistsAtOptimizer.Instance; } }
public StorageChaser(IPublisher leaderBus, IReadOnlyCheckpoint writerCheckpoint, ITransactionFileChaser chaser, IIndexCommitterService indexCommitterService, IEpochManager epochManager, QueueStatsManager queueStatsManager) { Ensure.NotNull(leaderBus, "leaderBus"); Ensure.NotNull(writerCheckpoint, "writerCheckpoint"); Ensure.NotNull(chaser, "chaser"); Ensure.NotNull(indexCommitterService, "indexCommitterService"); Ensure.NotNull(epochManager, "epochManager"); _leaderBus = leaderBus; _writerCheckpoint = writerCheckpoint; _chaser = chaser; _indexCommitterService = indexCommitterService; _epochManager = epochManager; _queueStats = queueStatsManager.CreateQueueStatsCollector("Storage Chaser"); _flushDelay = 0; _lastFlush = _watch.ElapsedTicks; }
public IndexCommitterService( IIndexCommitter indexCommitter, IPublisher publisher, IReadOnlyCheckpoint writerCheckpoint, IReadOnlyCheckpoint replicationCheckpoint, int commitCount, ITableIndex tableIndex, QueueStatsManager queueStatsManager) { Ensure.NotNull(indexCommitter, nameof(indexCommitter)); Ensure.NotNull(publisher, nameof(publisher)); Ensure.NotNull(writerCheckpoint, nameof(writerCheckpoint)); Ensure.NotNull(replicationCheckpoint, nameof(replicationCheckpoint)); Ensure.Positive(commitCount, nameof(commitCount)); _indexCommitter = indexCommitter; _publisher = publisher; _writerCheckpoint = writerCheckpoint; _replicationCheckpoint = replicationCheckpoint; _commitCount = commitCount; _tableIndex = tableIndex; _queueStats = queueStatsManager.CreateQueueStatsCollector("Index Committer"); }
public ElectionsService(IPublisher publisher, MemberInfo memberInfo, int clusterSize, IReadOnlyCheckpoint writerCheckpoint, IReadOnlyCheckpoint chaserCheckpoint, ICheckpoint proposalCheckpoint, IEpochManager epochManager, Func <long> getLastCommitPosition, int nodePriority, ITimeProvider timeProvider) { Ensure.NotNull(publisher, nameof(publisher)); Ensure.NotNull(memberInfo, nameof(memberInfo)); Ensure.Positive(clusterSize, nameof(clusterSize)); Ensure.NotNull(writerCheckpoint, nameof(writerCheckpoint)); Ensure.NotNull(chaserCheckpoint, nameof(chaserCheckpoint)); Ensure.NotNull(proposalCheckpoint, nameof(proposalCheckpoint)); Ensure.NotNull(epochManager, nameof(epochManager)); Ensure.NotNull(getLastCommitPosition, nameof(getLastCommitPosition)); Ensure.NotNull(timeProvider, nameof(timeProvider)); if (memberInfo.IsReadOnlyReplica) { throw new ArgumentException("Read-only replicas are not allowed to run the Elections service."); } _publisher = publisher; _memberInfo = memberInfo; _publisherEnvelope = new PublishEnvelope(_publisher); _clusterSize = clusterSize; _writerCheckpoint = writerCheckpoint; _chaserCheckpoint = chaserCheckpoint; _proposalCheckpoint = proposalCheckpoint; _epochManager = epochManager; _getLastCommitPosition = getLastCommitPosition; _nodePriority = nodePriority; _timeProvider = timeProvider; var lastEpoch = _epochManager.LastEpochNumber; if (_proposalCheckpoint.Read() < lastEpoch) { _proposalCheckpoint.Write(lastEpoch); _proposalCheckpoint.Flush(); } var ownInfo = GetOwnInfo(); _servers = new[] { MemberInfo.ForVNode(memberInfo.InstanceId, _timeProvider.UtcNow, VNodeState.Initializing, true, memberInfo.InternalTcpEndPoint, memberInfo.InternalSecureTcpEndPoint, memberInfo.ExternalTcpEndPoint, memberInfo.ExternalSecureTcpEndPoint, memberInfo.HttpEndPoint, memberInfo.AdvertiseHostToClientAs, memberInfo.AdvertiseHttpPortToClientAs, memberInfo.AdvertiseTcpPortToClientAs, ownInfo.LastCommitPosition, ownInfo.WriterCheckpoint, ownInfo.ChaserCheckpoint, ownInfo.EpochPosition, ownInfo.EpochNumber, ownInfo.EpochId, ownInfo.NodePriority, memberInfo.IsReadOnlyReplica) }; }