Example #1
0
        public ReadIndex(IPublisher bus,
                         ObjectPool <ITransactionFileReader> readerPool,
                         ITableIndex tableIndex,
                         int streamInfoCacheCapacity,
                         bool additionalCommitChecks,
                         long metastreamMaxCount,
                         int hashCollisionReadLimit,
                         bool skipIndexScanOnReads,
                         ICheckpoint replicationCheckpoint)
        {
            Ensure.NotNull(bus, "bus");
            Ensure.NotNull(readerPool, "readerPool");
            Ensure.NotNull(tableIndex, "tableIndex");
            Ensure.Nonnegative(streamInfoCacheCapacity, "streamInfoCacheCapacity");
            Ensure.Positive(metastreamMaxCount, "metastreamMaxCount");
            Ensure.NotNull(replicationCheckpoint, "replicationCheckpoint");

            var metastreamMetadata = new StreamMetadata(maxCount: metastreamMaxCount);

            _indexBackend          = new IndexBackend(readerPool, streamInfoCacheCapacity, streamInfoCacheCapacity);
            _indexReader           = new IndexReader(_indexBackend, tableIndex, metastreamMetadata, hashCollisionReadLimit, skipIndexScanOnReads);
            _indexWriter           = new IndexWriter(_indexBackend, _indexReader);
            _indexCommitter        = new IndexCommitter(bus, _indexBackend, _indexReader, tableIndex, additionalCommitChecks);
            _allReader             = new AllReader(_indexBackend, _indexCommitter, replicationCheckpoint);
            _replicationCheckpoint = replicationCheckpoint;
        }
Example #2
0
 public AllReader(IIndexBackend backend, IIndexCommitter indexCommitter)
 {
     Ensure.NotNull(backend, "backend");
     Ensure.NotNull(indexCommitter, "indexCommitter");
     _backend        = backend;
     _indexCommitter = indexCommitter;
 }
Example #3
0
 public AllReader(IIndexBackend backend, IIndexCommitter indexCommitter)
 {
     Ensure.NotNull(backend, "backend");
     Ensure.NotNull(indexCommitter, "indexCommitter");
     _backend = backend;
     _indexCommitter = indexCommitter;
 }
        public override async Task TestFixtureSetUp()
        {
            await base.TestFixtureSetUp();

            _logFormat = LogFormatHelper <TLogFormat, TStreamId> .LogFormatFactory.Create(new() {
                IndexDirectory = GetFilePathFor("index"),
            });

            _provider   = _logFormat.StreamNamesProvider;
            _publisher  = new InMemoryBus("publisher");
            _tfReader   = new FakeInMemoryTfReader(RecordOffset);
            _tableIndex = new FakeInMemoryTableIndex <TStreamId>();
            _provider.SetTableIndex(_tableIndex);
            _readerPool = new ObjectPool <ITransactionFileReader>(
                "ReadIndex readers pool", 5, 100,
                () => _tfReader);
            _indexBackend = new IndexBackend <TStreamId>(_readerPool, 100000, 100000);
            _streamIds    = _logFormat.StreamIds;
            _validator    = _logFormat.StreamIdValidator;
            var emptyStreamId = _logFormat.EmptyStreamId;

            _sizer       = _logFormat.StreamIdSizer;
            _indexReader = new IndexReader <TStreamId>(_indexBackend, _tableIndex, _provider, _validator,
                                                       _logFormat.StreamExistenceFilterReader, new StreamMetadata(maxCount: 100000), 100, false);
            _streamNames   = _logFormat.StreamNames;
            _systemStreams = _logFormat.SystemStreams;
            _indexWriter   = new IndexWriter <TStreamId>(_indexBackend, _indexReader, _streamIds, _streamNames,
                                                         _systemStreams, emptyStreamId, _sizer);
            _indexCommitter = new IndexCommitter <TStreamId>(_publisher, _indexBackend, _indexReader, _tableIndex,
                                                             _logFormat.StreamNameIndexConfirmer, _streamNames, _logFormat.EventTypeIndexConfirmer, _logFormat.EventTypes,
                                                             _systemStreams, _logFormat.StreamExistenceFilter, _logFormat.StreamExistenceFilterInitializer, new InMemoryCheckpoint(-1), false);

            WriteEvents();
        }
        public virtual void TestFixtureSetUp()
        {
            _logFormat  = LogFormatHelper <TLogFormat, TStreamId> .LogFormat;
            _publisher  = new InMemoryBus("publisher");
            _tfReader   = new FakeInMemoryTfReader(RecordOffset);
            _tableIndex = new FakeInMemoryTableIndex <TStreamId>();
            _readerPool = new ObjectPool <ITransactionFileReader>(
                "ReadIndex readers pool", 5, 100,
                () => _tfReader);
            _indexBackend = new IndexBackend <TStreamId>(_readerPool, 100000, 100000);
            var logFormat = LogFormatHelper <TLogFormat, TStreamId> .LogFormat;

            _streamIds     = logFormat.StreamIds;
            _streamNames   = logFormat.StreamNames;
            _systemStreams = logFormat.SystemStreams;
            _factory       = logFormat.StreamNamesProvider;
            _validator     = logFormat.StreamIdValidator;
            var emptyStreamId = logFormat.EmptyStreamId;

            _sizer          = logFormat.StreamIdSizer;
            _indexReader    = new IndexReader <TStreamId>(_indexBackend, _tableIndex, _factory, _validator, new StreamMetadata(maxCount: 100000), 100, false);
            _indexWriter    = new IndexWriter <TStreamId>(_indexBackend, _indexReader, _streamIds, _streamNames, _systemStreams, emptyStreamId, _sizer);
            _indexCommitter = new Core.Services.Storage.ReaderIndex.IndexCommitter <TStreamId>(_publisher, _indexBackend, _indexReader, _tableIndex, _streamNames, _systemStreams, new InMemoryCheckpoint(-1), false);

            WriteEvents();
        }
Example #6
0
        public ReadIndex(IPublisher bus,
                         ObjectPool <ITransactionFileReader> readerPool,
                         ITableIndex tableIndex,
                         IHasher hasher,
                         int streamInfoCacheCapacity,
                         bool additionalCommitChecks,
                         int metastreamMaxCount)
        {
            Ensure.NotNull(bus, "bus");
            Ensure.NotNull(readerPool, "readerPool");
            Ensure.NotNull(tableIndex, "tableIndex");
            Ensure.NotNull(hasher, "hasher");
            Ensure.Nonnegative(streamInfoCacheCapacity, "streamInfoCacheCapacity");
            Ensure.Positive(metastreamMaxCount, "metastreamMaxCount");

            var metastreamMetadata = new StreamMetadata(maxCount: metastreamMaxCount);

            _indexCache  = new IndexCache(readerPool, streamInfoCacheCapacity, streamInfoCacheCapacity);
            _indexReader = new IndexReader(_indexCache, hasher, tableIndex, metastreamMetadata);
            var writer = new IndexWriter(bus, tableIndex, hasher, _indexCache, _indexReader, additionalCommitChecks);

            _indexWriter    = writer;
            _indexCommitter = writer;
            _allReader      = new AllReader(_indexCache);
        }
Example #7
0
 public AllReader(IIndexBackend backend, IIndexCommitter indexCommitter, INameLookup <TStreamId> streamNames)
 {
     Ensure.NotNull(backend, "backend");
     Ensure.NotNull(indexCommitter, "indexCommitter");
     Ensure.NotNull(indexCommitter, nameof(streamNames));
     _backend        = backend;
     _indexCommitter = indexCommitter;
     _streamNames    = streamNames;
 }
Example #8
0
 public AllReader(IIndexBackend backend, IIndexCommitter indexCommitter, ICheckpoint replicationCheckpoint)
 {
     Ensure.NotNull(backend, "backend");
     Ensure.NotNull(indexCommitter, "indexCommitter");
     Ensure.NotNull(replicationCheckpoint, "replicationCheckpoint");
     _backend               = backend;
     _indexCommitter        = indexCommitter;
     _replicationCheckpoint = replicationCheckpoint;
 }
Example #9
0
        public ReadIndex(IPublisher bus,
                         ObjectPool <ITransactionFileReader> readerPool,
                         ITableIndex <TStreamId> tableIndex,
                         INameIndexConfirmer <TStreamId> streamNameIndex,
                         IValueLookup <TStreamId> streamIds,
                         IStreamNamesProvider <TStreamId> streamNamesProvider,
                         TStreamId emptyStreamName,
                         IValidator <TStreamId> streamIdValidator,
                         ISizer <TStreamId> sizer,
                         INameExistenceFilter streamExistenceFilter,
                         IExistenceFilterReader <TStreamId> streamExistenceFilterReader,
                         INameIndexConfirmer <TStreamId> eventTypeIndex,
                         int streamInfoCacheCapacity,
                         bool additionalCommitChecks,
                         long metastreamMaxCount,
                         int hashCollisionReadLimit,
                         bool skipIndexScanOnReads,
                         IReadOnlyCheckpoint replicationCheckpoint,
                         ICheckpoint indexCheckpoint)
        {
            Ensure.NotNull(bus, "bus");
            Ensure.NotNull(readerPool, "readerPool");
            Ensure.NotNull(tableIndex, "tableIndex");
            Ensure.NotNull(streamIds, nameof(streamIds));
            Ensure.NotNull(streamNamesProvider, nameof(streamNamesProvider));
            Ensure.NotNull(streamIdValidator, nameof(streamIdValidator));
            Ensure.NotNull(sizer, nameof(sizer));
            Ensure.NotNull(streamExistenceFilter, nameof(streamExistenceFilter));
            Ensure.NotNull(streamExistenceFilterReader, nameof(streamExistenceFilterReader));

            Ensure.Nonnegative(streamInfoCacheCapacity, "streamInfoCacheCapacity");
            Ensure.Positive(metastreamMaxCount, "metastreamMaxCount");
            Ensure.NotNull(replicationCheckpoint, "replicationCheckpoint");
            Ensure.NotNull(indexCheckpoint, "indexCheckpoint");

            var metastreamMetadata = new StreamMetadata(maxCount: metastreamMaxCount);

            var indexBackend = new IndexBackend <TStreamId>(readerPool, streamInfoCacheCapacity, streamInfoCacheCapacity);

            _indexReader = new IndexReader <TStreamId>(indexBackend, tableIndex, streamNamesProvider, streamIdValidator,
                                                       streamExistenceFilterReader, metastreamMetadata, hashCollisionReadLimit, skipIndexScanOnReads);

            _streamIds   = streamIds;
            _streamNames = streamNamesProvider.StreamNames;
            var systemStreams  = streamNamesProvider.SystemStreams;
            var eventTypeNames = streamNamesProvider.EventTypes;
            var streamExistenceFilterInitializer = streamNamesProvider.StreamExistenceFilterInitializer;

            _indexWriter    = new IndexWriter <TStreamId>(indexBackend, _indexReader, _streamIds, _streamNames, systemStreams, emptyStreamName, sizer);
            _indexCommitter = new IndexCommitter <TStreamId>(bus, indexBackend, _indexReader, tableIndex, streamNameIndex,
                                                             _streamNames, eventTypeIndex, eventTypeNames, systemStreams, streamExistenceFilter,
                                                             streamExistenceFilterInitializer, indexCheckpoint, additionalCommitChecks);
            _allReader = new AllReader <TStreamId>(indexBackend, _indexCommitter, _streamNames, eventTypeNames);
        }
        public IndexCommitterService(IIndexCommitter indexCommitter, IPublisher publisher, ICheckpoint replicationCheckpoint, ICheckpoint writerCheckpoint, int commitCount)
        {
            Ensure.NotNull(indexCommitter, "indexCommitter");
            Ensure.NotNull(publisher, "publisher");
            Ensure.NotNull(replicationCheckpoint, "replicationCheckpoint");
            Ensure.NotNull(writerCheckpoint, "writerCheckpoint");
            Ensure.Positive(commitCount, "commitCount");

            _indexCommitter        = indexCommitter;
            _publisher             = publisher;
            _replicationCheckpoint = replicationCheckpoint;
            _writerCheckpoint      = writerCheckpoint;
            _commitCount           = commitCount;
        }
Example #11
0
        public virtual void TestFixtureSetUp()
        {
            _publisher  = new InMemoryBus("publisher");
            _tfReader   = new FakeInMemoryTfReader(RecordOffset);
            _tableIndex = new FakeInMemoryTableIndex();
            _readerPool = new ObjectPool <ITransactionFileReader>(
                "ReadIndex readers pool", 5, 100,
                () => _tfReader);
            _indexBackend   = new IndexBackend(_readerPool, 100000, 100000);
            _indexReader    = new IndexReader(_indexBackend, _tableIndex, new StreamMetadata(maxCount: 100000), 100, false);
            _indexWriter    = new IndexWriter(_indexBackend, _indexReader);
            _indexCommitter = new IndexCommitter(_publisher, _indexBackend, _indexReader, _tableIndex, false);

            WriteEvents();
        }
        public IndexCommitterService(IIndexCommitter indexCommitter, IPublisher publisher,
                                     ICheckpoint replicationCheckpoint, ICheckpoint writerCheckpoint, int commitCount, ITableIndex tableIndex, QueueStatsManager queueStatsManager)
        {
            Ensure.NotNull(indexCommitter, "indexCommitter");
            Ensure.NotNull(publisher, "publisher");
            Ensure.NotNull(replicationCheckpoint, "replicationCheckpoint");
            Ensure.NotNull(writerCheckpoint, "writerCheckpoint");
            Ensure.Positive(commitCount, "commitCount");

            _indexCommitter        = indexCommitter;
            _publisher             = publisher;
            _replicationCheckpoint = replicationCheckpoint;
            _writerCheckpoint      = writerCheckpoint;
            _commitCount           = commitCount;
            _tableIndex            = tableIndex;
            _queueStats            = queueStatsManager.CreateQueueStatsCollector("Index Committer");
        }
Example #13
0
        public StorageChaser(IPublisher masterBus,
                             ICheckpoint writerCheckpoint,
                             ITransactionFileChaser chaser,
                             IIndexCommitter indexCommitter,
                             IEpochManager epochManager)
        {
            Ensure.NotNull(masterBus, "masterBus");
            Ensure.NotNull(writerCheckpoint, "writerCheckpoint");
            Ensure.NotNull(chaser, "chaser");
            Ensure.NotNull(indexCommitter, "indexCommitter");
            Ensure.NotNull(epochManager, "epochManager");

            _masterBus        = masterBus;
            _writerCheckpoint = writerCheckpoint;
            _chaser           = chaser;
            _indexCommitter   = indexCommitter;
            _epochManager     = epochManager;

            _flushDelay = 0;
            _lastFlush  = _watch.ElapsedTicks;
        }
Example #14
0
        public ReadIndex(IPublisher bus,
                         ObjectPool<ITransactionFileReader> readerPool,
                         ITableIndex tableIndex,
                         int streamInfoCacheCapacity,
                         bool additionalCommitChecks,
                         int metastreamMaxCount,
                         int hashCollisionReadLimit)
        {
            Ensure.NotNull(bus, "bus");
            Ensure.NotNull(readerPool, "readerPool");
            Ensure.NotNull(tableIndex, "tableIndex");
            Ensure.Nonnegative(streamInfoCacheCapacity, "streamInfoCacheCapacity");
            Ensure.Positive(metastreamMaxCount, "metastreamMaxCount");

            var metastreamMetadata = new StreamMetadata(maxCount: metastreamMaxCount);

            _indexBackend = new IndexBackend(readerPool, streamInfoCacheCapacity, streamInfoCacheCapacity);
            _indexReader = new IndexReader(_indexBackend, tableIndex, metastreamMetadata, hashCollisionReadLimit);
            _indexWriter = new IndexWriter(_indexBackend, _indexReader);
            _indexCommitter = new IndexCommitter(bus, _indexBackend, _indexReader, tableIndex, additionalCommitChecks);
            _allReader = new AllReader(_indexBackend, _indexCommitter);
        }
        public ReadIndex(IPublisher bus,
                         ObjectPool<ITransactionFileReader> readerPool,
                         ITableIndex tableIndex,
                         IHasher hasher,
                         int streamInfoCacheCapacity,
                         bool additionalCommitChecks,
                         int metastreamMaxCount)
        {
            Ensure.NotNull(bus, "bus");
            Ensure.NotNull(readerPool, "readerPool");
            Ensure.NotNull(tableIndex, "tableIndex");
            Ensure.NotNull(hasher, "hasher");
            Ensure.Nonnegative(streamInfoCacheCapacity, "streamInfoCacheCapacity");
            Ensure.Positive(metastreamMaxCount, "metastreamMaxCount");

            var metastreamMetadata = new StreamMetadata(maxCount: metastreamMaxCount);

            _indexCache = new IndexCache(readerPool, streamInfoCacheCapacity, streamInfoCacheCapacity);
            _indexReader = new IndexReader(_indexCache, hasher, tableIndex, metastreamMetadata);
            var writer = new IndexWriter(bus, tableIndex, hasher, _indexCache, _indexReader, additionalCommitChecks);
            _indexWriter = writer;
            _indexCommitter = writer;
            _allReader = new AllReader(_indexCache);
        }