public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            WriterCheckpoint = new InMemoryCheckpoint(0);
            var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0);

            Db = new TFChunkDb(new TFChunkDbConfig(PathName,
                                                   new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
                                                   10000,
                                                   0,
                                                   WriterCheckpoint,
                                                   new[] { chaserchk }));
            Db.OpenVerifyAndClean();
            // create db
            Writer = new TFChunkWriter(Db);
            Writer.Open();
            WriteTestScenario();
            Writer.Close();
            Writer = null;

            WriterCheckpoint.Flush();
            chaserchk.Write(WriterCheckpoint.Read());
            chaserchk.Flush();

            TableIndex = new TableIndex(Path.Combine(PathName, "index"), () => new HashListMemTable(), _maxEntriesInMemTable);
            TableIndex.Initialize();

            var reader = new TFChunkReader(Db, Db.Config.WriterCheckpoint);

            ReadIndex = new ReadIndex(new NoopPublisher(), 2, () => new TFChunkSequentialReader(Db, Db.Config.WriterCheckpoint, 0), () => reader, TableIndex, new ByLengthHasher());
            ReadIndex.Build();
        }
Example #2
0
        public void try_read_returns_false_when_writer_checksum_is_equal_to_reader_checksum()
        {
            var writerchk = new InMemoryCheckpoint();
            var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0);
            var db        = new TFChunkDb(new TFChunkDbConfig(PathName,
                                                              new PrefixFileNamingStrategy(PathName, "prefix.tf"),
                                                              10000,
                                                              0,
                                                              writerchk,
                                                              chaserchk,
                                                              new[] { writerchk, chaserchk }));

            db.OpenVerifyAndClean();
            writerchk.Write(12);
            writerchk.Flush();
            chaserchk.Write(12);
            chaserchk.Flush();

            var chaser = new TFChunkChaser(db, writerchk, chaserchk);

            chaser.Open();

            LogRecord record;

            Assert.IsFalse(chaser.TryReadNext(out record));
            Assert.AreEqual(12, chaserchk.Read());

            chaser.Close();
            db.Dispose();
        }
        public void try_read_returns_false_when_writer_checksum_is_equal_to_reader_checksum()
        {
            var writerchk = new InMemoryCheckpoint();
            var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0);
            var db        = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, writerchk, chaserchk));

            db.Open();
            writerchk.Write(12);
            writerchk.Flush();
            chaserchk.Write(12);
            chaserchk.Flush();

            var chaser = new TFChunkChaser(db, writerchk, chaserchk, false);

            chaser.Open();

            ILogRecord record;

            Assert.IsFalse(chaser.TryReadNext(out record));
            Assert.AreEqual(12, chaserchk.Read());

            chaser.Close();
            db.Dispose();
        }
        public override async Task TestFixtureSetUp()
        {
            await base.TestFixtureSetUp();

            var indexDirectory = GetFilePathFor("index");

            _logFormat = LogFormatHelper <TLogFormat, TStreamId> .LogFormatFactory.Create(new() {
                IndexDirectory = indexDirectory,
            });

            var writerCheckpoint = new InMemoryCheckpoint(0);
            var chaserCheckpoint = new InMemoryCheckpoint(0);

            var bus = new InMemoryBus("bus");

            new IODispatcher(bus, new PublishEnvelope(bus));

            _db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, writerCheckpoint, chaserCheckpoint));

            _db.Open();
            // create db
            Writer = new TFChunkWriter(_db);
            Writer.Open();
            SetupDB();
            Writer.Close();
            Writer = null;

            writerCheckpoint.Flush();
            chaserCheckpoint.Write(writerCheckpoint.Read());
            chaserCheckpoint.Flush();

            var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 5,
                                                                  () => new TFChunkReader(_db, _db.Config.WriterCheckpoint));
            var lowHasher     = _logFormat.LowHasher;
            var highHasher    = _logFormat.HighHasher;
            var emptyStreamId = _logFormat.EmptyStreamId;

            _tableIndex = new TableIndex <TStreamId>(indexDirectory, lowHasher, highHasher, emptyStreamId,
                                                     () => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2),
                                                     () => new TFReaderLease(readers),
                                                     IndexBitnessVersion,
                                                     int.MaxValue,
                                                     Constants.PTableMaxReaderCountDefault,
                                                     MaxEntriesInMemTable);
            _logFormat.StreamNamesProvider.SetTableIndex(_tableIndex);

            var readIndex = new ReadIndex <TStreamId>(new NoopPublisher(),
                                                      readers,
                                                      _tableIndex,
                                                      _logFormat.StreamNameIndexConfirmer,
                                                      _logFormat.StreamIds,
                                                      _logFormat.StreamNamesProvider,
                                                      _logFormat.EmptyStreamId,
                                                      _logFormat.StreamIdValidator,
                                                      _logFormat.StreamIdSizer,
                                                      _logFormat.StreamExistenceFilter,
                                                      _logFormat.StreamExistenceFilterReader,
                                                      _logFormat.EventTypeIndexConfirmer,
                                                      streamInfoCacheCapacity: 100_000,
                                                      additionalCommitChecks: PerformAdditionalCommitChecks,
                                                      metastreamMaxCount: MetastreamMaxCount,
                                                      hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
                                                      skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
                                                      replicationCheckpoint: _db.Config.ReplicationCheckpoint,
                                                      indexCheckpoint: _db.Config.IndexCheckpoint);


            readIndex.IndexCommitter.Init(chaserCheckpoint.Read());
            ReadIndex = readIndex;

            _tableIndex.Close(false);

            Writer = new TFChunkWriter(_db);
            Writer.Open();
            Given();
            Writer.Close();
            Writer = null;

            writerCheckpoint.Flush();
            chaserCheckpoint.Write(writerCheckpoint.Read());
            chaserCheckpoint.Flush();

            _tableIndex = new TableIndex <TStreamId>(indexDirectory, lowHasher, highHasher, emptyStreamId,
                                                     () => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2),
                                                     () => new TFReaderLease(readers),
                                                     IndexBitnessVersion,
                                                     int.MaxValue,
                                                     Constants.PTableMaxReaderCountDefault,
                                                     MaxEntriesInMemTable);

            readIndex = new ReadIndex <TStreamId>(new NoopPublisher(),
                                                  readers,
                                                  _tableIndex,
                                                  _logFormat.StreamNameIndexConfirmer,
                                                  _logFormat.StreamIds,
                                                  _logFormat.StreamNamesProvider,
                                                  _logFormat.EmptyStreamId,
                                                  _logFormat.StreamIdValidator,
                                                  _logFormat.StreamIdSizer,
                                                  _logFormat.StreamExistenceFilter,
                                                  _logFormat.StreamExistenceFilterReader,
                                                  _logFormat.EventTypeIndexConfirmer,
                                                  streamInfoCacheCapacity: 100_000,
                                                  additionalCommitChecks: PerformAdditionalCommitChecks,
                                                  metastreamMaxCount: MetastreamMaxCount,
                                                  hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
                                                  skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
                                                  replicationCheckpoint: _db.Config.ReplicationCheckpoint,
                                                  indexCheckpoint: _db.Config.IndexCheckpoint);

            readIndex.IndexCommitter.Init(chaserCheckpoint.Read());
            ReadIndex = readIndex;
        }
Example #5
0
        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            var writerCheckpoint = new InMemoryCheckpoint(0);
            var chaserCheckpoint = new InMemoryCheckpoint(0);

            var bus = new InMemoryBus("bus");

            new IODispatcher(bus, new PublishEnvelope(bus));

            _db = new TFChunkDb(new TFChunkDbConfig(PathName,
                                                    new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
                                                    10000,
                                                    0,
                                                    writerCheckpoint,
                                                    chaserCheckpoint,
                                                    new InMemoryCheckpoint(-1),
                                                    new InMemoryCheckpoint(-1)));

            _db.Open();
            // create db
            Writer = new TFChunkWriter(_db);
            Writer.Open();
            SetupDB();
            Writer.Close();
            Writer = null;

            writerCheckpoint.Flush();
            chaserCheckpoint.Write(writerCheckpoint.Read());
            chaserCheckpoint.Flush();

            var readers    = new ObjectPool <ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(_db, _db.Config.WriterCheckpoint));
            var lowHasher  = new XXHashUnsafe();
            var highHasher = new Murmur3AUnsafe();

            _tableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
                                         () => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2),
                                         () => new TFReaderLease(readers),
                                         IndexBitnessVersion,
                                         MaxEntriesInMemTable);

            ReadIndex = new ReadIndex(new NoopPublisher(),
                                      readers,
                                      _tableIndex,
                                      EventStore.Core.Settings.ESConsts.StreamInfoCacheCapacity,
                                      additionalCommitChecks: PerformAdditionalCommitChecks,
                                      metastreamMaxCount: MetastreamMaxCount,
                                      hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
                                      skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault);


            ReadIndex.Init(chaserCheckpoint.Read());

            _tableIndex.Close(false);

            Writer = new TFChunkWriter(_db);
            Writer.Open();
            Given();
            Writer.Close();
            Writer = null;

            writerCheckpoint.Flush();
            chaserCheckpoint.Write(writerCheckpoint.Read());
            chaserCheckpoint.Flush();

            _tableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
                                         () => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2),
                                         () => new TFReaderLease(readers),
                                         IndexBitnessVersion,
                                         MaxEntriesInMemTable);

            ReadIndex = new ReadIndex(new NoopPublisher(),
                                      readers,
                                      _tableIndex,
                                      EventStore.Core.Settings.ESConsts.StreamInfoCacheCapacity,
                                      additionalCommitChecks: PerformAdditionalCommitChecks,
                                      metastreamMaxCount: MetastreamMaxCount,
                                      hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
                                      skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault);

            ReadIndex.Init(chaserCheckpoint.Read());
        }