Exemple #1
0
        private void ReOpenDb()
        {
            Db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, WriterCheckpoint, ChaserCheckpoint));

            Db.Open();

            var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 5,
                                                                  () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
            var lowHasher  = new XXHashUnsafe();
            var highHasher = new Murmur3AUnsafe();

            TableIndex = new TableIndex(Path.Combine(PathName, "index"), lowHasher, highHasher,
                                        () => new HashListMemTable(PTableVersions.IndexV3, MaxEntriesInMemTable * 2),
                                        () => new TFReaderLease(readers),
                                        PTableVersions.IndexV3,
                                        int.MaxValue,
                                        Constants.PTableMaxReaderCountDefault,
                                        MaxEntriesInMemTable);
            ReadIndex = new ReadIndex(new NoopPublisher(),
                                      readers,
                                      TableIndex,
                                      0,
                                      additionalCommitChecks: true,
                                      metastreamMaxCount: MetastreamMaxCount,
                                      hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
                                      skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
                                      replicationCheckpoint: Db.Config.ReplicationCheckpoint);
            ((ReadIndex)ReadIndex).IndexCommitter.Init(ChaserCheckpoint.Read());
        }
Exemple #2
0
        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            ReadIndex.Close();
            ReadIndex.Dispose();
            TableIndex.Close(removeFiles: false);

            var readers    = new ObjectPool <ITransactionFileReader>("Readers", 2, 2, () => new TFChunkReader(Db, WriterCheckpoint));
            var lowHasher  = new XXHashUnsafe();
            var highHasher = new Murmur3AUnsafe();

            TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
                                        () => new HashListMemTable(PTableVersions.IndexV2, maxSize: MaxEntriesInMemTable * 2),
                                        () => new TFReaderLease(readers),
                                        PTableVersions.IndexV2,
                                        5,
                                        maxSizeForMemory: MaxEntriesInMemTable);
            ReadIndex = new ReadIndex(new NoopPublisher(),
                                      readers,
                                      TableIndex,
                                      0,
                                      additionalCommitChecks: true,
                                      metastreamMaxCount: 1,
                                      hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
                                      skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
                                      replicationCheckpoint: Db.Config.ReplicationCheckpoint);
            ReadIndex.Init(ChaserCheckpoint.Read());
        }
Exemple #3
0
        private void ReOpenDb()
        {
            Db = new TFChunkDb(new TFChunkDbConfig(PathName,
                                                   new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
                                                   10000,
                                                   0,
                                                   WriterCheckpoint,
                                                   ChaserCheckpoint,
                                                   new InMemoryCheckpoint(-1),
                                                   new InMemoryCheckpoint(-1)));

            Db.Open();

            var readers    = new ObjectPool <ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
            var lowHasher  = new XXHashUnsafe();
            var highHasher = new Murmur3AUnsafe();

            TableIndex = new TableIndex(Path.Combine(PathName, "index"), lowHasher, highHasher,
                                        () => new HashListMemTable(PTableVersions.Index64Bit, MaxEntriesInMemTable * 2),
                                        () => new TFReaderLease(readers),
                                        PTableVersions.Index64Bit,
                                        MaxEntriesInMemTable);
            ReadIndex = new ReadIndex(new NoopPublisher(),
                                      readers,
                                      TableIndex,
                                      0,
                                      additionalCommitChecks: true,
                                      metastreamMaxCount: MetastreamMaxCount,
                                      hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault);
            ReadIndex.Init(ChaserCheckpoint.Read());
        }
        private void ReOpenDb()
        {
            Db = new TFChunkDb(new TFChunkDbConfig(PathName,
                                                   new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
                                                   10000,
                                                   0,
                                                   WriterCheckpoint,
                                                   ChaserCheckpoint,
                                                   new InMemoryCheckpoint(-1),
                                                   new InMemoryCheckpoint(-1)));

            Db.Open();

            var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));

            TableIndex = new TableIndex(Path.Combine(PathName, "index"),
                                        () => new HashListMemTable(MaxEntriesInMemTable * 2),
                                        () => new TFReaderLease(readers),
                                        MaxEntriesInMemTable);
            ReadIndex = new ReadIndex(new NoopPublisher(),
                                      readers,
                                      TableIndex,
                                      new ByLengthHasher(),
                                      0,
                                      additionalCommitChecks: true,
                                      metastreamMaxCount: MetastreamMaxCount);
            ReadIndex.Init(ChaserCheckpoint.Read());
        }
        private void ReOpenDb()
        {
            Db = new TFChunkDb(new TFChunkDbConfig(PathName,
                                                   new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
                                                   10000,
                                                   0,
                                                   WriterCheckpoint,
                                                   ChaserCheckpoint,
                                                   new InMemoryCheckpoint(-1),
                                                   new InMemoryCheckpoint(-1)));

            Db.Open();

            TableIndex = new TableIndex(Path.Combine(PathName, "index"),
                                        () => new HashListMemTable(MaxEntriesInMemTable * 2),
                                        MaxEntriesInMemTable);
            ReadIndex = new ReadIndex(new NoopPublisher(),
                                      2,
                                      5,
                                      () => new TFChunkReader(Db, Db.Config.WriterCheckpoint),
                                      TableIndex,
                                      new ByLengthHasher(),
                                      new NoLRUCache <string, StreamCacheInfo>(),
                                      additionalCommitChecks: true,
                                      metastreamMaxCount: MetastreamMaxCount);
            ReadIndex.Init(WriterCheckpoint.Read(), ChaserCheckpoint.Read());
        }
Exemple #6
0
        private void ReOpenDb()
        {
            Db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, WriterCheckpoint, ChaserCheckpoint));

            Db.Open();

            var indexDirectory = GetFilePathFor("index");

            _logFormat = LogFormatHelper <TLogFormat, TStreamId> .LogFormatFactory.Create(new() {
                IndexDirectory = indexDirectory,
            });

            var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 5,
                                                                  () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
            var lowHasher     = _logFormat.LowHasher;
            var highHasher    = _logFormat.HighHasher;
            var emptyStreamId = _logFormat.EmptyStreamId;

            TableIndex = new TableIndex <TStreamId>(indexDirectory, lowHasher, highHasher, emptyStreamId,
                                                    () => new HashListMemTable(PTableVersions.IndexV3, MaxEntriesInMemTable * 2),
                                                    () => new TFReaderLease(readers),
                                                    PTableVersions.IndexV3,
                                                    int.MaxValue,
                                                    Constants.PTableMaxReaderCountDefault,
                                                    MaxEntriesInMemTable);
            _logFormat.StreamNamesProvider.SetTableIndex(TableIndex);
            var readIndex = new ReadIndex <TStreamId>(new NoopPublisher(),
                                                      readers,
                                                      TableIndex,
                                                      _logFormat.StreamNameIndexConfirmer,
                                                      _logFormat.StreamIds,
                                                      _logFormat.StreamNamesProvider,
                                                      _logFormat.EmptyStreamId,
                                                      _logFormat.StreamIdValidator,
                                                      _logFormat.StreamIdSizer,
                                                      _logFormat.StreamExistenceFilter,
                                                      _logFormat.StreamExistenceFilterReader,
                                                      _logFormat.EventTypeIndexConfirmer,
                                                      streamInfoCacheCapacity: 0,
                                                      additionalCommitChecks: true,
                                                      metastreamMaxCount: MetastreamMaxCount,
                                                      hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
                                                      skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
                                                      replicationCheckpoint: Db.Config.ReplicationCheckpoint,
                                                      indexCheckpoint: Db.Config.IndexCheckpoint);

            readIndex.IndexCommitter.Init(ChaserCheckpoint.Read());
            ReadIndex = readIndex;
        }
Exemple #7
0
        public override async Task TestFixtureSetUp()
        {
            await base.TestFixtureSetUp();

            ReadIndex.Close();
            ReadIndex.Dispose();
            TableIndex.Close(removeFiles: false);

            var readers =
                new ObjectPool <ITransactionFileReader>("Readers", 2, 2, () => new TFChunkReader(Db, WriterCheckpoint));
            var lowHasher     = _logFormat.LowHasher;
            var highHasher    = _logFormat.HighHasher;
            var emptyStreamId = _logFormat.EmptyStreamId;

            TableIndex = new TableIndex <TStreamId>(GetFilePathFor("index"), lowHasher, highHasher, emptyStreamId,
                                                    () => new HashListMemTable(PTableVersions.IndexV2, maxSize: MaxEntriesInMemTable * 2),
                                                    () => new TFReaderLease(readers),
                                                    PTableVersions.IndexV2,
                                                    5, Constants.PTableMaxReaderCountDefault,
                                                    maxSizeForMemory: MaxEntriesInMemTable);
            var readIndex = new ReadIndex <TStreamId>(new NoopPublisher(),
                                                      readers,
                                                      TableIndex,
                                                      _logFormat.StreamNameIndexConfirmer,
                                                      _logFormat.StreamIds,
                                                      _logFormat.StreamNamesProvider,
                                                      _logFormat.EmptyStreamId,
                                                      _logFormat.StreamIdValidator,
                                                      _logFormat.StreamIdSizer,
                                                      _logFormat.StreamExistenceFilter,
                                                      _logFormat.StreamExistenceFilterReader,
                                                      _logFormat.EventTypeIndexConfirmer,
                                                      streamInfoCacheCapacity: 0,
                                                      additionalCommitChecks: true,
                                                      metastreamMaxCount: 1,
                                                      hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
                                                      skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
                                                      replicationCheckpoint: Db.Config.ReplicationCheckpoint,
                                                      indexCheckpoint: Db.Config.IndexCheckpoint);

            readIndex.IndexCommitter.Init(ChaserCheckpoint.Read());
            ReadIndex = readIndex;
        }
        private void ReOpenDb()
        {
            Db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, WriterCheckpoint, ChaserCheckpoint));

            Db.Open();

            var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 5,
                                                                  () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
            var lowHasher     = _logFormat.LowHasher;
            var highHasher    = _logFormat.HighHasher;
            var emptyStreamId = _logFormat.EmptyStreamId;

            TableIndex = new TableIndex <TStreamId>(Path.Combine(PathName, "index"), lowHasher, highHasher, emptyStreamId,
                                                    () => new HashListMemTable(PTableVersions.IndexV3, MaxEntriesInMemTable * 2),
                                                    () => new TFReaderLease(readers),
                                                    PTableVersions.IndexV3,
                                                    int.MaxValue,
                                                    Constants.PTableMaxReaderCountDefault,
                                                    MaxEntriesInMemTable);
            var readIndex = new ReadIndex <TStreamId>(new NoopPublisher(),
                                                      readers,
                                                      TableIndex,
                                                      _logFormat.StreamIds,
                                                      _logFormat.StreamNamesProvider,
                                                      _logFormat.EmptyStreamId,
                                                      _logFormat.StreamIdValidator,
                                                      _logFormat.StreamIdSizer,
                                                      0,
                                                      additionalCommitChecks: true,
                                                      metastreamMaxCount: MetastreamMaxCount,
                                                      hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
                                                      skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
                                                      replicationCheckpoint: Db.Config.ReplicationCheckpoint,
                                                      indexCheckpoint: Db.Config.IndexCheckpoint);

            readIndex.IndexCommitter.Init(ChaserCheckpoint.Read());
            ReadIndex = new TestReadIndex <TStreamId>(readIndex, _logFormat.StreamNameIndex);
        }
Exemple #9
0
        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            ReadIndex.Close();
            ReadIndex.Dispose();
            TableIndex.Close(removeFiles: false);

            var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 2, () => new TFChunkReader(Db, WriterCheckpoint));

            TableIndex = new TableIndex(GetFilePathFor("index"),
                                        () => new HashListMemTable(maxSize: MaxEntriesInMemTable * 2),
                                        () => new TFReaderLease(readers),
                                        maxSizeForMemory: MaxEntriesInMemTable);
            ReadIndex = new ReadIndex(new NoopPublisher(),
                                      readers,
                                      TableIndex,
                                      new ByLengthHasher(),
                                      0,
                                      additionalCommitChecks: true,
                                      metastreamMaxCount: 1);
            ReadIndex.Init(ChaserCheckpoint.Read());
        }
Exemple #10
0
        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            ReadIndex.Close();
            ReadIndex.Dispose();
            TableIndex.Close(removeFiles: false);

            TableIndex = new TableIndex(GetFilePathFor("index"),
                                        () => new HashListMemTable(maxSize: MaxEntriesInMemTable * 2),
                                        maxSizeForMemory: MaxEntriesInMemTable);

            ReadIndex = new ReadIndex(new NoopPublisher(),
                                      2,
                                      2,
                                      () => new TFChunkReader(Db, WriterCheckpoint, 0),
                                      TableIndex,
                                      new ByLengthHasher(),
                                      new NoLRUCache <string, StreamCacheInfo>(),
                                      additionalCommitChecks: true,
                                      metastreamMaxCount: 1);
            ReadIndex.Init(WriterCheckpoint.Read(), ChaserCheckpoint.Read());
        }
 public void checksums_should_be_equal_to_ack_checksum()
 {
     Assert.AreEqual(TruncateCheckpoint, WriterCheckpoint.Read());
     Assert.AreEqual(TruncateCheckpoint, ChaserCheckpoint.Read());
 }
 public void checksums_should_be_equal_to_beginning_of_intersected_scavenged_chunk()
 {
     Assert.AreEqual(chunkEdge.TransactionPosition, WriterCheckpoint.Read());
     Assert.AreEqual(chunkEdge.TransactionPosition, ChaserCheckpoint.Read());
 }