private void Scavenge(ClientMessage.ScavengeDatabase message) { var sw = Stopwatch.StartNew(); ClientMessage.ScavengeDatabase.ScavengeResult result; string error = null; long spaceSaved = 0; try { if (message.User == null || (!message.User.IsInRole(SystemRoles.Admins) && !message.User.IsInRole(SystemRoles.Operations))) { result = ClientMessage.ScavengeDatabase.ScavengeResult.Failed; error = "Access denied."; } else { var scavenger = new TFChunkScavenger(_db, _tableIndex, _hasher, _readIndex); spaceSaved = scavenger.Scavenge(_alwaysKeepScavenged, _mergeChunks); result = ClientMessage.ScavengeDatabase.ScavengeResult.Success; } } catch (Exception exc) { Log.ErrorException(exc, "SCAVENGING: error while scavenging DB."); result = ClientMessage.ScavengeDatabase.ScavengeResult.Failed; error = string.Format("Error while scavenging DB: {0}.", exc.Message); } Interlocked.Exchange(ref _isScavengingRunning, 0); message.Envelope.ReplyWith( new ClientMessage.ScavengeDatabaseCompleted(message.CorrelationId, result, error, sw.Elapsed, spaceSaved)); }
public void Handle(ClientMessage.ScavengeDatabase message) { if (IsAllowed(message.User, message.CorrelationId, message.Envelope)) { lock (_lock) { if (_currentScavenge != null) { message.Envelope.ReplyWith(new ClientMessage.ScavengeDatabaseResponse(message.CorrelationId, ClientMessage.ScavengeDatabaseResponse.ScavengeResult.InProgress, _currentScavenge.ScavengeId)); } else { var tfChunkScavengerLog = _logManager.CreateLog(); _cancellationTokenSource = new CancellationTokenSource(); var newScavenge = _currentScavenge = new TFChunkScavenger(_db, tfChunkScavengerLog, _tableIndex, _readIndex, unsafeIgnoreHardDeletes: _unsafeIgnoreHardDeletes, threads: message.Threads); var newScavengeTask = _currentScavenge.Scavenge(_alwaysKeepScavenged, _mergeChunks, message.StartFromChunk, _cancellationTokenSource.Token); HandleCleanupWhenFinished(newScavengeTask, newScavenge); message.Envelope.ReplyWith(new ClientMessage.ScavengeDatabaseResponse(message.CorrelationId, ClientMessage.ScavengeDatabaseResponse.ScavengeResult.Started, tfChunkScavengerLog.ScavengeId)); } } } }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); var dbConfig = TFChunkHelper.CreateDbConfig(PathName, 0, chunkSize: 1024 * 1024); var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig); _dbResult = CreateDb(dbCreationHelper); _keptRecords = KeptRecords(_dbResult); _dbResult.Db.Config.WriterCheckpoint.Flush(); _dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read()); _dbResult.Db.Config.ChaserCheckpoint.Flush(); var indexPath = Path.Combine(PathName, "index"); var readerPool = new ObjectPool <ITransactionFileReader>( "ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount, () => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint)); var lowHasher = new XXHashUnsafe(); var highHasher = new Murmur3AUnsafe(); var tableIndex = new TableIndex(indexPath, lowHasher, highHasher, () => new HashListMemTable(PTableVersions.IndexV3, maxSize: 200), () => new TFReaderLease(readerPool), PTableVersions.IndexV3, maxSizeForMemory: 100, maxTablesPerLevel: 2); ReadIndex = new ReadIndex(new NoopPublisher(), readerPool, tableIndex, 100, true, _metastreamMaxCount, Opts.HashCollisionReadLimitDefault, Opts.SkipIndexScanOnReadsDefault, _dbResult.Db.Config.ReplicationCheckpoint); ReadIndex.Init(_dbResult.Db.Config.WriterCheckpoint.Read()); var scavenger = new TFChunkScavenger(_dbResult.Db, new FakeTFScavengerLog(), tableIndex, ReadIndex, unsafeIgnoreHardDeletes: UnsafeIgnoreHardDelete()); scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false).Wait(); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); var dbConfig = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 1024 * 1024, 0, new InMemoryCheckpoint(0), new InMemoryCheckpoint(0), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig); _dbResult = CreateDb(dbCreationHelper); _keptRecords = KeptRecords(_dbResult); _dbResult.Db.Config.WriterCheckpoint.Flush(); _dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read()); _dbResult.Db.Config.ChaserCheckpoint.Flush(); var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount); var scavenger = new TFChunkScavenger(_dbResult.Db, scavengeReadIndex); scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false); }
public override async Task TestFixtureSetUp() { await base.TestFixtureSetUp(); WriterCheckpoint = new InMemoryCheckpoint(0); ChaserCheckpoint = new InMemoryCheckpoint(0); ReplicationCheckpoint = new InMemoryCheckpoint(-1); Db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, WriterCheckpoint, ChaserCheckpoint, replicationCheckpoint: ReplicationCheckpoint)); Db.Open(); // create db Writer = new TFChunkWriter(Db); Writer.Open(); WriteTestScenario(); Writer.Close(); Writer = null; WriterCheckpoint.Flush(); ChaserCheckpoint.Write(WriterCheckpoint.Read()); ChaserCheckpoint.Flush(); var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint)); var lowHasher = new XXHashUnsafe(); var highHasher = new Murmur3AUnsafe(); TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher, () => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2), () => new TFReaderLease(readers), IndexBitnessVersion, int.MaxValue, Constants.PTableMaxReaderCountDefault, MaxEntriesInMemTable); ReadIndex = new ReadIndex(new NoopPublisher(), readers, TableIndex, 0, additionalCommitChecks: PerformAdditionalCommitChecks, metastreamMaxCount: MetastreamMaxCount, hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault, skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault, replicationCheckpoint: Db.Config.ReplicationCheckpoint); ReadIndex.Init(ChaserCheckpoint.Read()); // scavenge must run after readIndex is built if (_scavenge) { if (_completeLastChunkOnScavenge) { Db.Manager.GetChunk(Db.Manager.ChunksCount - 1).Complete(); } _scavenger = new TFChunkScavenger(Db, new FakeTFScavengerLog(), TableIndex, ReadIndex); await _scavenger.Scavenge(alwaysKeepScavenged : true, mergeChunks : _mergeChunks); } }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); WriterCheckpoint = new InMemoryCheckpoint(0); ChaserCheckpoint = new InMemoryCheckpoint(0); Db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, WriterCheckpoint, ChaserCheckpoint, new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); Db.Open(); // create db Writer = new TFChunkWriter(Db); Writer.Open(); WriteTestScenario(); Writer.Close(); Writer = null; WriterCheckpoint.Flush(); ChaserCheckpoint.Write(WriterCheckpoint.Read()); ChaserCheckpoint.Flush(); var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 2, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint)); TableIndex = new TableIndex(GetFilePathFor("index"), () => new HashListMemTable(MaxEntriesInMemTable * 2), () => new TFReaderLease(readers), MaxEntriesInMemTable); var hasher = new ByLengthHasher(); ReadIndex = new ReadIndex(new NoopPublisher(), readers, TableIndex, hasher, 0, additionalCommitChecks: true, metastreamMaxCount: MetastreamMaxCount); ReadIndex.Init(ChaserCheckpoint.Read()); // scavenge must run after readIndex is built if (_scavenge) { if (_completeLastChunkOnScavenge) { Db.Manager.GetChunk(Db.Manager.ChunksCount - 1).Complete(); } _scavenger = new TFChunkScavenger(Db, TableIndex, hasher, ReadIndex); _scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: _mergeChunks); } }
public override async Task TestFixtureSetUp() { await base.TestFixtureSetUp(); var indexDirectory = GetFilePathFor("index"); _logFormat = LogFormatHelper <TLogFormat, TStreamId> .LogFormatFactory.Create(new() { IndexDirectory = indexDirectory, }); var dbConfig = TFChunkHelper.CreateSizedDbConfig(PathName, 0, chunkSize: 1024 * 1024); var dbCreationHelper = new TFChunkDbCreationHelper <TLogFormat, TStreamId>(dbConfig, _logFormat); _dbResult = CreateDb(dbCreationHelper); _keptRecords = KeptRecords(_dbResult); _dbResult.Db.Config.WriterCheckpoint.Flush(); _dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read()); _dbResult.Db.Config.ChaserCheckpoint.Flush(); var readerPool = new ObjectPool <ITransactionFileReader>( "ReadIndex readers pool", Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, () => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint)); var lowHasher = _logFormat.LowHasher; var highHasher = _logFormat.HighHasher; var emptyStreamId = _logFormat.EmptyStreamId; var tableIndex = new TableIndex <TStreamId>(indexDirectory, lowHasher, highHasher, emptyStreamId, () => new HashListMemTable(PTableVersions.IndexV3, maxSize: 200), () => new TFReaderLease(readerPool), PTableVersions.IndexV3, 5, Constants.PTableMaxReaderCountDefault, maxSizeForMemory: 100, maxTablesPerLevel: 2); _logFormat.StreamNamesProvider.SetTableIndex(tableIndex); var readIndex = new ReadIndex <TStreamId>(new NoopPublisher(), readerPool, tableIndex, _logFormat.StreamNameIndexConfirmer, _logFormat.StreamIds, _logFormat.StreamNamesProvider, _logFormat.EmptyStreamId, _logFormat.StreamIdValidator, _logFormat.StreamIdSizer, _logFormat.StreamExistenceFilter, _logFormat.StreamExistenceFilterReader, _logFormat.EventTypeIndexConfirmer, 100, true, _metastreamMaxCount, Opts.HashCollisionReadLimitDefault, Opts.SkipIndexScanOnReadsDefault, _dbResult.Db.Config.ReplicationCheckpoint, _dbResult.Db.Config.IndexCheckpoint); readIndex.IndexCommitter.Init(_dbResult.Db.Config.WriterCheckpoint.Read()); ReadIndex = readIndex; var scavenger = new TFChunkScavenger <TStreamId>(_dbResult.Db, new FakeTFScavengerLog(), tableIndex, ReadIndex, _logFormat.Metastreams, unsafeIgnoreHardDeletes: UnsafeIgnoreHardDelete()); await scavenger.Scavenge(alwaysKeepScavenged : true, mergeChunks : false); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 16 * 1024, 0, new InMemoryCheckpoint(), new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); _db.Open(); var chunk = _db.Manager.GetChunkFor(0); _p1 = LogRecord.SingleWrite(0, Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1", new byte[] { 0, 1, 2 }, new byte[] { 5, 7 }); _res1 = chunk.TryAppend(_p1); _c1 = LogRecord.Commit(_res1.NewPosition, Guid.NewGuid(), _p1.LogPosition, 0); _cres1 = chunk.TryAppend(_c1); _p2 = LogRecord.SingleWrite(_cres1.NewPosition, Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1", new byte[] { 0, 1, 2 }, new byte[] { 5, 7 }); _res2 = chunk.TryAppend(_p2); _c2 = LogRecord.Commit(_res2.NewPosition, Guid.NewGuid(), _p2.LogPosition, 1); _cres2 = chunk.TryAppend(_c2); _p3 = LogRecord.SingleWrite(_cres2.NewPosition, Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1", new byte[] { 0, 1, 2 }, new byte[] { 5, 7 }); _res3 = chunk.TryAppend(_p3); _c3 = LogRecord.Commit(_res3.NewPosition, Guid.NewGuid(), _p3.LogPosition, 2); _cres3 = chunk.TryAppend(_c3); chunk.Complete(); _db.Config.WriterCheckpoint.Write(chunk.ChunkHeader.ChunkEndPosition); _db.Config.WriterCheckpoint.Flush(); _db.Config.ChaserCheckpoint.Write(chunk.ChunkHeader.ChunkEndPosition); _db.Config.ChaserCheckpoint.Flush(); var bus = new InMemoryBus("Bus"); var ioDispatcher = new IODispatcher(bus, new PublishEnvelope(bus)); var scavenger = new TFChunkScavenger(_db, ioDispatcher, new FakeTableIndex(), new FakeReadIndex(x => x == "es-to-scavenge"), Guid.NewGuid(), "fakeNodeIp"); scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false); _scavengedChunk = _db.Manager.GetChunk(0); }
public override async Task TestFixtureSetUp() { await base.TestFixtureSetUp(); _db = new TFChunkDb(TFChunkHelper.CreateSizedDbConfig(PathName, 0, chunkSize: 16 * 1024)); _db.Open(); var chunk = _db.Manager.GetChunkFor(0); var logFormat = LogFormatHelper <TLogFormat, TStreamId> .LogFormat; var streamName = "es-to-scavenge"; logFormat.StreamNameIndex.GetOrAddId(streamName, out var streamId, out _, out _); var expectedVersion = ExpectedVersion.NoStream; _p1 = LogRecord.SingleWrite(logFormat.RecordFactory, 0, Guid.NewGuid(), Guid.NewGuid(), streamId, expectedVersion++, "et1", new byte[2048], new byte[] { 5, 7 }); _res1 = chunk.TryAppend(_p1); _c1 = LogRecord.Commit(_res1.NewPosition, Guid.NewGuid(), _p1.LogPosition, 0); _cres1 = chunk.TryAppend(_c1); _p2 = LogRecord.SingleWrite(logFormat.RecordFactory, _cres1.NewPosition, Guid.NewGuid(), Guid.NewGuid(), streamId, expectedVersion++, "et1", new byte[2048], new byte[] { 5, 7 }); _res2 = chunk.TryAppend(_p2); _c2 = LogRecord.Commit(_res2.NewPosition, Guid.NewGuid(), _p2.LogPosition, 1); _cres2 = chunk.TryAppend(_c2); _p3 = LogRecord.SingleWrite(logFormat.RecordFactory, _cres2.NewPosition, Guid.NewGuid(), Guid.NewGuid(), streamId, expectedVersion++, "et1", new byte[2048], new byte[] { 5, 7 }); _res3 = chunk.TryAppend(_p3); _c3 = LogRecord.Commit(_res3.NewPosition, Guid.NewGuid(), _p3.LogPosition, 2); _cres3 = chunk.TryAppend(_c3); chunk.Complete(); _originalFileSize = chunk.FileSize; _db.Config.WriterCheckpoint.Write(chunk.ChunkHeader.ChunkEndPosition); _db.Config.WriterCheckpoint.Flush(); _db.Config.ChaserCheckpoint.Write(chunk.ChunkHeader.ChunkEndPosition); _db.Config.ChaserCheckpoint.Flush(); var scavenger = new TFChunkScavenger <TStreamId>(_db, new FakeTFScavengerLog(), new FakeTableIndex <TStreamId>(), new FakeReadIndex <TLogFormat, TStreamId>(x => EqualityComparer <TStreamId> .Default.Equals(x, streamId)), logFormat.SystemStreams); await scavenger.Scavenge(alwaysKeepScavenged : true, mergeChunks : false); _scavengedChunk = _db.Manager.GetChunk(0); }
private void Scavenge() { try { var scavenger = new TFChunkScavenger(_db, _readIndex); scavenger.Scavenge(_alwaysKeepScavenged, _mergeChunks); } catch (Exception exc) { Log.ErrorException(exc, "SCAVENGING: error while scavenging DB."); } Interlocked.Exchange(ref _isScavengingRunning, 0); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); WriterChecksum = new InMemoryCheckpoint(0); ChaserChecksum = new InMemoryCheckpoint(0); Db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, WriterChecksum, ChaserChecksum, new[] { WriterChecksum, ChaserChecksum })); Db.OpenVerifyAndClean(); // create db Writer = new TFChunkWriter(Db); Writer.Open(); WriteTestScenario(); Writer.Close(); Writer = null; WriterChecksum.Flush(); ChaserChecksum.Write(WriterChecksum.Read()); ChaserChecksum.Flush(); TableIndex = new TableIndex(Path.Combine(PathName, "index"), () => new HashListMemTable(MaxEntriesInMemTable * 2), MaxEntriesInMemTable); var reader = new TFChunkReader(Db, Db.Config.WriterCheckpoint); ReadIndex = new ReadIndex(new NoopPublisher(), 2, () => new TFChunkSequentialReader(Db, Db.Config.WriterCheckpoint, 0), () => reader, TableIndex, new ByLengthHasher(), new NoLRUCache <string, StreamCacheInfo>()); ReadIndex.Build(); // scavenge must run after readIndex is built if (_scavenge) { _scavenger = new TFChunkScavenger(Db, ReadIndex); _scavenger.Scavenge(alwaysKeepScavenged: true); } }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 4096, 0, new InMemoryCheckpoint(), new InMemoryCheckpoint(), new ICheckpoint[0])); _db.OpenVerifyAndClean(); var chunk = _db.Manager.GetChunk(0); _records = new LogRecord[RecordsCount]; _results = new RecordWriteResult[RecordsCount]; var pos = 0; for (int i = 0; i < RecordsCount; ++i) { if (i > 0 && i % 3 == 0) { pos = i / 3 * _db.Config.ChunkSize; chunk.Complete(); chunk = _db.Manager.AddNewChunk(); } _records[i] = LogRecord.SingleWrite(pos, Guid.NewGuid(), Guid.NewGuid(), i % 2 == 1 ? "es-to-scavenge": "es1", ExpectedVersion.Any, "et1", new byte[1200], new byte[] { 5, 7 }); _results[i] = chunk.TryAppend(_records[i]); pos += _records[i].GetSizeWithLengthPrefixAndSuffix(); } _keptRecords = _records.Where((x, i) => i % 2 == 0).ToArray(); chunk.Flush(); chunk.Complete(); _db.Config.WriterCheckpoint.Write((RecordsCount / 3) * _db.Config.ChunkSize + _results[RecordsCount - 1].NewPosition); _db.Config.WriterCheckpoint.Flush(); var scavenger = new TFChunkScavenger(_db, new FakeReadIndex(x => x == "es-to-scavenge")); scavenger.Scavenge(alwaysKeepScavenged: true); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, 0, chunkSize: 16 * 1024)); _db.Open(); var chunk = _db.Manager.GetChunkFor(0); _p1 = LogRecord.SingleWrite(0, Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1", new byte[2048], new byte[] { 5, 7 }); _res1 = chunk.TryAppend(_p1); _c1 = LogRecord.Commit(_res1.NewPosition, Guid.NewGuid(), _p1.LogPosition, 0); _cres1 = chunk.TryAppend(_c1); _p2 = LogRecord.SingleWrite(_cres1.NewPosition, Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1", new byte[2048], new byte[] { 5, 7 }); _res2 = chunk.TryAppend(_p2); _c2 = LogRecord.Commit(_res2.NewPosition, Guid.NewGuid(), _p2.LogPosition, 1); _cres2 = chunk.TryAppend(_c2); _p3 = LogRecord.SingleWrite(_cres2.NewPosition, Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1", new byte[2048], new byte[] { 5, 7 }); _res3 = chunk.TryAppend(_p3); _c3 = LogRecord.Commit(_res3.NewPosition, Guid.NewGuid(), _p3.LogPosition, 2); _cres3 = chunk.TryAppend(_c3); chunk.Complete(); _originalFileSize = chunk.FileSize; _db.Config.WriterCheckpoint.Write(chunk.ChunkHeader.ChunkEndPosition); _db.Config.WriterCheckpoint.Flush(); _db.Config.ChaserCheckpoint.Write(chunk.ChunkHeader.ChunkEndPosition); _db.Config.ChaserCheckpoint.Flush(); var scavenger = new TFChunkScavenger(_db, new FakeTFScavengerLog(), new FakeTableIndex(), new FakeReadIndex(x => x == "es-to-scavenge")); scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false).Wait(); _scavengedChunk = _db.Manager.GetChunk(0); }
private void Scavenge() { try { var scavenger = new TFChunkScavenger(_db, _readIndex); scavenger.Scavenge(); } catch (Exception exc) { Log.ErrorException(exc, "Error while scavenging DB."); } lock (_scavengingLock) { _isScavengingRunning = false; } }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); var dbConfig = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 1024 * 1024, 0, new InMemoryCheckpoint(0), new InMemoryCheckpoint(0), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig); _dbResult = CreateDb(dbCreationHelper); _keptRecords = KeptRecords(_dbResult); _dbResult.Db.Config.WriterCheckpoint.Flush(); _dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read()); _dbResult.Db.Config.ChaserCheckpoint.Flush(); var indexPath = Path.Combine(PathName, "index"); var readerPool = new ObjectPool <ITransactionFileReader>( "ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount, () => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint)); var lowHasher = new XXHashUnsafe(); var highHasher = new Murmur3AUnsafe(); var tableIndex = new TableIndex(indexPath, lowHasher, highHasher, () => new HashListMemTable(PTableVersions.IndexV3, maxSize: 200), () => new TFReaderLease(readerPool), PTableVersions.IndexV3, maxSizeForMemory: 100, maxTablesPerLevel: 2); ReadIndex = new ReadIndex(new NoopPublisher(), readerPool, tableIndex, 100, true, _metastreamMaxCount, Opts.HashCollisionReadLimitDefault); ReadIndex.Init(_dbResult.Db.Config.WriterCheckpoint.Read()); //var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount); var bus = new InMemoryBus("Bus"); var ioDispatcher = new IODispatcher(bus, new PublishEnvelope(bus)); var scavenger = new TFChunkScavenger(_dbResult.Db, ioDispatcher, tableIndex, ReadIndex, Guid.NewGuid(), "fakeNodeIp", unsafeIgnoreHardDeletes: UnsafeIgnoreHardDelete()); scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false); }
private void Scavenge(ClientMessage.ScavengeDatabase message) { var sw = Stopwatch.StartNew(); Guid scavengeId = Guid.NewGuid(); var streamName = string.Format("{0}-{1}", SystemStreams.ScavengesStream, scavengeId); ClientMessage.ScavengeDatabase.ScavengeResult result; string error = null; long spaceSaved = 0; try { if (message.User == null || (!message.User.IsInRole(SystemRoles.Admins) && !message.User.IsInRole(SystemRoles.Operations))) { result = ClientMessage.ScavengeDatabase.ScavengeResult.Failed; error = "Access denied."; } else { WriteScavengeStartedEvent(streamName, scavengeId); var scavenger = new TFChunkScavenger(_db, _ioDispatcher, _tableIndex, _hasher, _readIndex, scavengeId, _nodeEndpoint, unsafeIgnoreHardDeletes: _unsafeIgnoreHardDeletes); spaceSaved = scavenger.Scavenge(_alwaysKeepScavenged, _mergeChunks); result = ClientMessage.ScavengeDatabase.ScavengeResult.Success; } } catch (Exception exc) { Log.ErrorException(exc, "SCAVENGING: error while scavenging DB."); result = ClientMessage.ScavengeDatabase.ScavengeResult.Failed; error = string.Format("Error while scavenging DB: {0}.", exc.Message); } Interlocked.Exchange(ref _isScavengingRunning, 0); WriteScavengeCompletedEvent(streamName, scavengeId, result, error, spaceSaved, sw.Elapsed); message.Envelope.ReplyWith( new ClientMessage.ScavengeDatabaseCompleted(message.CorrelationId, result, error, sw.Elapsed, spaceSaved) ); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); var dbConfig = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 1024 * 1024, 0, new InMemoryCheckpoint(0), new InMemoryCheckpoint(0), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig); _dbResult = CreateDb(dbCreationHelper); _keptRecords = KeptRecords(_dbResult); _dbResult.Db.Config.WriterCheckpoint.Flush(); _dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read()); _dbResult.Db.Config.ChaserCheckpoint.Flush(); var indexPath = Path.Combine(PathName, "index"); var readerPool = new ObjectPool <ITransactionFileReader>( "ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount, () => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint)); var tableIndex = new TableIndex(indexPath, () => new HashListMemTable(maxSize: 200), () => new TFReaderLease(readerPool), maxSizeForMemory: 100, maxTablesPerLevel: 2); var hasher = new XXHashUnsafe(); ReadIndex = new ReadIndex(new NoopPublisher(), readerPool, tableIndex, hasher, 100, true, _metastreamMaxCount); ReadIndex.Init(_dbResult.Db.Config.WriterCheckpoint.Read()); //var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount); var scavenger = new TFChunkScavenger(_dbResult.Db, tableIndex, hasher, ReadIndex); scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 16 * 1024, 0, new InMemoryCheckpoint(), new InMemoryCheckpoint(), new ICheckpoint[0])); _db.OpenVerifyAndClean(); var chunk = _db.Manager.GetChunk(0); _rec1 = LogRecord.SingleWrite(0, Guid.NewGuid(), Guid.NewGuid(), "es1", ExpectedVersion.Any, "et1", new byte[] { 0, 1, 2 }, new byte[] { 5, 7 }); _res1 = chunk.TryAppend(_rec1); _rec2 = LogRecord.SingleWrite(_res1.NewPosition, Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1", new byte[] { 0, 1, 2 }, new byte[] { 5, 7 }); _res2 = chunk.TryAppend(_rec2); _rec3 = LogRecord.SingleWrite(_res2.NewPosition, Guid.NewGuid(), Guid.NewGuid(), "es1", ExpectedVersion.Any, "et1", new byte[] { 0, 1, 2 }, new byte[] { 5, 7 }); _res3 = chunk.TryAppend(_rec3); chunk.Complete(); var scavenger = new TFChunkScavenger(_db, new FakeReadIndex(x => x == "es-to-scavenge")); scavenger.Scavenge(alwaysKeepScavenged: true); _scavengedChunk = _db.Manager.GetChunk(0); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); WriterCheckpoint = new InMemoryCheckpoint(0); ChaserCheckpoint = new InMemoryCheckpoint(0); Bus = new InMemoryBus("bus"); IODispatcher = new IODispatcher(Bus, new PublishEnvelope(Bus)); Db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, WriterCheckpoint, ChaserCheckpoint, new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); Db.Open(); // create db Writer = new TFChunkWriter(Db); Writer.Open(); WriteTestScenario(); Writer.Close(); Writer = null; WriterCheckpoint.Flush(); ChaserCheckpoint.Write(WriterCheckpoint.Read()); ChaserCheckpoint.Flush(); var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint)); var lowHasher = new XXHashUnsafe(); var highHasher = new Murmur3AUnsafe(); TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher, () => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2), () => new TFReaderLease(readers), IndexBitnessVersion, MaxEntriesInMemTable); ReadIndex = new ReadIndex(new NoopPublisher(), readers, TableIndex, 0, additionalCommitChecks: PerformAdditionalCommitChecks, metastreamMaxCount: MetastreamMaxCount, hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault); ReadIndex.Init(ChaserCheckpoint.Read()); // scavenge must run after readIndex is built if (_scavenge) { if (_completeLastChunkOnScavenge) { Db.Manager.GetChunk(Db.Manager.ChunksCount - 1).Complete(); } _scavenger = new TFChunkScavenger(Db, IODispatcher, TableIndex, ReadIndex, Guid.NewGuid(), "fakeNodeIp"); _scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: _mergeChunks); } }
public override async Task TestFixtureSetUp() { await base.TestFixtureSetUp(); _logFormat = LogFormatHelper <TLogFormat, TStreamId> .LogFormatFactory.Create(new() { IndexDirectory = GetFilePathFor("index"), }); _db = new TFChunkDb(TFChunkHelper.CreateSizedDbConfig(PathName, 0, chunkSize: 16 * 1024)); _db.Open(); var chunk = _db.Manager.GetChunkFor(0); var streamName = "es-to-scavenge"; var pos = 0L; _logFormat.StreamNameIndex.GetOrReserve(_logFormat.RecordFactory, streamName, 0, out var streamId, out var streamRecord); if (streamRecord is not null) { var res = chunk.TryAppend(streamRecord); pos = res.NewPosition; } _logFormat.EventTypeIndex.GetOrReserveEventType(_logFormat.RecordFactory, "et1", pos, out var eventTypeId, out var eventTypeRecord); if (eventTypeRecord is not null) { var res = chunk.TryAppend(eventTypeRecord); pos = res.NewPosition; } var expectedVersion = ExpectedVersion.NoStream; _p1 = LogRecord.SingleWrite(_logFormat.RecordFactory, pos, Guid.NewGuid(), Guid.NewGuid(), streamId, expectedVersion++, eventTypeId, new byte[2048], new byte[] { 5, 7 }); _res1 = chunk.TryAppend(_p1); _c1 = LogRecord.Commit(_res1.NewPosition, Guid.NewGuid(), _p1.LogPosition, 0); _cres1 = chunk.TryAppend(_c1); _p2 = LogRecord.SingleWrite(_logFormat.RecordFactory, _cres1.NewPosition, Guid.NewGuid(), Guid.NewGuid(), streamId, expectedVersion++, eventTypeId, new byte[2048], new byte[] { 5, 7 }); _res2 = chunk.TryAppend(_p2); _c2 = LogRecord.Commit(_res2.NewPosition, Guid.NewGuid(), _p2.LogPosition, 1); _cres2 = chunk.TryAppend(_c2); _p3 = LogRecord.SingleWrite(_logFormat.RecordFactory, _cres2.NewPosition, Guid.NewGuid(), Guid.NewGuid(), streamId, expectedVersion++, eventTypeId, new byte[2048], new byte[] { 5, 7 }); _res3 = chunk.TryAppend(_p3); _c3 = LogRecord.Commit(_res3.NewPosition, Guid.NewGuid(), _p3.LogPosition, 2); _cres3 = chunk.TryAppend(_c3); chunk.Complete(); _originalFileSize = chunk.FileSize; _db.Config.WriterCheckpoint.Write(chunk.ChunkHeader.ChunkEndPosition); _db.Config.WriterCheckpoint.Flush(); _db.Config.ChaserCheckpoint.Write(chunk.ChunkHeader.ChunkEndPosition); _db.Config.ChaserCheckpoint.Flush(); var scavenger = new TFChunkScavenger <TStreamId>(_db, new FakeTFScavengerLog(), new FakeTableIndex <TStreamId>(), new FakeReadIndex <TLogFormat, TStreamId>(x => EqualityComparer <TStreamId> .Default.Equals(x, streamId), _logFormat.Metastreams), _logFormat.Metastreams); await scavenger.Scavenge(alwaysKeepScavenged : true, mergeChunks : false); _scavengedChunk = _db.Manager.GetChunk(0); }
public override async Task TestFixtureSetUp() { await base.TestFixtureSetUp(); var indexDirectory = GetFilePathFor("index"); _logFormat = LogFormatHelper <TLogFormat, TStreamId> .LogFormatFactory.Create(new() { IndexDirectory = indexDirectory, }); _recordFactory = _logFormat.RecordFactory; _streamNameIndex = _logFormat.StreamNameIndex; _eventTypeIndex = _logFormat.EventTypeIndex; WriterCheckpoint = new InMemoryCheckpoint(0); ChaserCheckpoint = new InMemoryCheckpoint(0); Db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, WriterCheckpoint, ChaserCheckpoint, replicationCheckpoint: new InMemoryCheckpoint(-1), chunkSize: _chunkSize)); Db.Open(); // create db Writer = new TFChunkWriter(Db); Writer.Open(); WriteTestScenario(); Writer.Close(); Writer = null; WriterCheckpoint.Flush(); ChaserCheckpoint.Write(WriterCheckpoint.Read()); ChaserCheckpoint.Flush(); var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint)); var lowHasher = _logFormat.LowHasher; var highHasher = _logFormat.HighHasher; var emptyStreamId = _logFormat.EmptyStreamId; TableIndex = new TableIndex <TStreamId>(indexDirectory, lowHasher, highHasher, emptyStreamId, () => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2), () => new TFReaderLease(readers), IndexBitnessVersion, int.MaxValue, Constants.PTableMaxReaderCountDefault, MaxEntriesInMemTable); _logFormat.StreamNamesProvider.SetTableIndex(TableIndex); var readIndex = new ReadIndex <TStreamId>(new NoopPublisher(), readers, TableIndex, _logFormat.StreamNameIndexConfirmer, _logFormat.StreamIds, _logFormat.StreamNamesProvider, _logFormat.EmptyStreamId, _logFormat.StreamIdValidator, _logFormat.StreamIdSizer, _logFormat.StreamExistenceFilter, _logFormat.StreamExistenceFilterReader, _logFormat.EventTypeIndexConfirmer, streamInfoCacheCapacity: StreamInfoCacheCapacity, additionalCommitChecks: PerformAdditionalCommitChecks, metastreamMaxCount: MetastreamMaxCount, hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault, skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault, replicationCheckpoint: Db.Config.ReplicationCheckpoint, indexCheckpoint: Db.Config.IndexCheckpoint); readIndex.IndexCommitter.Init(ChaserCheckpoint.Read()); ReadIndex = readIndex; // scavenge must run after readIndex is built if (_scavenge) { if (_completeLastChunkOnScavenge) { Db.Manager.GetChunk(Db.Manager.ChunksCount - 1).Complete(); } _scavenger = new TFChunkScavenger <TStreamId>(Db, new FakeTFScavengerLog(), TableIndex, ReadIndex, _logFormat.Metastreams); await _scavenger.Scavenge(alwaysKeepScavenged : true, mergeChunks : _mergeChunks); } }