private PrepareLogRecord ReadPrepareSkipScan(TFReaderLease reader, string streamId, long eventNumber) { long position; if (_tableIndex.TryGetOneValue(streamId, eventNumber, out position)) { var rec = ReadPrepareInternal(reader, position); if (rec != null && rec.EventStreamId == streamId) { return(rec); } foreach (var indexEntry in _tableIndex.GetRange(streamId, eventNumber, eventNumber)) { Interlocked.Increment(ref _hashCollisions); if (indexEntry.Position == position) { continue; } rec = ReadPrepareInternal(reader, indexEntry.Position); if (rec != null && rec.EventStreamId == streamId) { return(rec); } } } return(null); }
public void Setup() { given(); _indexDir = PathName; _fakeReader = new TFReaderLease(new FakeReader()); _indexBackend = new FakeIndexBackend <string>(_fakeReader); _logFormat = LogFormatHelper <LogFormat.V2, string> .LogFormatFactory.Create(new() { InMemory = true, }); _lowHasher = _logFormat.LowHasher; _highHasher = _logFormat.HighHasher; _tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, _logFormat.EmptyStreamId, () => new HashListMemTable(PTableVersions.IndexV1, maxSize: _maxMemTableSize), () => _fakeReader, PTableVersions.IndexV1, 5, Constants.PTableMaxReaderCountDefault, maxSizeForMemory: _maxMemTableSize, maxTablesPerLevel: 2); _logFormat.StreamNamesProvider.SetTableIndex(_tableIndex); _tableIndex.Initialize(long.MaxValue); _indexReader = new IndexReader <string>(_indexBackend, _tableIndex, _logFormat.StreamNamesProvider, _logFormat.StreamIdValidator, _logFormat.StreamExistenceFilterReader, new EventStore.Core.Data.StreamMetadata(), _hashCollisionReadLimit, skipIndexScanOnRead: false); when(); //wait for the mem table to be dumped System.Threading.Thread.Sleep(500); }
private PrepareLogRecord ReadPrepareInternal(TFReaderLease reader, string streamId, int eventNumber) { // we assume that you already did check for stream deletion Ensure.NotNullOrEmpty(streamId, "streamId"); Ensure.Nonnegative(eventNumber, "eventNumber"); var streamHash = _hasher.Hash(streamId); long position; if (_tableIndex.TryGetOneValue(streamHash, eventNumber, out position)) { var rec = ReadPrepareInternal(reader, position); if (rec != null && rec.EventStreamId == streamId) { return(rec); } foreach (var indexEntry in _tableIndex.GetRange(streamHash, eventNumber, eventNumber)) { Interlocked.Increment(ref _hashCollisions); if (indexEntry.Position == position) // already checked that { continue; } rec = ReadPrepareInternal(reader, indexEntry.Position); if (rec != null && rec.EventStreamId == streamId) { return(rec); } } } return(null); }
private IndexReadEventResult ReadEventInternal(TFReaderLease reader, string streamId, long eventNumber) { var lastEventNumber = GetStreamLastEventNumberCached(reader, streamId); var metadata = GetStreamMetadataCached(reader, streamId); var originalStreamExists = OriginalStreamExists(reader, streamId); if (lastEventNumber == EventNumber.DeletedStream) { return(new IndexReadEventResult(ReadEventResult.StreamDeleted, metadata, lastEventNumber, originalStreamExists)); } if (lastEventNumber == ExpectedVersion.NoStream || metadata.TruncateBefore == EventNumber.DeletedStream) { return(new IndexReadEventResult(ReadEventResult.NoStream, metadata, lastEventNumber, originalStreamExists)); } if (lastEventNumber == EventNumber.Invalid) { return(new IndexReadEventResult(ReadEventResult.NoStream, metadata, lastEventNumber, originalStreamExists)); } if (eventNumber == -1) { eventNumber = lastEventNumber; } long minEventNumber = 0; if (metadata.MaxCount.HasValue) { minEventNumber = Math.Max(minEventNumber, lastEventNumber - metadata.MaxCount.Value + 1); } if (metadata.TruncateBefore.HasValue) { minEventNumber = Math.Max(minEventNumber, metadata.TruncateBefore.Value); } //TODO(clc): confirm this logic, it seems that reads less than min should be invaild rather than found if (eventNumber < minEventNumber || eventNumber > lastEventNumber) { return(new IndexReadEventResult(ReadEventResult.NotFound, metadata, lastEventNumber, originalStreamExists)); } PrepareLogRecord prepare = ReadPrepareInternal(reader, streamId, eventNumber); if (prepare != null) { if (metadata.MaxAge.HasValue && prepare.TimeStamp < DateTime.UtcNow - metadata.MaxAge.Value) { return(new IndexReadEventResult(ReadEventResult.NotFound, metadata, lastEventNumber, originalStreamExists)); } return(new IndexReadEventResult(ReadEventResult.Success, new EventRecord(eventNumber, prepare), metadata, lastEventNumber, originalStreamExists)); } return(new IndexReadEventResult(ReadEventResult.NotFound, metadata, lastEventNumber, originalStreamExists: originalStreamExists)); }
private int GetStreamLastEventNumberCached(TFReaderLease reader, string streamId) { // if this is metastream -- check if original stream was deleted, if yes -- metastream is deleted as well if (SystemStreams.IsMetastream(streamId) && GetStreamLastEventNumberCached(reader, SystemStreams.OriginalStreamOf(streamId)) == EventNumber.DeletedStream) { return(EventNumber.DeletedStream); } var cache = _cache.TryGetStreamLastEventNumber(streamId); if (cache.LastEventNumber != null) { Interlocked.Increment(ref _cachedStreamInfo); return(cache.LastEventNumber.GetValueOrDefault()); } Interlocked.Increment(ref _notCachedStreamInfo); var lastEventNumber = GetStreamLastEventNumberUncached(reader, streamId); // Conditional update depending on previously returned cache info version. // If version is not correct -- nothing is changed in cache. // This update is conditioned to not interfere with updating stream cache info by commit procedure // (which is the source of truth). var res = _cache.UpdateStreamLastEventNumber(cache.Version, streamId, lastEventNumber); return(res ?? lastEventNumber); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _indexDir = PathName; var fakeReader = new TFReaderLease(new FakeTfReader()); _tableIndex = new TableIndex(_indexDir, () => new HashListMemTable(maxSize: 10), () => fakeReader, maxSizeForMemory: 5); _tableIndex.Initialize(long.MaxValue); _tableIndex.Add(0, 0xDEAD, 0, 0xFF00); _tableIndex.Add(0, 0xDEAD, 1, 0xFF01); _tableIndex.Add(0, 0xBEEF, 0, 0xFF00); _tableIndex.Add(0, 0xBEEF, 1, 0xFF01); _tableIndex.Add(0, 0xABBA, 0, 0xFF00); // 1st ptable0 _tableIndex.Add(0, 0xABBA, 1, 0xFF01); _tableIndex.Add(0, 0xABBA, 2, 0xFF02); _tableIndex.Add(0, 0xABBA, 3, 0xFF03); _tableIndex.Add(0, 0xADA, 0, 0xFF00); // simulates duplicate due to concurrency in TableIndex (see memtable below) _tableIndex.Add(0, 0xDEAD, 0, 0xFF10); // 2nd ptable0 _tableIndex.Add(0, 0xDEAD, 1, 0xFF11); // in memtable _tableIndex.Add(0, 0xADA, 0, 0xFF00); // in memtable }
private int GetStreamLastEventNumberUncached(TFReaderLease reader, string streamId) { var streamHash = _hasher.Hash(streamId); IndexEntry latestEntry; if (!_tableIndex.TryGetLatestEntry(streamHash, out latestEntry)) { return(ExpectedVersion.NoStream); } var rec = ReadPrepareInternal(reader, latestEntry.Position); if (rec == null) { throw new Exception("Couldn't read latest stream's prepare! That shouldn't happen EVER!"); } if (rec.EventStreamId == streamId) // LUCKY!!! { return(latestEntry.Version); } // TODO AN here lies the problem of out of memory if the stream has A LOT of events in them foreach (var indexEntry in _tableIndex.GetRange(streamHash, 0, int.MaxValue)) { var r = ReadPrepareInternal(reader, indexEntry.Position); if (r != null && r.EventStreamId == streamId) { return(indexEntry.Version); // AT LAST!!! } Interlocked.Increment(ref _hashCollisions); } return(ExpectedVersion.NoStream); // no such event stream }
private StreamMetadata GetStreamMetadataCached(TFReaderLease reader, string streamId) { // if this is metastream -- check if original stream was deleted, if yes -- metastream is deleted as well if (SystemStreams.IsMetastream(streamId)) { return(_metastreamMetadata); } var cache = _cache.TryGetStreamMetadata(streamId); if (cache.Metadata != null) { Interlocked.Increment(ref _cachedStreamInfo); return(cache.Metadata); } Interlocked.Increment(ref _notCachedStreamInfo); var streamMetadata = GetStreamMetadataUncached(reader, streamId); // Conditional update depending on previously returned cache info version. // If version is not correct -- nothing is changed in cache. // This update is conditioned to not interfere with updating stream cache info by commit procedure // (which is the source of truth). var res = _cache.UpdateStreamMetadata(cache.Version, streamId, streamMetadata); return(res ?? streamMetadata); }
private StreamMetadata GetStreamMetadataUncached(TFReaderLease reader, string streamId) { var metastreamId = SystemStreams.MetastreamOf(streamId); var metaEventNumber = GetStreamLastEventNumberCached(reader, metastreamId); if (metaEventNumber == ExpectedVersion.NoStream || metaEventNumber == EventNumber.DeletedStream) { return(StreamMetadata.Empty); } PrepareLogRecord prepare = ReadPrepareInternal(reader, metastreamId, metaEventNumber); if (prepare == null) { throw new Exception(string.Format("ReadPrepareInternal couldn't find metaevent #{0} on metastream '{1}'. " + "That should never happen.", metaEventNumber, metastreamId)); } if (prepare.Data.Length == 0 || prepare.Flags.HasNoneOf(PrepareFlags.IsJson)) { return(StreamMetadata.Empty); } try { return(StreamMetadata.FromJsonBytes(prepare.Data)); } catch (Exception) { return(StreamMetadata.Empty); } }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _indexDir = PathName; var fakeReader = new TFReaderLease(new FakeTfReader()); _lowHasher = new XXHashUnsafe(); _highHasher = new Murmur3AUnsafe(); _tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher, () => new HashListMemTable(_ptableVersion, maxSize: 10), () => fakeReader, _ptableVersion, maxSizeForMemory: 5, skipIndexVerify: _skipIndexVerify); _tableIndex.Initialize(long.MaxValue); _tableIndex.Add(0, "0xDEAD", 0, 0xFF00); _tableIndex.Add(0, "0xDEAD", 1, 0xFF01); _tableIndex.Add(0, "0xBEEF", 0, 0xFF00); _tableIndex.Add(0, "0xBEEF", 1, 0xFF01); _tableIndex.Add(0, "0xABBA", 0, 0xFF00); // 1st ptable0 _tableIndex.Add(0, "0xABBA", 1, 0xFF01); _tableIndex.Add(0, "0xABBA", 2, 0xFF02); _tableIndex.Add(0, "0xABBA", 3, 0xFF03); _tableIndex.Add(0, "0xADA", 0, 0xFF00); // simulates duplicate due to concurrency in TableIndex (see memtable below) _tableIndex.Add(0, "0xDEAD", 0, 0xFF10); // 2nd ptable0 _tableIndex.Add(0, "0xDEAD", 1, 0xFF11); // in memtable _tableIndex.Add(0, "0xADA", 0, 0xFF00); // in memtable }
public void ConstructTableIndexWithCorruptIndexEntries(byte version, bool skipIndexVerify, bool createForceVerifyFile = false) { base.TestFixtureSetUp(); var lowHasher = new XXHashUnsafe(); var highHasher = new Murmur3AUnsafe(); var fakeReader = new TFReaderLease(new FakeIndexReader()); _tableIndex = new TableIndex(PathName, lowHasher, highHasher, () => new HashListMemTable(version, maxSize: NumIndexEntries), () => fakeReader, version, 5, maxSizeForMemory: NumIndexEntries, skipIndexVerify: skipIndexVerify); _tableIndex.Initialize(long.MaxValue); //create index entries for (int i = 1; i <= NumIndexEntries; i++) { _tableIndex.Add(i * 1337, StreamName, i, i * 1337); } _tableIndex.Close(false); //load index map to obtain ptable filenames _indexMap = IndexMap.FromFile(Path.Combine(PathName, TableIndex.IndexMapFilename)); List <string> ptableFiles = new List <string>(); foreach (string ptableFilename in _indexMap.GetAllFilenames()) { ptableFiles.Add(ptableFilename); } _indexMap.Dispose(TimeSpan.FromSeconds(5)); //corrupt ptable files foreach (string ptableFilename in ptableFiles) { CorruptPTableFile(ptableFilename, version, "zeroOutMiddleEntry"); } //create force verify file if requested if (createForceVerifyFile) { using (FileStream fs = new FileStream(Path.Combine(PathName, TableIndex.ForceIndexVerifyFilename), FileMode.OpenOrCreate)){ }; } //load table index again _tableIndex = new TableIndex(PathName, lowHasher, highHasher, () => new HashListMemTable(version, maxSize: NumIndexEntries), () => fakeReader, version, 5, maxSizeForMemory: NumIndexEntries, skipIndexVerify: skipIndexVerify, indexCacheDepth: 8); _tableIndex.Initialize(long.MaxValue); }
private PrepareLogRecord ReadPrepareInternal(TFReaderLease reader, string streamId, long eventNumber) { // we assume that you already did check for stream deletion Ensure.NotNullOrEmpty(streamId, "streamId"); Ensure.Nonnegative(eventNumber, "eventNumber"); return(_skipIndexScanOnRead ? ReadPrepareSkipScan(reader, streamId, eventNumber) : ReadPrepare(reader, streamId, eventNumber)); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _indexDir = PathName; var fakeReader = new TFReaderLease(new FakeIndexReader()); _lowHasher = new XXHashUnsafe(); _highHasher = new Murmur3AUnsafe(); _tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher, () => new HashListMemTable(PTableVersions.IndexV1, maxSize: 5), () => fakeReader, PTableVersions.IndexV1, 5, maxSizeForMemory: 5 + _extraStreamHashesAtBeginning + _extraStreamHashesAtEnd, maxTablesPerLevel: 2); _tableIndex.Initialize(long.MaxValue); Assert.Greater(_lowHasher.Hash("abcd"), _lowHasher.Hash("LPN-FC002_LPK51001")); for (int i = 0; i < _extraStreamHashesAtBeginning; i++) { _tableIndex.Add(1, "abcd", i, i + 1); } Assert.Less(_lowHasher.Hash("wxyz"), _lowHasher.Hash("LPN-FC002_LPK51001")); for (int i = 0; i < _extraStreamHashesAtEnd; i++) { _tableIndex.Add(1, "wxyz", i, i + 1); } _tableIndex.Add(1, "LPN-FC002_LPK51001", 0, 1); _tableIndex.Add(1, "account--696193173", 0, 2); _tableIndex.Add(1, "LPN-FC002_LPK51001", 1, 3); _tableIndex.Add(1, "account--696193173", 1, 4); _tableIndex.Add(1, "LPN-FC002_LPK51001", 2, 5); _tableIndex.Close(false); _tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher, () => new HashListMemTable(_ptableVersion, maxSize: 5), () => fakeReader, _ptableVersion, 5, maxSizeForMemory: 5, maxTablesPerLevel: 2); _tableIndex.Initialize(long.MaxValue); _tableIndex.Add(1, "account--696193173", 2, 6); _tableIndex.Add(1, "LPN-FC002_LPK51001", 3, 7); _tableIndex.Add(1, "account--696193173", 3, 8); _tableIndex.Add(1, "LPN-FC002_LPK51001", 4, 9); _tableIndex.Add(1, "account--696193173", 4, 10); Thread.Sleep(500); }
private long GetStreamLastEventNumberUncached(TFReaderLease reader, string streamId) { IndexEntry latestEntry; if (!_tableIndex.TryGetLatestEntry(streamId, out latestEntry)) { return(ExpectedVersion.NoStream); } var rec = ReadPrepareInternal(reader, latestEntry.Position); if (rec == null) { throw new Exception( $"Could not read latest stream's prepare for stream '{streamId}' at position {latestEntry.Position}"); } int count = 0; long startVersion = 0; long latestVersion = long.MinValue; if (rec.EventStreamId == streamId) { startVersion = Math.Max(latestEntry.Version, latestEntry.Version + 1); latestVersion = latestEntry.Version; } foreach (var indexEntry in _tableIndex.GetRange(streamId, startVersion, long.MaxValue, limit: _hashCollisionReadLimit + 1)) { var r = ReadPrepareInternal(reader, indexEntry.Position); if (r != null && r.EventStreamId == streamId) { if (latestVersion == long.MinValue) { latestVersion = indexEntry.Version; continue; } return(latestVersion < indexEntry.Version ? indexEntry.Version : latestVersion); } count++; Interlocked.Increment(ref _hashCollisions); if (count > _hashCollisionReadLimit) { Log.Error("A hash collision resulted in not finding the last event number for the stream {stream}.", streamId); return(EventNumber.Invalid); } } return(latestVersion == long.MinValue ? ExpectedVersion.NoStream : latestVersion); }
public override async Task TestFixtureSetUp() { await base.TestFixtureSetUp(); _indexDir = PathName; var fakeReader = new TFReaderLease(new FakeIndexReader()); int readerCount = 0; _lowHasher = new XXHashUnsafe(); _highHasher = new Murmur3AUnsafe(); _tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher, () => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5), () => { readerCount++; if (readerCount < 4) // One for each table add. { return(fakeReader); } throw new Exception("Expected exception"); }, PTableVersions.IndexV4, 5, Constants.PTableMaxReaderCountDefault, maxSizeForMemory: 2, maxTablesPerLevel: 5); _tableIndex.Initialize(long.MaxValue); _tableIndex.Add(1, "testStream-1", 0, 0); _tableIndex.Add(1, "testStream-1", 1, 100); _tableIndex.Add(1, "testStream-1", 2, 200); _tableIndex.Add(1, "testStream-1", 3, 300); _tableIndex.Add(1, "testStream-1", 4, 400); _tableIndex.Add(1, "testStream-1", 5, 500); _log = new FakeTFScavengerLog(); Assert.That(() => _tableIndex.Scavenge(_log, CancellationToken.None), Throws.Exception.With.Message.EqualTo("Expected exception")); // Check it's loadable still. _tableIndex.Close(false); _tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher, () => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5), () => fakeReader, PTableVersions.IndexV4, 5, Constants.PTableMaxReaderCountDefault, maxSizeForMemory: 2, maxTablesPerLevel: 5); _tableIndex.Initialize(long.MaxValue); }
private bool OriginalStreamExists(TFReaderLease reader, string metaStreamId) { if (SystemStreams.IsSystemStream(metaStreamId)) { var originalStreamId = SystemStreams.OriginalStreamOf(metaStreamId); var lastEventNumber = GetStreamLastEventNumberCached(reader, originalStreamId); if (lastEventNumber == ExpectedVersion.NoStream || lastEventNumber == EventNumber.DeletedStream) { return(false); } return(true); } return(false); }
public override async Task TestFixtureSetUp() { await base.TestFixtureSetUp(); _indexDir = PathName; var cancellationTokenSource = new CancellationTokenSource(); var fakeReader = new TFReaderLease(new FakeIndexReader(l => { cancellationTokenSource.Cancel(); return(true); })); _lowHasher = new XXHashUnsafe(); _highHasher = new Murmur3AUnsafe(); _tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher, () => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5), () => fakeReader, PTableVersions.IndexV4, 5, Constants.PTableMaxReaderCountDefault, maxSizeForMemory: 2, maxTablesPerLevel: 5); _tableIndex.Initialize(long.MaxValue); _tableIndex.Add(1, "testStream-1", 0, 0); _tableIndex.Add(1, "testStream-1", 1, 100); _tableIndex.Add(1, "testStream-1", 2, 200); _tableIndex.Add(1, "testStream-1", 3, 300); _tableIndex.Add(1, "testStream-1", 4, 400); _tableIndex.Add(1, "testStream-1", 5, 500); _log = new FakeTFScavengerLog(); Assert.That(() => _tableIndex.Scavenge(_log, cancellationTokenSource.Token), Throws.InstanceOf <OperationCanceledException>()); // Check it's loadable still. _tableIndex.Close(false); _tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher, () => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5), () => fakeReader, PTableVersions.IndexV4, 5, Constants.PTableMaxReaderCountDefault, maxSizeForMemory: 2, maxTablesPerLevel: 5); _tableIndex.Initialize(long.MaxValue); }
private PrepareLogRecord ReadPrepare(TFReaderLease reader, string streamId, long eventNumber) { var recordsQuery = _tableIndex.GetRange(streamId, eventNumber, eventNumber) .Select(x => new { x.Version, Prepare = ReadPrepareInternal(reader, x.Position) }) .Where(x => x.Prepare != null && x.Prepare.EventStreamId == streamId) .GroupBy(x => x.Version).Select(x => x.Last()).ToList(); if (recordsQuery.Count() == 1) { return(recordsQuery.First().Prepare); } return(null); }
private static PrepareLogRecord ReadPrepareInternal(TFReaderLease reader, long logPosition) { RecordReadResult result = reader.TryReadAt(logPosition); if (!result.Success) { return(null); } if (result.LogRecord.RecordType != LogRecordType.Prepare) { throw new Exception(string.Format("Incorrect type of log record {0}, expected Prepare record.", result.LogRecord.RecordType)); } return((PrepareLogRecord)result.LogRecord); }
private static Tuple <string, bool> ReadEntry(TFReaderLease reader, long position) { RecordReadResult result = reader.TryReadAt(position); if (!result.Success) { return(new Tuple <string, bool>(String.Empty, false)); } if (result.LogRecord.RecordType != TransactionLog.LogRecords.LogRecordType.Prepare) { throw new Exception(string.Format("Incorrect type of log record {0}, expected Prepare record.", result.LogRecord.RecordType)); } return(new Tuple <string, bool>(((TransactionLog.LogRecords.PrepareLogRecord)result.LogRecord).EventStreamId, true)); }
private IndexReadEventResult ReadEventInternal(TFReaderLease reader, string streamId, int eventNumber) { var lastEventNumber = GetStreamLastEventNumberCached(reader, streamId); var metadata = GetStreamMetadataCached(reader, streamId); if (lastEventNumber == EventNumber.DeletedStream) { return(new IndexReadEventResult(ReadEventResult.StreamDeleted, metadata, lastEventNumber)); } if (lastEventNumber == ExpectedVersion.NoStream || metadata.TruncateBefore == EventNumber.DeletedStream) { return(new IndexReadEventResult(ReadEventResult.NoStream, metadata, lastEventNumber)); } if (eventNumber == -1) { eventNumber = lastEventNumber; } int minEventNumber = 0; if (metadata.MaxCount.HasValue) { minEventNumber = Math.Max(minEventNumber, lastEventNumber - metadata.MaxCount.Value + 1); } if (metadata.TruncateBefore.HasValue) { minEventNumber = Math.Max(minEventNumber, metadata.TruncateBefore.Value); } if (eventNumber < minEventNumber || eventNumber > lastEventNumber) { return(new IndexReadEventResult(ReadEventResult.NotFound, metadata, lastEventNumber)); } PrepareLogRecord prepare = ReadPrepareInternal(reader, streamId, eventNumber); if (prepare != null) { if (metadata.MaxAge.HasValue && prepare.TimeStamp < DateTime.UtcNow - metadata.MaxAge.Value) { return(new IndexReadEventResult(ReadEventResult.NotFound, metadata, lastEventNumber)); } return(new IndexReadEventResult(ReadEventResult.Success, new EventRecord(eventNumber, prepare), metadata, lastEventNumber)); } return(new IndexReadEventResult(ReadEventResult.NotFound, metadata, lastEventNumber)); }
public override async Task TestFixtureSetUp() { await base.TestFixtureSetUp(); _indexDir = PathName; var fakeReader = new TFReaderLease(new FakeIndexReader()); _lowHasher = new XXHashUnsafe(); _highHasher = new Murmur3AUnsafe(); _tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, "", () => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5), () => throw new Exception("Expected exception") /* throw an exception when the first PTable scavenge starts and tries to acquire a reader */, PTableVersions.IndexV4, 5, Constants.PTableMaxReaderCountDefault, maxSizeForMemory: 2, maxTablesPerLevel: 5, useBloomFilter: _useBloomFilter); _tableIndex.Initialize(long.MaxValue); _tableIndex.Add(1, "testStream-1", 0, 0); _tableIndex.Add(1, "testStream-1", 1, 100); _tableIndex.Add(1, "testStream-1", 2, 200); _tableIndex.Add(1, "testStream-1", 3, 300); _tableIndex.Add(1, "testStream-1", 4, 400); _tableIndex.Add(1, "testStream-1", 5, 500); _log = new FakeTFScavengerLog(); Assert.That(() => _tableIndex.Scavenge(_log, CancellationToken.None), Throws.Exception.With.Message.EqualTo("Expected exception")); // Check it's loadable still. _tableIndex.Close(false); _tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, "", () => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5), () => fakeReader, PTableVersions.IndexV4, 5, Constants.PTableMaxReaderCountDefault, maxSizeForMemory: 2, maxTablesPerLevel: 5, useBloomFilter: _useBloomFilter); _tableIndex.Initialize(long.MaxValue); }
public override async Task TestFixtureSetUp() { await base.TestFixtureSetUp(); _indexDir = PathName; var fakeReader = new TFReaderLease(new FakeIndexReader(l => !Deleted.Contains(l))); _lowHasher = new XXHashUnsafe(); _highHasher = new Murmur3AUnsafe(); _tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, "", () => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5), () => fakeReader, PTableVersions.IndexV4, 5, Constants.PTableMaxReaderCountDefault, maxSizeForMemory: 2, maxTablesPerLevel: 5, skipIndexVerify: _skipIndexVerify, useBloomFilter: _useBloomFilter); _tableIndex.Initialize(long.MaxValue); _tableIndex.Add(1, "testStream-1", 0, 0); _tableIndex.Add(1, "testStream-1", 1, 100); _tableIndex.Add(1, "testStream-1", 2, 200); _tableIndex.Add(1, "testStream-1", 3, 300); _tableIndex.Add(1, "testStream-1", 4, 400); _tableIndex.Add(1, "testStream-1", 5, 500); _log = new FakeTFScavengerLog(); _tableIndex.Scavenge(_log, CancellationToken.None); // Check it's loadable. _tableIndex.Close(false); _tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, "", () => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5), () => fakeReader, PTableVersions.IndexV4, 5, Constants.PTableMaxReaderCountDefault, maxSizeForMemory: 2, maxTablesPerLevel: 5, useBloomFilter: _useBloomFilter); _tableIndex.Initialize(long.MaxValue); }
private PrepareLogRecord ReadPrepareInternal(TFReaderLease reader, string streamId, int eventNumber) { // we assume that you already did check for stream deletion Ensure.NotNullOrEmpty(streamId, "streamId"); Ensure.Nonnegative(eventNumber, "eventNumber"); var recordsQuery = _tableIndex.GetRange(streamId, eventNumber, eventNumber) .Select(x => new { x.Version, Prepare = ReadPrepareInternal(reader, x.Position) }) .Where(x => x.Prepare != null && x.Prepare.EventStreamId == streamId) .GroupBy(x => x.Version).Select(x => x.Last()); if (recordsQuery.Count() == 1) { return(recordsQuery.First().Prepare); } return(null); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _indexDir = PathName; var fakeReader = new TFReaderLease(new FakeIndexReader()); _lowHasher = new FakeIndexHasher(); _highHasher = new FakeIndexHasher(); _tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher, () => new HashListMemTable(_ptableVersion, maxSize: 10), () => fakeReader, _ptableVersion, 5, maxSizeForMemory: 2, maxTablesPerLevel: 2, skipIndexVerify: _skipIndexVerify); _tableIndex.Initialize(long.MaxValue); // ptable level 2 _tableIndex.Add(0, "1", 0, 0xFF00); _tableIndex.Add(0, "1", 1, 0xFF01); _tableIndex.Add(0, "2", 0, 0xFF00); _tableIndex.Add(0, "2", 1, 0xFF01); _tableIndex.Add(0, "3", 0, 0xFF00); _tableIndex.Add(0, "3", 1, 0xFF01); _tableIndex.Add(0, "3", 0, 0xFF02); _tableIndex.Add(0, "3", 1, 0xFF03); // ptable level 1 _tableIndex.Add(0, "4", 0, 0xFF00); _tableIndex.Add(0, "5", 10, 0xFFF1); _tableIndex.Add(0, "6", 0, 0xFF00); _tableIndex.Add(0, "1", 0, 0xFF10); // ptable level 0 _tableIndex.Add(0, "6", 1, 0xFF01); _tableIndex.Add(0, "1", 1, 0xFF11); // memtable _tableIndex.Add(0, "4", 0, 0xFF01); Thread.Sleep(500); }
public override async Task TestFixtureSetUp() { await base.TestFixtureSetUp(); _indexDir = PathName; var fakeReader = new TFReaderLease(new FakeIndexReader()); _lowHasher = new FakeIndexHasher(); _highHasher = new FakeIndexHasher(); _tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher, () => new HashListMemTable(_ptableVersion, maxSize: 10), () => fakeReader, _ptableVersion, 5, Constants.PTableMaxReaderCountDefault, maxSizeForMemory: 2, maxTablesPerLevel: 2, skipIndexVerify: _skipIndexVerify); _tableIndex.Initialize(long.MaxValue); // ptable level 2 _tableIndex.Add(0, "1", 0, 0xFF00); _tableIndex.Add(0, "1", 1, 0xFF01); _tableIndex.Add(0, "2", 0, 0xFF00); _tableIndex.Add(0, "2", 1, 0xFF01); _tableIndex.Add(0, "3", 0, 0xFF00); _tableIndex.Add(0, "3", 1, 0xFF01); _tableIndex.Add(0, "3", 0, 0xFF02); _tableIndex.Add(0, "3", 1, 0xFF03); // ptable level 1 _tableIndex.Add(0, "4", 0, 0xFF00); _tableIndex.Add(0, "5", 10, 0xFFF1); _tableIndex.Add(0, "6", 0, 0xFF00); _tableIndex.Add(0, "1", 0, 0xFF10); // ptable level 0 _tableIndex.Add(0, "6", 1, 0xFF01); _tableIndex.Add(0, "1", 1, 0xFF11); // memtable _tableIndex.Add(0, "4", 0, 0xFF01); await Task.Delay(500); }
public override async Task TestFixtureSetUp() { await base.TestFixtureSetUp(); _indexDir = PathName; var fakeReader = new TFReaderLease(new FakeIndexReader()); _lowHasher = new XXHashUnsafe(); _highHasher = new Murmur3AUnsafe(); _tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher, () => new HashListMemTable(PTableVersions.IndexV2, maxSize: 5), () => fakeReader, PTableVersions.IndexV2, 5, Constants.PTableMaxReaderCountDefault, maxSizeForMemory: 5, maxTablesPerLevel: 2); _tableIndex.Initialize(long.MaxValue); _tableIndex.Add(1, "testStream-1", 0, 1); _tableIndex.Add(1, "testStream-2", 0, 2); _tableIndex.Add(1, "testStream-1", 1, 3); _tableIndex.Add(1, "testStream-2", 1, 4); _tableIndex.Add(1, "testStream-1", 2, 5); _tableIndex.Close(false); _tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher, () => new HashListMemTable(_ptableVersion, maxSize: 5), () => fakeReader, _ptableVersion, 5, Constants.PTableMaxReaderCountDefault, maxSizeForMemory: 5, maxTablesPerLevel: 2); _tableIndex.Initialize(long.MaxValue); _tableIndex.Add(1, "testStream-2", 2, 6); _tableIndex.Add(1, "testStream-1", 3, 7); _tableIndex.Add(1, "testStream-2", 3, 8); _tableIndex.Add(1, "testStream-1", 4, 9); _tableIndex.Add(1, "testStream-2", 4, 10); await Task.Delay(500); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _indexDir = PathName; var fakeReader = new TFReaderLease(new FakeIndexReader()); _lowHasher = new XXHashUnsafe(); _highHasher = new Murmur3AUnsafe(); _tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher, () => new HashListMemTable(PTableVersions.IndexV1, maxSize: 5), () => fakeReader, PTableVersions.IndexV1, maxSizeForMemory: 5, maxTablesPerLevel: 2); _tableIndex.Initialize(long.MaxValue); _tableIndex.Add(1, "LPN-FC002_LPK51001", 0, 1); _tableIndex.Add(1, "account--696193173", 0, 2); _tableIndex.Add(1, "LPN-FC002_LPK51001", 1, 3); _tableIndex.Add(1, "account--696193173", 1, 4); _tableIndex.Add(1, "LPN-FC002_LPK51001", 2, 5); _tableIndex.Close(false); _tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher, () => new HashListMemTable(_ptableVersion, maxSize: 5), () => fakeReader, _ptableVersion, maxSizeForMemory: 5, maxTablesPerLevel: 2); _tableIndex.Initialize(long.MaxValue); _tableIndex.Add(1, "account--696193173", 2, 6); _tableIndex.Add(1, "LPN-FC002_LPK51001", 3, 7); _tableIndex.Add(1, "account--696193173", 3, 8); _tableIndex.Add(1, "LPN-FC002_LPK51001", 4, 9); _tableIndex.Add(1, "account--696193173", 4, 10); Thread.Sleep(500); }
public void Setup() { given(); _indexDir = PathName; _fakeReader = new TFReaderLease(new FakeReader()); _indexBackend = new FakeIndexBackend(_fakeReader); _lowHasher = new XXHashUnsafe(); _highHasher = new Murmur3AUnsafe(); _tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher, () => new HashListMemTable(PTableVersions.IndexV1, maxSize: _maxMemTableSize), () => _fakeReader, PTableVersions.IndexV1, maxSizeForMemory: _maxMemTableSize, maxTablesPerLevel: 2); _tableIndex.Initialize(long.MaxValue); _indexReader = new IndexReader(_indexBackend, _tableIndex, new EventStore.Core.Data.StreamMetadata(), _hashCollisionReadLimit); when(); //wait for the mem table to be dumped System.Threading.Thread.Sleep(500); }
private int GetStreamLastEventNumberUncached(TFReaderLease reader, string streamId) { IndexEntry latestEntry; if (!_tableIndex.TryGetLatestEntry(streamId, out latestEntry)) { return(ExpectedVersion.NoStream); } var rec = ReadPrepareInternal(reader, latestEntry.Position); if (rec == null) { throw new Exception("Could not read latest stream's prepare. That should never happen."); } if (rec.EventStreamId == streamId) // LUCKY!!! { return(latestEntry.Version); } int count = 0; foreach (var indexEntry in _tableIndex.GetRange(streamId, 0, int.MaxValue, limit: _hashCollisionReadLimit)) { var r = ReadPrepareInternal(reader, indexEntry.Position); if (r != null && r.EventStreamId == streamId) { return(indexEntry.Version); // AT LAST!!! } count++; Interlocked.Increment(ref _hashCollisions); if (count > _hashCollisionReadLimit) { Log.Error("A hash collision resulted in not finding the last event number for the stream {0}", streamId); return(ExpectedVersion.NoStream); } } return(ExpectedVersion.NoStream); // no such event stream }