private PrepareLogRecord ReadPrepareSkipScan(TFReaderLease reader, string streamId, long eventNumber)
{
long position;
if (_tableIndex.TryGetOneValue(streamId, eventNumber, out position))
{
var rec = ReadPrepareInternal(reader, position);
if (rec != null && rec.EventStreamId == streamId)
{
return(rec);
}
foreach (var indexEntry in _tableIndex.GetRange(streamId, eventNumber, eventNumber))
{
Interlocked.Increment(ref _hashCollisions);
if (indexEntry.Position == position)
{
continue;
}
rec = ReadPrepareInternal(reader, indexEntry.Position);
if (rec != null && rec.EventStreamId == streamId)
{
return(rec);
}
}
}
return(null);
}
public void Setup()
{
given();
_indexDir = PathName;
_fakeReader = new TFReaderLease(new FakeReader());
_indexBackend = new FakeIndexBackend <string>(_fakeReader);
_logFormat = LogFormatHelper <LogFormat.V2, string> .LogFormatFactory.Create(new() {
InMemory = true,
});
_lowHasher = _logFormat.LowHasher;
_highHasher = _logFormat.HighHasher;
_tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, _logFormat.EmptyStreamId,
() => new HashListMemTable(PTableVersions.IndexV1, maxSize: _maxMemTableSize),
() => _fakeReader,
PTableVersions.IndexV1,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: _maxMemTableSize,
maxTablesPerLevel: 2);
_logFormat.StreamNamesProvider.SetTableIndex(_tableIndex);
_tableIndex.Initialize(long.MaxValue);
_indexReader = new IndexReader <string>(_indexBackend, _tableIndex,
_logFormat.StreamNamesProvider,
_logFormat.StreamIdValidator,
_logFormat.StreamExistenceFilterReader,
new EventStore.Core.Data.StreamMetadata(),
_hashCollisionReadLimit, skipIndexScanOnRead: false);
when();
//wait for the mem table to be dumped
System.Threading.Thread.Sleep(500);
}
private PrepareLogRecord ReadPrepareInternal(TFReaderLease reader, string streamId, int eventNumber)
{
// we assume that you already did check for stream deletion
Ensure.NotNullOrEmpty(streamId, "streamId");
Ensure.Nonnegative(eventNumber, "eventNumber");
var streamHash = _hasher.Hash(streamId);
long position;
if (_tableIndex.TryGetOneValue(streamHash, eventNumber, out position))
{
var rec = ReadPrepareInternal(reader, position);
if (rec != null && rec.EventStreamId == streamId)
{
return(rec);
}
foreach (var indexEntry in _tableIndex.GetRange(streamHash, eventNumber, eventNumber))
{
Interlocked.Increment(ref _hashCollisions);
if (indexEntry.Position == position) // already checked that
{
continue;
}
rec = ReadPrepareInternal(reader, indexEntry.Position);
if (rec != null && rec.EventStreamId == streamId)
{
return(rec);
}
}
}
return(null);
}
private IndexReadEventResult ReadEventInternal(TFReaderLease reader, string streamId, long eventNumber)
{
var lastEventNumber = GetStreamLastEventNumberCached(reader, streamId);
var metadata = GetStreamMetadataCached(reader, streamId);
var originalStreamExists = OriginalStreamExists(reader, streamId);
if (lastEventNumber == EventNumber.DeletedStream)
{
return(new IndexReadEventResult(ReadEventResult.StreamDeleted, metadata, lastEventNumber,
originalStreamExists));
}
if (lastEventNumber == ExpectedVersion.NoStream || metadata.TruncateBefore == EventNumber.DeletedStream)
{
return(new IndexReadEventResult(ReadEventResult.NoStream, metadata, lastEventNumber,
originalStreamExists));
}
if (lastEventNumber == EventNumber.Invalid)
{
return(new IndexReadEventResult(ReadEventResult.NoStream, metadata, lastEventNumber,
originalStreamExists));
}
if (eventNumber == -1)
{
eventNumber = lastEventNumber;
}
long minEventNumber = 0;
if (metadata.MaxCount.HasValue)
{
minEventNumber = Math.Max(minEventNumber, lastEventNumber - metadata.MaxCount.Value + 1);
}
if (metadata.TruncateBefore.HasValue)
{
minEventNumber = Math.Max(minEventNumber, metadata.TruncateBefore.Value);
}
//TODO(clc): confirm this logic, it seems that reads less than min should be invaild rather than found
if (eventNumber < minEventNumber || eventNumber > lastEventNumber)
{
return(new IndexReadEventResult(ReadEventResult.NotFound, metadata, lastEventNumber,
originalStreamExists));
}
PrepareLogRecord prepare = ReadPrepareInternal(reader, streamId, eventNumber);
if (prepare != null)
{
if (metadata.MaxAge.HasValue && prepare.TimeStamp < DateTime.UtcNow - metadata.MaxAge.Value)
{
return(new IndexReadEventResult(ReadEventResult.NotFound, metadata, lastEventNumber,
originalStreamExists));
}
return(new IndexReadEventResult(ReadEventResult.Success, new EventRecord(eventNumber, prepare),
metadata, lastEventNumber, originalStreamExists));
}
return(new IndexReadEventResult(ReadEventResult.NotFound, metadata, lastEventNumber,
originalStreamExists: originalStreamExists));
}
private int GetStreamLastEventNumberCached(TFReaderLease reader, string streamId)
{
// if this is metastream -- check if original stream was deleted, if yes -- metastream is deleted as well
if (SystemStreams.IsMetastream(streamId) &&
GetStreamLastEventNumberCached(reader, SystemStreams.OriginalStreamOf(streamId)) == EventNumber.DeletedStream)
{
return(EventNumber.DeletedStream);
}
var cache = _cache.TryGetStreamLastEventNumber(streamId);
if (cache.LastEventNumber != null)
{
Interlocked.Increment(ref _cachedStreamInfo);
return(cache.LastEventNumber.GetValueOrDefault());
}
Interlocked.Increment(ref _notCachedStreamInfo);
var lastEventNumber = GetStreamLastEventNumberUncached(reader, streamId);
// Conditional update depending on previously returned cache info version.
// If version is not correct -- nothing is changed in cache.
// This update is conditioned to not interfere with updating stream cache info by commit procedure
// (which is the source of truth).
var res = _cache.UpdateStreamLastEventNumber(cache.Version, streamId, lastEventNumber);
return(res ?? lastEventNumber);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeTfReader());
_tableIndex = new TableIndex(_indexDir,
() => new HashListMemTable(maxSize: 10),
() => fakeReader,
maxSizeForMemory: 5);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(0, 0xDEAD, 0, 0xFF00);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF01);
_tableIndex.Add(0, 0xBEEF, 0, 0xFF00);
_tableIndex.Add(0, 0xBEEF, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 0, 0xFF00); // 1st ptable0
_tableIndex.Add(0, 0xABBA, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 2, 0xFF02);
_tableIndex.Add(0, 0xABBA, 3, 0xFF03);
_tableIndex.Add(0, 0xADA, 0, 0xFF00); // simulates duplicate due to concurrency in TableIndex (see memtable below)
_tableIndex.Add(0, 0xDEAD, 0, 0xFF10); // 2nd ptable0
_tableIndex.Add(0, 0xDEAD, 1, 0xFF11); // in memtable
_tableIndex.Add(0, 0xADA, 0, 0xFF00); // in memtable
}
private int GetStreamLastEventNumberUncached(TFReaderLease reader, string streamId)
{
var streamHash = _hasher.Hash(streamId);
IndexEntry latestEntry;
if (!_tableIndex.TryGetLatestEntry(streamHash, out latestEntry))
{
return(ExpectedVersion.NoStream);
}
var rec = ReadPrepareInternal(reader, latestEntry.Position);
if (rec == null)
{
throw new Exception("Couldn't read latest stream's prepare! That shouldn't happen EVER!");
}
if (rec.EventStreamId == streamId) // LUCKY!!!
{
return(latestEntry.Version);
}
// TODO AN here lies the problem of out of memory if the stream has A LOT of events in them
foreach (var indexEntry in _tableIndex.GetRange(streamHash, 0, int.MaxValue))
{
var r = ReadPrepareInternal(reader, indexEntry.Position);
if (r != null && r.EventStreamId == streamId)
{
return(indexEntry.Version); // AT LAST!!!
}
Interlocked.Increment(ref _hashCollisions);
}
return(ExpectedVersion.NoStream); // no such event stream
}
private StreamMetadata GetStreamMetadataCached(TFReaderLease reader, string streamId)
{
// if this is metastream -- check if original stream was deleted, if yes -- metastream is deleted as well
if (SystemStreams.IsMetastream(streamId))
{
return(_metastreamMetadata);
}
var cache = _cache.TryGetStreamMetadata(streamId);
if (cache.Metadata != null)
{
Interlocked.Increment(ref _cachedStreamInfo);
return(cache.Metadata);
}
Interlocked.Increment(ref _notCachedStreamInfo);
var streamMetadata = GetStreamMetadataUncached(reader, streamId);
// Conditional update depending on previously returned cache info version.
// If version is not correct -- nothing is changed in cache.
// This update is conditioned to not interfere with updating stream cache info by commit procedure
// (which is the source of truth).
var res = _cache.UpdateStreamMetadata(cache.Version, streamId, streamMetadata);
return(res ?? streamMetadata);
}
private StreamMetadata GetStreamMetadataUncached(TFReaderLease reader, string streamId)
{
var metastreamId = SystemStreams.MetastreamOf(streamId);
var metaEventNumber = GetStreamLastEventNumberCached(reader, metastreamId);
if (metaEventNumber == ExpectedVersion.NoStream || metaEventNumber == EventNumber.DeletedStream)
{
return(StreamMetadata.Empty);
}
PrepareLogRecord prepare = ReadPrepareInternal(reader, metastreamId, metaEventNumber);
if (prepare == null)
{
throw new Exception(string.Format("ReadPrepareInternal couldn't find metaevent #{0} on metastream '{1}'. "
+ "That should never happen.", metaEventNumber, metastreamId));
}
if (prepare.Data.Length == 0 || prepare.Flags.HasNoneOf(PrepareFlags.IsJson))
{
return(StreamMetadata.Empty);
}
try
{
return(StreamMetadata.FromJsonBytes(prepare.Data));
}
catch (Exception)
{
return(StreamMetadata.Empty);
}
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeTfReader());
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 10),
() => fakeReader,
_ptableVersion,
maxSizeForMemory: 5,
skipIndexVerify: _skipIndexVerify);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(0, "0xDEAD", 0, 0xFF00);
_tableIndex.Add(0, "0xDEAD", 1, 0xFF01);
_tableIndex.Add(0, "0xBEEF", 0, 0xFF00);
_tableIndex.Add(0, "0xBEEF", 1, 0xFF01);
_tableIndex.Add(0, "0xABBA", 0, 0xFF00); // 1st ptable0
_tableIndex.Add(0, "0xABBA", 1, 0xFF01);
_tableIndex.Add(0, "0xABBA", 2, 0xFF02);
_tableIndex.Add(0, "0xABBA", 3, 0xFF03);
_tableIndex.Add(0, "0xADA", 0, 0xFF00); // simulates duplicate due to concurrency in TableIndex (see memtable below)
_tableIndex.Add(0, "0xDEAD", 0, 0xFF10); // 2nd ptable0
_tableIndex.Add(0, "0xDEAD", 1, 0xFF11); // in memtable
_tableIndex.Add(0, "0xADA", 0, 0xFF00); // in memtable
}
public void ConstructTableIndexWithCorruptIndexEntries(byte version, bool skipIndexVerify, bool createForceVerifyFile = false)
{
base.TestFixtureSetUp();
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
var fakeReader = new TFReaderLease(new FakeIndexReader());
_tableIndex = new TableIndex(PathName, lowHasher, highHasher,
() => new HashListMemTable(version, maxSize: NumIndexEntries),
() => fakeReader,
version,
5,
maxSizeForMemory: NumIndexEntries,
skipIndexVerify: skipIndexVerify);
_tableIndex.Initialize(long.MaxValue);
//create index entries
for (int i = 1; i <= NumIndexEntries; i++)
{
_tableIndex.Add(i * 1337, StreamName, i, i * 1337);
}
_tableIndex.Close(false);
//load index map to obtain ptable filenames
_indexMap = IndexMap.FromFile(Path.Combine(PathName, TableIndex.IndexMapFilename));
List <string> ptableFiles = new List <string>();
foreach (string ptableFilename in _indexMap.GetAllFilenames())
{
ptableFiles.Add(ptableFilename);
}
_indexMap.Dispose(TimeSpan.FromSeconds(5));
//corrupt ptable files
foreach (string ptableFilename in ptableFiles)
{
CorruptPTableFile(ptableFilename, version, "zeroOutMiddleEntry");
}
//create force verify file if requested
if (createForceVerifyFile)
{
using (FileStream fs = new FileStream(Path.Combine(PathName, TableIndex.ForceIndexVerifyFilename), FileMode.OpenOrCreate)){
};
}
//load table index again
_tableIndex = new TableIndex(PathName, lowHasher, highHasher,
() => new HashListMemTable(version, maxSize: NumIndexEntries),
() => fakeReader,
version,
5,
maxSizeForMemory: NumIndexEntries,
skipIndexVerify: skipIndexVerify,
indexCacheDepth: 8);
_tableIndex.Initialize(long.MaxValue);
}
private PrepareLogRecord ReadPrepareInternal(TFReaderLease reader, string streamId, long eventNumber)
{
// we assume that you already did check for stream deletion
Ensure.NotNullOrEmpty(streamId, "streamId");
Ensure.Nonnegative(eventNumber, "eventNumber");
return(_skipIndexScanOnRead ? ReadPrepareSkipScan(reader, streamId, eventNumber) :
ReadPrepare(reader, streamId, eventNumber));
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV1, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV1,
5,
maxSizeForMemory: 5 + _extraStreamHashesAtBeginning + _extraStreamHashesAtEnd,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
Assert.Greater(_lowHasher.Hash("abcd"), _lowHasher.Hash("LPN-FC002_LPK51001"));
for (int i = 0; i < _extraStreamHashesAtBeginning; i++)
{
_tableIndex.Add(1, "abcd", i, i + 1);
}
Assert.Less(_lowHasher.Hash("wxyz"), _lowHasher.Hash("LPN-FC002_LPK51001"));
for (int i = 0; i < _extraStreamHashesAtEnd; i++)
{
_tableIndex.Add(1, "wxyz", i, i + 1);
}
_tableIndex.Add(1, "LPN-FC002_LPK51001", 0, 1);
_tableIndex.Add(1, "account--696193173", 0, 2);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 1, 3);
_tableIndex.Add(1, "account--696193173", 1, 4);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 2, 5);
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 5),
() => fakeReader,
_ptableVersion,
5,
maxSizeForMemory: 5,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "account--696193173", 2, 6);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 3, 7);
_tableIndex.Add(1, "account--696193173", 3, 8);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 4, 9);
_tableIndex.Add(1, "account--696193173", 4, 10);
Thread.Sleep(500);
}
private long GetStreamLastEventNumberUncached(TFReaderLease reader, string streamId)
{
IndexEntry latestEntry;
if (!_tableIndex.TryGetLatestEntry(streamId, out latestEntry))
{
return(ExpectedVersion.NoStream);
}
var rec = ReadPrepareInternal(reader, latestEntry.Position);
if (rec == null)
{
throw new Exception(
$"Could not read latest stream's prepare for stream '{streamId}' at position {latestEntry.Position}");
}
int count = 0;
long startVersion = 0;
long latestVersion = long.MinValue;
if (rec.EventStreamId == streamId)
{
startVersion = Math.Max(latestEntry.Version, latestEntry.Version + 1);
latestVersion = latestEntry.Version;
}
foreach (var indexEntry in _tableIndex.GetRange(streamId, startVersion, long.MaxValue,
limit: _hashCollisionReadLimit + 1))
{
var r = ReadPrepareInternal(reader, indexEntry.Position);
if (r != null && r.EventStreamId == streamId)
{
if (latestVersion == long.MinValue)
{
latestVersion = indexEntry.Version;
continue;
}
return(latestVersion < indexEntry.Version ? indexEntry.Version : latestVersion);
}
count++;
Interlocked.Increment(ref _hashCollisions);
if (count > _hashCollisionReadLimit)
{
Log.Error("A hash collision resulted in not finding the last event number for the stream {stream}.",
streamId);
return(EventNumber.Invalid);
}
}
return(latestVersion == long.MinValue ? ExpectedVersion.NoStream : latestVersion);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
int readerCount = 0;
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => {
readerCount++;
if (readerCount < 4) // One for each table add.
{
return(fakeReader);
}
throw new Exception("Expected exception");
},
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-1", 0, 0);
_tableIndex.Add(1, "testStream-1", 1, 100);
_tableIndex.Add(1, "testStream-1", 2, 200);
_tableIndex.Add(1, "testStream-1", 3, 300);
_tableIndex.Add(1, "testStream-1", 4, 400);
_tableIndex.Add(1, "testStream-1", 5, 500);
_log = new FakeTFScavengerLog();
Assert.That(() => _tableIndex.Scavenge(_log, CancellationToken.None),
Throws.Exception.With.Message.EqualTo("Expected exception"));
// Check it's loadable still.
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5);
_tableIndex.Initialize(long.MaxValue);
}
private bool OriginalStreamExists(TFReaderLease reader, string metaStreamId)
{
if (SystemStreams.IsSystemStream(metaStreamId))
{
var originalStreamId = SystemStreams.OriginalStreamOf(metaStreamId);
var lastEventNumber = GetStreamLastEventNumberCached(reader, originalStreamId);
if (lastEventNumber == ExpectedVersion.NoStream || lastEventNumber == EventNumber.DeletedStream)
{
return(false);
}
return(true);
}
return(false);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var cancellationTokenSource = new CancellationTokenSource();
var fakeReader = new TFReaderLease(new FakeIndexReader(l => {
cancellationTokenSource.Cancel();
return(true);
}));
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-1", 0, 0);
_tableIndex.Add(1, "testStream-1", 1, 100);
_tableIndex.Add(1, "testStream-1", 2, 200);
_tableIndex.Add(1, "testStream-1", 3, 300);
_tableIndex.Add(1, "testStream-1", 4, 400);
_tableIndex.Add(1, "testStream-1", 5, 500);
_log = new FakeTFScavengerLog();
Assert.That(() => _tableIndex.Scavenge(_log, cancellationTokenSource.Token),
Throws.InstanceOf <OperationCanceledException>());
// Check it's loadable still.
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5);
_tableIndex.Initialize(long.MaxValue);
}
private PrepareLogRecord ReadPrepare(TFReaderLease reader, string streamId, long eventNumber)
{
var recordsQuery = _tableIndex.GetRange(streamId, eventNumber, eventNumber)
.Select(x => new { x.Version, Prepare = ReadPrepareInternal(reader, x.Position) })
.Where(x => x.Prepare != null && x.Prepare.EventStreamId == streamId)
.GroupBy(x => x.Version).Select(x => x.Last()).ToList();
if (recordsQuery.Count() == 1)
{
return(recordsQuery.First().Prepare);
}
return(null);
}
private static PrepareLogRecord ReadPrepareInternal(TFReaderLease reader, long logPosition)
{
RecordReadResult result = reader.TryReadAt(logPosition);
if (!result.Success)
{
return(null);
}
if (result.LogRecord.RecordType != LogRecordType.Prepare)
{
throw new Exception(string.Format("Incorrect type of log record {0}, expected Prepare record.",
result.LogRecord.RecordType));
}
return((PrepareLogRecord)result.LogRecord);
}
private static Tuple <string, bool> ReadEntry(TFReaderLease reader, long position)
{
RecordReadResult result = reader.TryReadAt(position);
if (!result.Success)
{
return(new Tuple <string, bool>(String.Empty, false));
}
if (result.LogRecord.RecordType != TransactionLog.LogRecords.LogRecordType.Prepare)
{
throw new Exception(string.Format("Incorrect type of log record {0}, expected Prepare record.",
result.LogRecord.RecordType));
}
return(new Tuple <string, bool>(((TransactionLog.LogRecords.PrepareLogRecord)result.LogRecord).EventStreamId, true));
}
private IndexReadEventResult ReadEventInternal(TFReaderLease reader, string streamId, int eventNumber)
{
var lastEventNumber = GetStreamLastEventNumberCached(reader, streamId);
var metadata = GetStreamMetadataCached(reader, streamId);
if (lastEventNumber == EventNumber.DeletedStream)
{
return(new IndexReadEventResult(ReadEventResult.StreamDeleted, metadata, lastEventNumber));
}
if (lastEventNumber == ExpectedVersion.NoStream || metadata.TruncateBefore == EventNumber.DeletedStream)
{
return(new IndexReadEventResult(ReadEventResult.NoStream, metadata, lastEventNumber));
}
if (eventNumber == -1)
{
eventNumber = lastEventNumber;
}
int minEventNumber = 0;
if (metadata.MaxCount.HasValue)
{
minEventNumber = Math.Max(minEventNumber, lastEventNumber - metadata.MaxCount.Value + 1);
}
if (metadata.TruncateBefore.HasValue)
{
minEventNumber = Math.Max(minEventNumber, metadata.TruncateBefore.Value);
}
if (eventNumber < minEventNumber || eventNumber > lastEventNumber)
{
return(new IndexReadEventResult(ReadEventResult.NotFound, metadata, lastEventNumber));
}
PrepareLogRecord prepare = ReadPrepareInternal(reader, streamId, eventNumber);
if (prepare != null)
{
if (metadata.MaxAge.HasValue && prepare.TimeStamp < DateTime.UtcNow - metadata.MaxAge.Value)
{
return(new IndexReadEventResult(ReadEventResult.NotFound, metadata, lastEventNumber));
}
return(new IndexReadEventResult(ReadEventResult.Success, new EventRecord(eventNumber, prepare), metadata, lastEventNumber));
}
return(new IndexReadEventResult(ReadEventResult.NotFound, metadata, lastEventNumber));
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, "",
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => throw new Exception("Expected exception") /* throw an exception when the first PTable scavenge starts and tries to acquire a reader */,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5,
useBloomFilter: _useBloomFilter);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-1", 0, 0);
_tableIndex.Add(1, "testStream-1", 1, 100);
_tableIndex.Add(1, "testStream-1", 2, 200);
_tableIndex.Add(1, "testStream-1", 3, 300);
_tableIndex.Add(1, "testStream-1", 4, 400);
_tableIndex.Add(1, "testStream-1", 5, 500);
_log = new FakeTFScavengerLog();
Assert.That(() => _tableIndex.Scavenge(_log, CancellationToken.None),
Throws.Exception.With.Message.EqualTo("Expected exception"));
// Check it's loadable still.
_tableIndex.Close(false);
_tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, "",
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5,
useBloomFilter: _useBloomFilter);
_tableIndex.Initialize(long.MaxValue);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader(l => !Deleted.Contains(l)));
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, "",
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5, skipIndexVerify: _skipIndexVerify,
useBloomFilter: _useBloomFilter);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-1", 0, 0);
_tableIndex.Add(1, "testStream-1", 1, 100);
_tableIndex.Add(1, "testStream-1", 2, 200);
_tableIndex.Add(1, "testStream-1", 3, 300);
_tableIndex.Add(1, "testStream-1", 4, 400);
_tableIndex.Add(1, "testStream-1", 5, 500);
_log = new FakeTFScavengerLog();
_tableIndex.Scavenge(_log, CancellationToken.None);
// Check it's loadable.
_tableIndex.Close(false);
_tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, "",
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5,
useBloomFilter: _useBloomFilter);
_tableIndex.Initialize(long.MaxValue);
}
private PrepareLogRecord ReadPrepareInternal(TFReaderLease reader, string streamId, int eventNumber)
{
// we assume that you already did check for stream deletion
Ensure.NotNullOrEmpty(streamId, "streamId");
Ensure.Nonnegative(eventNumber, "eventNumber");
var recordsQuery = _tableIndex.GetRange(streamId, eventNumber, eventNumber)
.Select(x => new { x.Version, Prepare = ReadPrepareInternal(reader, x.Position) })
.Where(x => x.Prepare != null && x.Prepare.EventStreamId == streamId)
.GroupBy(x => x.Version).Select(x => x.Last());
if (recordsQuery.Count() == 1)
{
return(recordsQuery.First().Prepare);
}
return(null);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
_lowHasher = new FakeIndexHasher();
_highHasher = new FakeIndexHasher();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 10),
() => fakeReader,
_ptableVersion,
5,
maxSizeForMemory: 2,
maxTablesPerLevel: 2,
skipIndexVerify: _skipIndexVerify);
_tableIndex.Initialize(long.MaxValue);
// ptable level 2
_tableIndex.Add(0, "1", 0, 0xFF00);
_tableIndex.Add(0, "1", 1, 0xFF01);
_tableIndex.Add(0, "2", 0, 0xFF00);
_tableIndex.Add(0, "2", 1, 0xFF01);
_tableIndex.Add(0, "3", 0, 0xFF00);
_tableIndex.Add(0, "3", 1, 0xFF01);
_tableIndex.Add(0, "3", 0, 0xFF02);
_tableIndex.Add(0, "3", 1, 0xFF03);
// ptable level 1
_tableIndex.Add(0, "4", 0, 0xFF00);
_tableIndex.Add(0, "5", 10, 0xFFF1);
_tableIndex.Add(0, "6", 0, 0xFF00);
_tableIndex.Add(0, "1", 0, 0xFF10);
// ptable level 0
_tableIndex.Add(0, "6", 1, 0xFF01);
_tableIndex.Add(0, "1", 1, 0xFF11);
// memtable
_tableIndex.Add(0, "4", 0, 0xFF01);
Thread.Sleep(500);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
_lowHasher = new FakeIndexHasher();
_highHasher = new FakeIndexHasher();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 10),
() => fakeReader,
_ptableVersion,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 2,
skipIndexVerify: _skipIndexVerify);
_tableIndex.Initialize(long.MaxValue);
// ptable level 2
_tableIndex.Add(0, "1", 0, 0xFF00);
_tableIndex.Add(0, "1", 1, 0xFF01);
_tableIndex.Add(0, "2", 0, 0xFF00);
_tableIndex.Add(0, "2", 1, 0xFF01);
_tableIndex.Add(0, "3", 0, 0xFF00);
_tableIndex.Add(0, "3", 1, 0xFF01);
_tableIndex.Add(0, "3", 0, 0xFF02);
_tableIndex.Add(0, "3", 1, 0xFF03);
// ptable level 1
_tableIndex.Add(0, "4", 0, 0xFF00);
_tableIndex.Add(0, "5", 10, 0xFFF1);
_tableIndex.Add(0, "6", 0, 0xFF00);
_tableIndex.Add(0, "1", 0, 0xFF10);
// ptable level 0
_tableIndex.Add(0, "6", 1, 0xFF01);
_tableIndex.Add(0, "1", 1, 0xFF11);
// memtable
_tableIndex.Add(0, "4", 0, 0xFF01);
await Task.Delay(500);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV2, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV2,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 5,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-1", 0, 1);
_tableIndex.Add(1, "testStream-2", 0, 2);
_tableIndex.Add(1, "testStream-1", 1, 3);
_tableIndex.Add(1, "testStream-2", 1, 4);
_tableIndex.Add(1, "testStream-1", 2, 5);
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 5),
() => fakeReader,
_ptableVersion,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 5,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-2", 2, 6);
_tableIndex.Add(1, "testStream-1", 3, 7);
_tableIndex.Add(1, "testStream-2", 3, 8);
_tableIndex.Add(1, "testStream-1", 4, 9);
_tableIndex.Add(1, "testStream-2", 4, 10);
await Task.Delay(500);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV1, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV1,
maxSizeForMemory: 5,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 0, 1);
_tableIndex.Add(1, "account--696193173", 0, 2);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 1, 3);
_tableIndex.Add(1, "account--696193173", 1, 4);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 2, 5);
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 5),
() => fakeReader,
_ptableVersion,
maxSizeForMemory: 5,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "account--696193173", 2, 6);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 3, 7);
_tableIndex.Add(1, "account--696193173", 3, 8);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 4, 9);
_tableIndex.Add(1, "account--696193173", 4, 10);
Thread.Sleep(500);
}
public void Setup()
{
given();
_indexDir = PathName;
_fakeReader = new TFReaderLease(new FakeReader());
_indexBackend = new FakeIndexBackend(_fakeReader);
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV1, maxSize: _maxMemTableSize),
() => _fakeReader,
PTableVersions.IndexV1,
maxSizeForMemory: _maxMemTableSize,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_indexReader = new IndexReader(_indexBackend, _tableIndex, new EventStore.Core.Data.StreamMetadata(), _hashCollisionReadLimit);
when();
//wait for the mem table to be dumped
System.Threading.Thread.Sleep(500);
}
private int GetStreamLastEventNumberUncached(TFReaderLease reader, string streamId)
{
IndexEntry latestEntry;
if (!_tableIndex.TryGetLatestEntry(streamId, out latestEntry))
{
return(ExpectedVersion.NoStream);
}
var rec = ReadPrepareInternal(reader, latestEntry.Position);
if (rec == null)
{
throw new Exception("Could not read latest stream's prepare. That should never happen.");
}
if (rec.EventStreamId == streamId) // LUCKY!!!
{
return(latestEntry.Version);
}
int count = 0;
foreach (var indexEntry in _tableIndex.GetRange(streamId, 0, int.MaxValue, limit: _hashCollisionReadLimit))
{
var r = ReadPrepareInternal(reader, indexEntry.Position);
if (r != null && r.EventStreamId == streamId)
{
return(indexEntry.Version); // AT LAST!!!
}
count++;
Interlocked.Increment(ref _hashCollisions);
if (count > _hashCollisionReadLimit)
{
Log.Error("A hash collision resulted in not finding the last event number for the stream {0}", streamId);
return(ExpectedVersion.NoStream);
}
}
return(ExpectedVersion.NoStream); // no such event stream
}
