public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeTfReader());
_tableIndex = new TableIndex(_indexDir,
() => new HashListMemTable(maxSize: 10),
() => fakeReader,
maxSizeForMemory: 5);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(0, 0xDEAD, 0, 0xFF00);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF01);
_tableIndex.Add(0, 0xBEEF, 0, 0xFF00);
_tableIndex.Add(0, 0xBEEF, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 0, 0xFF00); // 1st ptable0
_tableIndex.Add(0, 0xABBA, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 2, 0xFF02);
_tableIndex.Add(0, 0xABBA, 3, 0xFF03);
_tableIndex.Add(0, 0xADA, 0, 0xFF00); // simulates duplicate due to concurrency in TableIndex (see memtable below)
_tableIndex.Add(0, 0xDEAD, 0, 0xFF10); // 2nd ptable0
_tableIndex.Add(0, 0xDEAD, 1, 0xFF11); // in memtable
_tableIndex.Add(0, 0xADA, 0, 0xFF00); // in memtable
}
private IndexReadEventResult ReadEventInternal(TFReaderLease reader, string streamId, int eventNumber)
{
var lastEventNumber = GetStreamLastEventNumberCached(reader, streamId);
var metadata = GetStreamMetadataCached(reader, streamId);
var originalStreamExists = OriginalStreamExists(reader, streamId);
if (lastEventNumber == EventNumber.DeletedStream)
return new IndexReadEventResult(ReadEventResult.StreamDeleted, metadata, lastEventNumber, originalStreamExists);
if (lastEventNumber == ExpectedVersion.NoStream || metadata.TruncateBefore == EventNumber.DeletedStream)
return new IndexReadEventResult(ReadEventResult.NoStream, metadata, lastEventNumber, originalStreamExists);
if (lastEventNumber == EventNumber.Invalid)
return new IndexReadEventResult(ReadEventResult.NoStream, metadata, lastEventNumber, originalStreamExists);
if (eventNumber == -1)
eventNumber = lastEventNumber;
int minEventNumber = 0;
if (metadata.MaxCount.HasValue)
minEventNumber = Math.Max(minEventNumber, lastEventNumber - metadata.MaxCount.Value + 1);
if (metadata.TruncateBefore.HasValue)
minEventNumber = Math.Max(minEventNumber, metadata.TruncateBefore.Value);
if (eventNumber < minEventNumber || eventNumber > lastEventNumber)
return new IndexReadEventResult(ReadEventResult.NotFound, metadata, lastEventNumber, originalStreamExists);
PrepareLogRecord prepare = ReadPrepareInternal(reader, streamId, eventNumber);
if (prepare != null)
{
if (metadata.MaxAge.HasValue && prepare.TimeStamp < DateTime.UtcNow - metadata.MaxAge.Value)
return new IndexReadEventResult(ReadEventResult.NotFound, metadata, lastEventNumber, originalStreamExists);
return new IndexReadEventResult(ReadEventResult.Success, new EventRecord(eventNumber, prepare), metadata, lastEventNumber, originalStreamExists);
}
return new IndexReadEventResult(ReadEventResult.NotFound, metadata, lastEventNumber, originalStreamExists: originalStreamExists);
}
public void Setup(){
given();
_indexDir = PathName;
_fakeReader = new TFReaderLease(new FakeReader());
_indexBackend = new FakeIndexBackend(_fakeReader);
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.Index32Bit, maxSize: _maxMemTableSize),
() => _fakeReader,
PTableVersions.Index32Bit,
maxSizeForMemory: _maxMemTableSize,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_indexReader = new IndexReader(_indexBackend, _tableIndex, new EventStore.Core.Data.StreamMetadata(), _hashCollisionReadLimit);
when();
//wait for the mem table to be dumped
System.Threading.Thread.Sleep(500);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
_lowHasher = new FakeIndexHasher();
_highHasher = new FakeIndexHasher();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 10),
() => fakeReader,
_ptableVersion,
maxSizeForMemory: 2,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
// ptable level 2
_tableIndex.Add(0, "1", 0, 0xFF00);
_tableIndex.Add(0, "1", 1, 0xFF01);
_tableIndex.Add(0, "2", 0, 0xFF00);
_tableIndex.Add(0, "2", 1, 0xFF01);
_tableIndex.Add(0, "3", 0, 0xFF00);
_tableIndex.Add(0, "3", 1, 0xFF01);
_tableIndex.Add(0, "3", 0, 0xFF02);
_tableIndex.Add(0, "3", 1, 0xFF03);
// ptable level 1
_tableIndex.Add(0, "4", 0, 0xFF00);
_tableIndex.Add(0, "5", 10, 0xFFF1);
_tableIndex.Add(0, "6", 0, 0xFF00);
_tableIndex.Add(0, "1", 0, 0xFF10);
// ptable level 0
_tableIndex.Add(0, "6", 1, 0xFF01);
_tableIndex.Add(0, "1", 1, 0xFF11);
// memtable
_tableIndex.Add(0, "4", 0, 0xFF01);
Thread.Sleep(500);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.Index32Bit, maxSize: 5),
() => fakeReader,
PTableVersions.Index32Bit,
maxSizeForMemory: 5,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 0, 1);
_tableIndex.Add(1, "account--696193173", 0, 2);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 1, 3);
_tableIndex.Add(1, "account--696193173", 1, 4);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 2, 5);
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.Index64Bit, maxSize: 5),
() => fakeReader,
PTableVersions.Index64Bit,
maxSizeForMemory: 5,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "account--696193173", 2, 6);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 3, 7);
_tableIndex.Add(1, "account--696193173", 3, 8);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 4, 9);
_tableIndex.Add(1, "account--696193173", 4, 10);
Thread.Sleep(500);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeTfReader());
_tableIndex = new TableIndex(_indexDir,
() => new HashListMemTable(maxSize: 10),
() => fakeReader,
maxSizeForMemory: 2,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
// ptable level 2
_tableIndex.Add(0, 0xDEAD, 0, 0xFF00);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF01);
_tableIndex.Add(0, 0xBEEF, 0, 0xFF00);
_tableIndex.Add(0, 0xBEEF, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 0, 0xFF00);
_tableIndex.Add(0, 0xABBA, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 0, 0xFF02);
_tableIndex.Add(0, 0xABBA, 1, 0xFF03);
// ptable level 1
_tableIndex.Add(0, 0xADA, 0, 0xFF00);
_tableIndex.Add(0, 0xCEED, 10, 0xFFF1);
_tableIndex.Add(0, 0xBABA, 0, 0xFF00);
_tableIndex.Add(0, 0xDEAD, 0, 0xFF10);
// ptable level 0
_tableIndex.Add(0, 0xBABA, 1, 0xFF01);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF11);
// memtable
_tableIndex.Add(0, 0xADA, 0, 0xFF01);
Thread.Sleep(500);
}
private StreamMetadata GetStreamMetadataUncached(TFReaderLease reader, string streamId)
{
var metastreamId = SystemStreams.MetastreamOf(streamId);
var metaEventNumber = GetStreamLastEventNumberCached(reader, metastreamId);
if (metaEventNumber == ExpectedVersion.NoStream || metaEventNumber == EventNumber.DeletedStream)
return StreamMetadata.Empty;
PrepareLogRecord prepare = ReadPrepareInternal(reader, metastreamId, metaEventNumber);
if (prepare == null)
throw new Exception(string.Format("ReadPrepareInternal could not find metaevent #{0} on metastream '{1}'. "
+ "That should never happen.", metaEventNumber, metastreamId));
if (prepare.Data.Length == 0 || prepare.Flags.HasNoneOf(PrepareFlags.IsJson))
return StreamMetadata.Empty;
try
{
return StreamMetadata.FromJsonBytes(prepare.Data);
}
catch (Exception)
{
return StreamMetadata.Empty;
}
}
private StreamMetadata GetStreamMetadataCached(TFReaderLease reader, string streamId)
{
// if this is metastream -- check if original stream was deleted, if yes -- metastream is deleted as well
if (SystemStreams.IsMetastream(streamId))
return _metastreamMetadata;
var cache = _backend.TryGetStreamMetadata(streamId);
if (cache.Metadata != null)
{
Interlocked.Increment(ref _cachedStreamInfo);
return cache.Metadata;
}
Interlocked.Increment(ref _notCachedStreamInfo);
var streamMetadata = GetStreamMetadataUncached(reader, streamId);
// Conditional update depending on previously returned cache info version.
// If version is not correct -- nothing is changed in cache.
// This update is conditioned to not interfere with updating stream cache info by commit procedure
// (which is the source of truth).
var res = _backend.UpdateStreamMetadata(cache.Version, streamId, streamMetadata);
return res ?? streamMetadata;
}
private bool OriginalStreamExists(TFReaderLease reader, string metaStreamId)
{
if (SystemStreams.IsSystemStream(metaStreamId))
{
var originalStreamId = SystemStreams.OriginalStreamOf(metaStreamId);
var lastEventNumber = GetStreamLastEventNumberCached(reader, originalStreamId);
if (lastEventNumber == ExpectedVersion.NoStream || lastEventNumber == EventNumber.DeletedStream)
return false;
return true;
}
return false;
}
private int GetStreamLastEventNumberUncached(TFReaderLease reader, string streamId)
{
IndexEntry latestEntry;
if (!_tableIndex.TryGetLatestEntry(streamId, out latestEntry))
return ExpectedVersion.NoStream;
var rec = ReadPrepareInternal(reader, latestEntry.Position);
if (rec == null) throw new Exception("Could not read latest stream's prepare. That should never happen.");
int count = 0;
int startVersion = 0;
int latestVersion = int.MinValue;
if(rec.EventStreamId == streamId){
startVersion = Math.Max(latestEntry.Version, latestEntry.Version + 1);
latestVersion = latestEntry.Version;
}
foreach (var indexEntry in _tableIndex.GetRange(streamId, startVersion, int.MaxValue, limit: _hashCollisionReadLimit + 1))
{
var r = ReadPrepareInternal(reader, indexEntry.Position);
if (r != null && r.EventStreamId == streamId){
if(latestVersion == int.MinValue){
latestVersion = indexEntry.Version;
continue;
}
return latestVersion < indexEntry.Version ? indexEntry.Version : latestVersion;
}
count++;
Interlocked.Increment(ref _hashCollisions);
if(count > _hashCollisionReadLimit)
{
Log.Error("A hash collision resulted in not finding the last event number for the stream {0}.", streamId);
return EventNumber.Invalid;
}
}
return latestVersion == int.MinValue ? ExpectedVersion.NoStream : latestVersion;
}
private int GetStreamLastEventNumberCached(TFReaderLease reader, string streamId)
{
// if this is metastream -- check if original stream was deleted, if yes -- metastream is deleted as well
if (SystemStreams.IsMetastream(streamId)
&& GetStreamLastEventNumberCached(reader, SystemStreams.OriginalStreamOf(streamId)) == EventNumber.DeletedStream)
return EventNumber.DeletedStream;
var cache = _backend.TryGetStreamLastEventNumber(streamId);
if (cache.LastEventNumber != null)
{
Interlocked.Increment(ref _cachedStreamInfo);
return cache.LastEventNumber.GetValueOrDefault();
}
Interlocked.Increment(ref _notCachedStreamInfo);
var lastEventNumber = GetStreamLastEventNumberUncached(reader, streamId);
// Conditional update depending on previously returned cache info version.
// If version is not correct -- nothing is changed in cache.
// This update is conditioned to not interfere with updating stream cache info by commit procedure
// (which is the source of truth).
var res = _backend.UpdateStreamLastEventNumber(cache.Version, streamId, lastEventNumber);
return res ?? lastEventNumber;
}
private StreamAccess CheckStreamAccessInternal(TFReaderLease reader, string streamId,
StreamAccessType streamAccessType, IPrincipal user)
{
if (SystemStreams.IsMetastream(streamId))
{
switch (streamAccessType)
{
case StreamAccessType.Read:
return CheckStreamAccessInternal(reader, SystemStreams.OriginalStreamOf(streamId), StreamAccessType.MetaRead, user);
case StreamAccessType.Write:
return CheckStreamAccessInternal(reader, SystemStreams.OriginalStreamOf(streamId), StreamAccessType.MetaWrite, user);
case StreamAccessType.Delete:
case StreamAccessType.MetaRead:
case StreamAccessType.MetaWrite:
return new StreamAccess(false);
default:
throw new ArgumentOutOfRangeException("streamAccessType");
}
}
if ((streamAccessType == StreamAccessType.Write || streamAccessType == StreamAccessType.Delete)
&& streamId == SystemStreams.AllStream)
return new StreamAccess(false);
var sysSettings = _backend.GetSystemSettings() ?? SystemSettings.Default;
var meta = GetStreamMetadataCached(reader, streamId);
StreamAcl acl;
StreamAcl sysAcl;
StreamAcl defAcl;
if (SystemStreams.IsSystemStream(streamId))
{
defAcl = SystemSettings.Default.SystemStreamAcl;
sysAcl = sysSettings.SystemStreamAcl ?? defAcl;
acl = meta.Acl ?? sysAcl;
}
else
{
defAcl = SystemSettings.Default.UserStreamAcl;
sysAcl = sysSettings.UserStreamAcl ?? defAcl;
acl = meta.Acl ?? sysAcl;
}
string[] roles;
switch (streamAccessType)
{
case StreamAccessType.Read: roles = acl.ReadRoles ?? sysAcl.ReadRoles ?? defAcl.ReadRoles; break;
case StreamAccessType.Write: roles = acl.WriteRoles ?? sysAcl.WriteRoles ?? defAcl.WriteRoles; break;
case StreamAccessType.Delete: roles = acl.DeleteRoles ?? sysAcl.DeleteRoles ?? defAcl.DeleteRoles; break;
case StreamAccessType.MetaRead: roles = acl.MetaReadRoles ?? sysAcl.MetaReadRoles ?? defAcl.MetaReadRoles; break;
case StreamAccessType.MetaWrite: roles = acl.MetaWriteRoles ?? sysAcl.MetaWriteRoles ?? defAcl.MetaWriteRoles; break;
default: throw new ArgumentOutOfRangeException("streamAccessType");
}
var isPublic = roles.Contains(x => x == SystemRoles.All);
if (isPublic) return new StreamAccess(true, true);
if (user == null) return new StreamAccess(false);
if (user.IsInRole(SystemRoles.Admins)) return new StreamAccess(true);
for (int i = 0; i < roles.Length; ++i)
{
if (user.IsInRole(roles[i]))
return new StreamAccess(true);
}
return new StreamAccess(false);
}
private static PrepareLogRecord ReadPrepareInternal(TFReaderLease reader, long logPosition)
{
RecordReadResult result = reader.TryReadAt(logPosition);
if (!result.Success)
return null;
if (result.LogRecord.RecordType != LogRecordType.Prepare)
throw new Exception(string.Format("Incorrect type of log record {0}, expected Prepare record.",
result.LogRecord.RecordType));
return (PrepareLogRecord)result.LogRecord;
}
private PrepareLogRecord ReadPrepareInternal(TFReaderLease reader, string streamId, int eventNumber)
{
// we assume that you already did check for stream deletion
Ensure.NotNullOrEmpty(streamId, "streamId");
Ensure.Nonnegative(eventNumber, "eventNumber");
var recordsQuery = _tableIndex.GetRange(streamId, eventNumber, eventNumber)
.Select(x => new { x.Version, Prepare = ReadPrepareInternal(reader, x.Position) })
.Where(x => x.Prepare != null && x.Prepare.EventStreamId == streamId)
.GroupBy(x => x.Version).Select(x => x.Last());
if(recordsQuery.Count() == 1){
return recordsQuery.First().Prepare;
}
return null;
}
public FakeIndexBackend(TFReaderLease readerLease){
_readerLease = readerLease;
}
private static Tuple<string, bool> ReadEntry(TFReaderLease reader, long position)
{
RecordReadResult result = reader.TryReadAt(position);
if (!result.Success)
return new Tuple<string, bool>(String.Empty, false);
if (result.LogRecord.RecordType != TransactionLog.LogRecords.LogRecordType.Prepare)
throw new Exception(string.Format("Incorrect type of log record {0}, expected Prepare record.",
result.LogRecord.RecordType));
return new Tuple<string, bool>(((TransactionLog.LogRecords.PrepareLogRecord)result.LogRecord).EventStreamId, true);
}
private int GetStreamLastEventNumberUncached(TFReaderLease reader, string streamId)
{
IndexEntry latestEntry;
if (!_tableIndex.TryGetLatestEntry(streamId, out latestEntry))
return ExpectedVersion.NoStream;
var rec = ReadPrepareInternal(reader, latestEntry.Position);
if (rec == null) throw new Exception("Could not read latest stream's prepare. That should never happen.");
if (rec.EventStreamId == streamId) // LUCKY!!!
return latestEntry.Version;
int count = 0;
foreach (var indexEntry in _tableIndex.GetRange(streamId, 0, int.MaxValue, limit: _hashCollisionReadLimit))
{
var r = ReadPrepareInternal(reader, indexEntry.Position);
if (r != null && r.EventStreamId == streamId)
return indexEntry.Version; // AT LAST!!!
count++;
Interlocked.Increment(ref _hashCollisions);
if(count > _hashCollisionReadLimit)
{
Log.Error("A hash collision resulted in not finding the last event number for the stream {0}", streamId);
return ExpectedVersion.NoStream;
}
}
return ExpectedVersion.NoStream; // no such event stream
}
private PrepareLogRecord ReadPrepareInternal(TFReaderLease reader, string streamId, int eventNumber)
{
// we assume that you already did check for stream deletion
Ensure.NotNullOrEmpty(streamId, "streamId");
Ensure.Nonnegative(eventNumber, "eventNumber");
long position;
if (_tableIndex.TryGetOneValue(streamId, eventNumber, out position))
{
var rec = ReadPrepareInternal(reader, position);
if (rec != null && rec.EventStreamId == streamId)
return rec;
foreach (var indexEntry in _tableIndex.GetRange(streamId, eventNumber, eventNumber))
{
Interlocked.Increment(ref _hashCollisions);
if (indexEntry.Position == position) // already checked that
continue;
rec = ReadPrepareInternal(reader, indexEntry.Position);
if (rec != null && rec.EventStreamId == streamId)
return rec;
}
}
return null;
}
private int GetStreamLastEventNumberUncached(TFReaderLease reader, string streamId)
{
var streamHash = _hasher.Hash(streamId);
IndexEntry latestEntry;
if (!_tableIndex.TryGetLatestEntry(streamHash, out latestEntry))
return ExpectedVersion.NoStream;
var rec = ReadPrepareInternal(reader, latestEntry.Position);
if (rec == null) throw new Exception("Couldn't read latest stream's prepare! That shouldn't happen EVER!");
if (rec.EventStreamId == streamId) // LUCKY!!!
return latestEntry.Version;
// TODO AN here lies the problem of out of memory if the stream has A LOT of events in them
foreach (var indexEntry in _tableIndex.GetRange(streamHash, 0, int.MaxValue))
{
var r = ReadPrepareInternal(reader, indexEntry.Position);
if (r != null && r.EventStreamId == streamId)
return indexEntry.Version; // AT LAST!!!
Interlocked.Increment(ref _hashCollisions);
}
return ExpectedVersion.NoStream; // no such event stream
}
