public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _indexDir = PathName; var fakeReader = new TFReaderLease(new FakeTfReader()); _tableIndex = new TableIndex(_indexDir, () => new HashListMemTable(maxSize: 10), () => fakeReader, maxSizeForMemory: 5); _tableIndex.Initialize(long.MaxValue); _tableIndex.Add(0, 0xDEAD, 0, 0xFF00); _tableIndex.Add(0, 0xDEAD, 1, 0xFF01); _tableIndex.Add(0, 0xBEEF, 0, 0xFF00); _tableIndex.Add(0, 0xBEEF, 1, 0xFF01); _tableIndex.Add(0, 0xABBA, 0, 0xFF00); // 1st ptable0 _tableIndex.Add(0, 0xABBA, 1, 0xFF01); _tableIndex.Add(0, 0xABBA, 2, 0xFF02); _tableIndex.Add(0, 0xABBA, 3, 0xFF03); _tableIndex.Add(0, 0xADA, 0, 0xFF00); // simulates duplicate due to concurrency in TableIndex (see memtable below) _tableIndex.Add(0, 0xDEAD, 0, 0xFF10); // 2nd ptable0 _tableIndex.Add(0, 0xDEAD, 1, 0xFF11); // in memtable _tableIndex.Add(0, 0xADA, 0, 0xFF00); // in memtable }
private IndexReadEventResult ReadEventInternal(TFReaderLease reader, string streamId, int eventNumber) { var lastEventNumber = GetStreamLastEventNumberCached(reader, streamId); var metadata = GetStreamMetadataCached(reader, streamId); var originalStreamExists = OriginalStreamExists(reader, streamId); if (lastEventNumber == EventNumber.DeletedStream) return new IndexReadEventResult(ReadEventResult.StreamDeleted, metadata, lastEventNumber, originalStreamExists); if (lastEventNumber == ExpectedVersion.NoStream || metadata.TruncateBefore == EventNumber.DeletedStream) return new IndexReadEventResult(ReadEventResult.NoStream, metadata, lastEventNumber, originalStreamExists); if (lastEventNumber == EventNumber.Invalid) return new IndexReadEventResult(ReadEventResult.NoStream, metadata, lastEventNumber, originalStreamExists); if (eventNumber == -1) eventNumber = lastEventNumber; int minEventNumber = 0; if (metadata.MaxCount.HasValue) minEventNumber = Math.Max(minEventNumber, lastEventNumber - metadata.MaxCount.Value + 1); if (metadata.TruncateBefore.HasValue) minEventNumber = Math.Max(minEventNumber, metadata.TruncateBefore.Value); if (eventNumber < minEventNumber || eventNumber > lastEventNumber) return new IndexReadEventResult(ReadEventResult.NotFound, metadata, lastEventNumber, originalStreamExists); PrepareLogRecord prepare = ReadPrepareInternal(reader, streamId, eventNumber); if (prepare != null) { if (metadata.MaxAge.HasValue && prepare.TimeStamp < DateTime.UtcNow - metadata.MaxAge.Value) return new IndexReadEventResult(ReadEventResult.NotFound, metadata, lastEventNumber, originalStreamExists); return new IndexReadEventResult(ReadEventResult.Success, new EventRecord(eventNumber, prepare), metadata, lastEventNumber, originalStreamExists); } return new IndexReadEventResult(ReadEventResult.NotFound, metadata, lastEventNumber, originalStreamExists: originalStreamExists); }
public void Setup(){ given(); _indexDir = PathName; _fakeReader = new TFReaderLease(new FakeReader()); _indexBackend = new FakeIndexBackend(_fakeReader); _lowHasher = new XXHashUnsafe(); _highHasher = new Murmur3AUnsafe(); _tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher, () => new HashListMemTable(PTableVersions.Index32Bit, maxSize: _maxMemTableSize), () => _fakeReader, PTableVersions.Index32Bit, maxSizeForMemory: _maxMemTableSize, maxTablesPerLevel: 2); _tableIndex.Initialize(long.MaxValue); _indexReader = new IndexReader(_indexBackend, _tableIndex, new EventStore.Core.Data.StreamMetadata(), _hashCollisionReadLimit); when(); //wait for the mem table to be dumped System.Threading.Thread.Sleep(500); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _indexDir = PathName; var fakeReader = new TFReaderLease(new FakeIndexReader()); _lowHasher = new FakeIndexHasher(); _highHasher = new FakeIndexHasher(); _tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher, () => new HashListMemTable(_ptableVersion, maxSize: 10), () => fakeReader, _ptableVersion, maxSizeForMemory: 2, maxTablesPerLevel: 2); _tableIndex.Initialize(long.MaxValue); // ptable level 2 _tableIndex.Add(0, "1", 0, 0xFF00); _tableIndex.Add(0, "1", 1, 0xFF01); _tableIndex.Add(0, "2", 0, 0xFF00); _tableIndex.Add(0, "2", 1, 0xFF01); _tableIndex.Add(0, "3", 0, 0xFF00); _tableIndex.Add(0, "3", 1, 0xFF01); _tableIndex.Add(0, "3", 0, 0xFF02); _tableIndex.Add(0, "3", 1, 0xFF03); // ptable level 1 _tableIndex.Add(0, "4", 0, 0xFF00); _tableIndex.Add(0, "5", 10, 0xFFF1); _tableIndex.Add(0, "6", 0, 0xFF00); _tableIndex.Add(0, "1", 0, 0xFF10); // ptable level 0 _tableIndex.Add(0, "6", 1, 0xFF01); _tableIndex.Add(0, "1", 1, 0xFF11); // memtable _tableIndex.Add(0, "4", 0, 0xFF01); Thread.Sleep(500); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _indexDir = PathName; var fakeReader = new TFReaderLease(new FakeIndexReader()); _lowHasher = new XXHashUnsafe(); _highHasher = new Murmur3AUnsafe(); _tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher, () => new HashListMemTable(PTableVersions.Index32Bit, maxSize: 5), () => fakeReader, PTableVersions.Index32Bit, maxSizeForMemory: 5, maxTablesPerLevel: 2); _tableIndex.Initialize(long.MaxValue); _tableIndex.Add(1, "LPN-FC002_LPK51001", 0, 1); _tableIndex.Add(1, "account--696193173", 0, 2); _tableIndex.Add(1, "LPN-FC002_LPK51001", 1, 3); _tableIndex.Add(1, "account--696193173", 1, 4); _tableIndex.Add(1, "LPN-FC002_LPK51001", 2, 5); _tableIndex.Close(false); _tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher, () => new HashListMemTable(PTableVersions.Index64Bit, maxSize: 5), () => fakeReader, PTableVersions.Index64Bit, maxSizeForMemory: 5, maxTablesPerLevel: 2); _tableIndex.Initialize(long.MaxValue); _tableIndex.Add(1, "account--696193173", 2, 6); _tableIndex.Add(1, "LPN-FC002_LPK51001", 3, 7); _tableIndex.Add(1, "account--696193173", 3, 8); _tableIndex.Add(1, "LPN-FC002_LPK51001", 4, 9); _tableIndex.Add(1, "account--696193173", 4, 10); Thread.Sleep(500); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _indexDir = PathName; var fakeReader = new TFReaderLease(new FakeTfReader()); _tableIndex = new TableIndex(_indexDir, () => new HashListMemTable(maxSize: 10), () => fakeReader, maxSizeForMemory: 2, maxTablesPerLevel: 2); _tableIndex.Initialize(long.MaxValue); // ptable level 2 _tableIndex.Add(0, 0xDEAD, 0, 0xFF00); _tableIndex.Add(0, 0xDEAD, 1, 0xFF01); _tableIndex.Add(0, 0xBEEF, 0, 0xFF00); _tableIndex.Add(0, 0xBEEF, 1, 0xFF01); _tableIndex.Add(0, 0xABBA, 0, 0xFF00); _tableIndex.Add(0, 0xABBA, 1, 0xFF01); _tableIndex.Add(0, 0xABBA, 0, 0xFF02); _tableIndex.Add(0, 0xABBA, 1, 0xFF03); // ptable level 1 _tableIndex.Add(0, 0xADA, 0, 0xFF00); _tableIndex.Add(0, 0xCEED, 10, 0xFFF1); _tableIndex.Add(0, 0xBABA, 0, 0xFF00); _tableIndex.Add(0, 0xDEAD, 0, 0xFF10); // ptable level 0 _tableIndex.Add(0, 0xBABA, 1, 0xFF01); _tableIndex.Add(0, 0xDEAD, 1, 0xFF11); // memtable _tableIndex.Add(0, 0xADA, 0, 0xFF01); Thread.Sleep(500); }
private StreamMetadata GetStreamMetadataUncached(TFReaderLease reader, string streamId) { var metastreamId = SystemStreams.MetastreamOf(streamId); var metaEventNumber = GetStreamLastEventNumberCached(reader, metastreamId); if (metaEventNumber == ExpectedVersion.NoStream || metaEventNumber == EventNumber.DeletedStream) return StreamMetadata.Empty; PrepareLogRecord prepare = ReadPrepareInternal(reader, metastreamId, metaEventNumber); if (prepare == null) throw new Exception(string.Format("ReadPrepareInternal could not find metaevent #{0} on metastream '{1}'. " + "That should never happen.", metaEventNumber, metastreamId)); if (prepare.Data.Length == 0 || prepare.Flags.HasNoneOf(PrepareFlags.IsJson)) return StreamMetadata.Empty; try { return StreamMetadata.FromJsonBytes(prepare.Data); } catch (Exception) { return StreamMetadata.Empty; } }
private StreamMetadata GetStreamMetadataCached(TFReaderLease reader, string streamId) { // if this is metastream -- check if original stream was deleted, if yes -- metastream is deleted as well if (SystemStreams.IsMetastream(streamId)) return _metastreamMetadata; var cache = _backend.TryGetStreamMetadata(streamId); if (cache.Metadata != null) { Interlocked.Increment(ref _cachedStreamInfo); return cache.Metadata; } Interlocked.Increment(ref _notCachedStreamInfo); var streamMetadata = GetStreamMetadataUncached(reader, streamId); // Conditional update depending on previously returned cache info version. // If version is not correct -- nothing is changed in cache. // This update is conditioned to not interfere with updating stream cache info by commit procedure // (which is the source of truth). var res = _backend.UpdateStreamMetadata(cache.Version, streamId, streamMetadata); return res ?? streamMetadata; }
private bool OriginalStreamExists(TFReaderLease reader, string metaStreamId) { if (SystemStreams.IsSystemStream(metaStreamId)) { var originalStreamId = SystemStreams.OriginalStreamOf(metaStreamId); var lastEventNumber = GetStreamLastEventNumberCached(reader, originalStreamId); if (lastEventNumber == ExpectedVersion.NoStream || lastEventNumber == EventNumber.DeletedStream) return false; return true; } return false; }
private int GetStreamLastEventNumberUncached(TFReaderLease reader, string streamId) { IndexEntry latestEntry; if (!_tableIndex.TryGetLatestEntry(streamId, out latestEntry)) return ExpectedVersion.NoStream; var rec = ReadPrepareInternal(reader, latestEntry.Position); if (rec == null) throw new Exception("Could not read latest stream's prepare. That should never happen."); int count = 0; int startVersion = 0; int latestVersion = int.MinValue; if(rec.EventStreamId == streamId){ startVersion = Math.Max(latestEntry.Version, latestEntry.Version + 1); latestVersion = latestEntry.Version; } foreach (var indexEntry in _tableIndex.GetRange(streamId, startVersion, int.MaxValue, limit: _hashCollisionReadLimit + 1)) { var r = ReadPrepareInternal(reader, indexEntry.Position); if (r != null && r.EventStreamId == streamId){ if(latestVersion == int.MinValue){ latestVersion = indexEntry.Version; continue; } return latestVersion < indexEntry.Version ? indexEntry.Version : latestVersion; } count++; Interlocked.Increment(ref _hashCollisions); if(count > _hashCollisionReadLimit) { Log.Error("A hash collision resulted in not finding the last event number for the stream {0}.", streamId); return EventNumber.Invalid; } } return latestVersion == int.MinValue ? ExpectedVersion.NoStream : latestVersion; }
private int GetStreamLastEventNumberCached(TFReaderLease reader, string streamId) { // if this is metastream -- check if original stream was deleted, if yes -- metastream is deleted as well if (SystemStreams.IsMetastream(streamId) && GetStreamLastEventNumberCached(reader, SystemStreams.OriginalStreamOf(streamId)) == EventNumber.DeletedStream) return EventNumber.DeletedStream; var cache = _backend.TryGetStreamLastEventNumber(streamId); if (cache.LastEventNumber != null) { Interlocked.Increment(ref _cachedStreamInfo); return cache.LastEventNumber.GetValueOrDefault(); } Interlocked.Increment(ref _notCachedStreamInfo); var lastEventNumber = GetStreamLastEventNumberUncached(reader, streamId); // Conditional update depending on previously returned cache info version. // If version is not correct -- nothing is changed in cache. // This update is conditioned to not interfere with updating stream cache info by commit procedure // (which is the source of truth). var res = _backend.UpdateStreamLastEventNumber(cache.Version, streamId, lastEventNumber); return res ?? lastEventNumber; }
private StreamAccess CheckStreamAccessInternal(TFReaderLease reader, string streamId, StreamAccessType streamAccessType, IPrincipal user) { if (SystemStreams.IsMetastream(streamId)) { switch (streamAccessType) { case StreamAccessType.Read: return CheckStreamAccessInternal(reader, SystemStreams.OriginalStreamOf(streamId), StreamAccessType.MetaRead, user); case StreamAccessType.Write: return CheckStreamAccessInternal(reader, SystemStreams.OriginalStreamOf(streamId), StreamAccessType.MetaWrite, user); case StreamAccessType.Delete: case StreamAccessType.MetaRead: case StreamAccessType.MetaWrite: return new StreamAccess(false); default: throw new ArgumentOutOfRangeException("streamAccessType"); } } if ((streamAccessType == StreamAccessType.Write || streamAccessType == StreamAccessType.Delete) && streamId == SystemStreams.AllStream) return new StreamAccess(false); var sysSettings = _backend.GetSystemSettings() ?? SystemSettings.Default; var meta = GetStreamMetadataCached(reader, streamId); StreamAcl acl; StreamAcl sysAcl; StreamAcl defAcl; if (SystemStreams.IsSystemStream(streamId)) { defAcl = SystemSettings.Default.SystemStreamAcl; sysAcl = sysSettings.SystemStreamAcl ?? defAcl; acl = meta.Acl ?? sysAcl; } else { defAcl = SystemSettings.Default.UserStreamAcl; sysAcl = sysSettings.UserStreamAcl ?? defAcl; acl = meta.Acl ?? sysAcl; } string[] roles; switch (streamAccessType) { case StreamAccessType.Read: roles = acl.ReadRoles ?? sysAcl.ReadRoles ?? defAcl.ReadRoles; break; case StreamAccessType.Write: roles = acl.WriteRoles ?? sysAcl.WriteRoles ?? defAcl.WriteRoles; break; case StreamAccessType.Delete: roles = acl.DeleteRoles ?? sysAcl.DeleteRoles ?? defAcl.DeleteRoles; break; case StreamAccessType.MetaRead: roles = acl.MetaReadRoles ?? sysAcl.MetaReadRoles ?? defAcl.MetaReadRoles; break; case StreamAccessType.MetaWrite: roles = acl.MetaWriteRoles ?? sysAcl.MetaWriteRoles ?? defAcl.MetaWriteRoles; break; default: throw new ArgumentOutOfRangeException("streamAccessType"); } var isPublic = roles.Contains(x => x == SystemRoles.All); if (isPublic) return new StreamAccess(true, true); if (user == null) return new StreamAccess(false); if (user.IsInRole(SystemRoles.Admins)) return new StreamAccess(true); for (int i = 0; i < roles.Length; ++i) { if (user.IsInRole(roles[i])) return new StreamAccess(true); } return new StreamAccess(false); }
private static PrepareLogRecord ReadPrepareInternal(TFReaderLease reader, long logPosition) { RecordReadResult result = reader.TryReadAt(logPosition); if (!result.Success) return null; if (result.LogRecord.RecordType != LogRecordType.Prepare) throw new Exception(string.Format("Incorrect type of log record {0}, expected Prepare record.", result.LogRecord.RecordType)); return (PrepareLogRecord)result.LogRecord; }
private PrepareLogRecord ReadPrepareInternal(TFReaderLease reader, string streamId, int eventNumber) { // we assume that you already did check for stream deletion Ensure.NotNullOrEmpty(streamId, "streamId"); Ensure.Nonnegative(eventNumber, "eventNumber"); var recordsQuery = _tableIndex.GetRange(streamId, eventNumber, eventNumber) .Select(x => new { x.Version, Prepare = ReadPrepareInternal(reader, x.Position) }) .Where(x => x.Prepare != null && x.Prepare.EventStreamId == streamId) .GroupBy(x => x.Version).Select(x => x.Last()); if(recordsQuery.Count() == 1){ return recordsQuery.First().Prepare; } return null; }
public FakeIndexBackend(TFReaderLease readerLease){ _readerLease = readerLease; }
private static Tuple<string, bool> ReadEntry(TFReaderLease reader, long position) { RecordReadResult result = reader.TryReadAt(position); if (!result.Success) return new Tuple<string, bool>(String.Empty, false); if (result.LogRecord.RecordType != TransactionLog.LogRecords.LogRecordType.Prepare) throw new Exception(string.Format("Incorrect type of log record {0}, expected Prepare record.", result.LogRecord.RecordType)); return new Tuple<string, bool>(((TransactionLog.LogRecords.PrepareLogRecord)result.LogRecord).EventStreamId, true); }
private int GetStreamLastEventNumberUncached(TFReaderLease reader, string streamId) { IndexEntry latestEntry; if (!_tableIndex.TryGetLatestEntry(streamId, out latestEntry)) return ExpectedVersion.NoStream; var rec = ReadPrepareInternal(reader, latestEntry.Position); if (rec == null) throw new Exception("Could not read latest stream's prepare. That should never happen."); if (rec.EventStreamId == streamId) // LUCKY!!! return latestEntry.Version; int count = 0; foreach (var indexEntry in _tableIndex.GetRange(streamId, 0, int.MaxValue, limit: _hashCollisionReadLimit)) { var r = ReadPrepareInternal(reader, indexEntry.Position); if (r != null && r.EventStreamId == streamId) return indexEntry.Version; // AT LAST!!! count++; Interlocked.Increment(ref _hashCollisions); if(count > _hashCollisionReadLimit) { Log.Error("A hash collision resulted in not finding the last event number for the stream {0}", streamId); return ExpectedVersion.NoStream; } } return ExpectedVersion.NoStream; // no such event stream }
private PrepareLogRecord ReadPrepareInternal(TFReaderLease reader, string streamId, int eventNumber) { // we assume that you already did check for stream deletion Ensure.NotNullOrEmpty(streamId, "streamId"); Ensure.Nonnegative(eventNumber, "eventNumber"); long position; if (_tableIndex.TryGetOneValue(streamId, eventNumber, out position)) { var rec = ReadPrepareInternal(reader, position); if (rec != null && rec.EventStreamId == streamId) return rec; foreach (var indexEntry in _tableIndex.GetRange(streamId, eventNumber, eventNumber)) { Interlocked.Increment(ref _hashCollisions); if (indexEntry.Position == position) // already checked that continue; rec = ReadPrepareInternal(reader, indexEntry.Position); if (rec != null && rec.EventStreamId == streamId) return rec; } } return null; }
private int GetStreamLastEventNumberUncached(TFReaderLease reader, string streamId) { var streamHash = _hasher.Hash(streamId); IndexEntry latestEntry; if (!_tableIndex.TryGetLatestEntry(streamHash, out latestEntry)) return ExpectedVersion.NoStream; var rec = ReadPrepareInternal(reader, latestEntry.Position); if (rec == null) throw new Exception("Couldn't read latest stream's prepare! That shouldn't happen EVER!"); if (rec.EventStreamId == streamId) // LUCKY!!! return latestEntry.Version; // TODO AN here lies the problem of out of memory if the stream has A LOT of events in them foreach (var indexEntry in _tableIndex.GetRange(streamHash, 0, int.MaxValue)) { var r = ReadPrepareInternal(reader, indexEntry.Position); if (r != null && r.EventStreamId == streamId) return indexEntry.Version; // AT LAST!!! Interlocked.Increment(ref _hashCollisions); } return ExpectedVersion.NoStream; // no such event stream }