public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 16 * 1024, 0, new InMemoryCheckpoint(), new ICheckpoint[0])); _db.OpenVerifyAndClean(); var chunk = _db.Manager.GetChunk(0); _rec1 = LogRecord.SingleWrite(0, Guid.NewGuid(), Guid.NewGuid(), "es1", ExpectedVersion.Any, "et1", new byte[] { 0, 1, 2 }, new byte[] { 5, 7 }); _res1 = chunk.TryAppend(_rec1); _rec2 = LogRecord.SingleWrite(_res1.NewPosition, Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1", new byte[] { 0, 1, 2 }, new byte[] { 5, 7 }); _res2 = chunk.TryAppend(_rec2); _rec3 = LogRecord.SingleWrite(_res2.NewPosition, Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1", new byte[] { 0, 1, 2 }, new byte[] { 5, 7 }); _res3 = chunk.TryAppend(_rec3); chunk.Complete(); var scavenger = new TFChunkScavenger(_db, new FakeReadIndex(x => x == "es-to-scavenge")); scavenger.Scavenge(alwaysKeepScavenged: true); _scavengedChunk = _db.Manager.GetChunk(0); }
public RecordReadResult(bool success, long nextPosition, LogRecord logRecord, int recordLength) { Success = success; LogRecord = logRecord; NextPosition = nextPosition; RecordLength = recordLength; }
public bool Write(LogRecord record, out long newPos) { var result = _currentChunk.TryAppend(record); if (result.Success) _writerCheckpoint.Write(result.NewPosition + _currentChunk.ChunkHeader.ChunkStartPosition); else CompleteChunk(); // complete updates checkpoint internally newPos = _writerCheckpoint.ReadNonFlushed(); return result.Success; }
public SeqReadResult(bool success, bool eof, LogRecord logRecord, int recordLength, long recordPrePosition, long recordPostPosition) { Success = success; Eof = eof; LogRecord = logRecord; RecordLength = recordLength; RecordPrePosition = recordPrePosition; RecordPostPosition = recordPostPosition; }
public bool Write(LogRecord record, out long newPos) { var chunkNum = (int)(_writerPos / _db.Config.ChunkSize); var chunkPos = _writerPos % _db.Config.ChunkSize; var result = _writerChunk.TryAppend(record); if (result.Success) { Debug.Assert(result.OldPosition == chunkPos); _writerPos = chunkNum * (long)_db.Config.ChunkSize + result.NewPosition; _writerCheckpoint.Write(_writerPos); } else { CompleteChunk(); // complete updates checkpoint internally } newPos = _writerPos; return result.Success; }
public bool Write(LogRecord record, out long newPos) { var chunkNum = (int)(_writerPos / _db.Config.ChunkSize); var chunkPos = _writerPos % _db.Config.ChunkSize; var result = _writerChunk.TryAppend(record); if (result.Success) { Debug.Assert(result.OldPosition == chunkPos); _writerPos = chunkNum * (long)_db.Config.ChunkSize + result.NewPosition; } else { _writerChunk.Flush(); _writerChunk.Complete(); _writerCheckpoint.Flush(); //flush our checkpoint _writerChunk = _db.Manager.AddNewChunk(); _writerPos = (chunkNum + 1) * (long)_db.Config.ChunkSize; // we just moved to a new chunk at pos 0 } _writerCheckpoint.Write(_writerPos); newPos = _writerPos; return result.Success; }
public RecordWriteResult TryAppend(LogRecord record) { if (_isReadonly) throw new InvalidOperationException("Cannot write to a read-only block."); var workItem = _writerWorkItem; var buffer = workItem.Buffer; var bufferWriter = workItem.BufferWriter; var stream = workItem.Stream; buffer.SetLength(4); buffer.Position = 4; record.WriteTo(bufferWriter); var length = (int) buffer.Length - 4; bufferWriter.Write(length); // length suffix buffer.Position = 0; bufferWriter.Write(length); // length prefix if (stream.Position + length + 8 > ChunkHeader.Size + _chunkHeader.ChunkSize) return RecordWriteResult.Failed(GetLogicalPosition(workItem)); var oldPosition = WriteRawData(buffer); _actualDataSize = GetLogicalPosition(workItem); return RecordWriteResult.Successful(oldPosition, _actualDataSize); }
private bool TryReadForwardInternal(ReaderWorkItem workItem, int actualPosition, out int length, out LogRecord record) { length = -1; record = null; workItem.Stream.Position = GetRealPosition(actualPosition, workItem.IsMemory); if (!VerifyDataLengthForward(workItem, 2*sizeof(int))) return false; length = workItem.Reader.ReadInt32(); if (length <= 0) { throw new ArgumentException( string.Format("Log record at actual pos {0} has non-positive length: {1}. " + "Something is seriously wrong.", actualPosition, length)); } if (length > TFConsts.MaxLogRecordSize) { throw new ArgumentException( string.Format("Log record at actual pos {0} has too large length: {1} bytes, " + "while limit is {2} bytes.", actualPosition, length, TFConsts.MaxLogRecordSize)); } if (!VerifyDataLengthForward(workItem, length + sizeof(int) /*suffix*/)) throw new UnableToReadPastEndOfStreamException(); record = LogRecord.ReadFrom(workItem.Reader); Debug.Assert(workItem.Reader.ReadInt32() == length); // verify suffix length == prefix length return true; }
public RecordWriteResult TryAppend(LogRecord record) { if (_isReadOnly) throw new InvalidOperationException("Cannot write to a read-only block."); var workItem = _writerWorkItem; var buffer = workItem.Buffer; var bufferWriter = workItem.BufferWriter; buffer.SetLength(4); buffer.Position = 4; record.WriteTo(bufferWriter); var length = (int) buffer.Length - 4; bufferWriter.Write(length); // length suffix buffer.Position = 0; bufferWriter.Write(length); // length prefix if (workItem.StreamPosition + length + 2*sizeof(int) > ChunkHeader.Size + _chunkHeader.ChunkSize) return RecordWriteResult.Failed(GetDataPosition(workItem)); var oldPosition = WriteRawData(workItem, buffer); _physicalDataSize = (int)GetDataPosition(workItem); // should fit 32 bits _logicalDataSize = ChunkHeader.GetLocalLogPosition(record.LogPosition + length + 2*sizeof(int)); // for non-scavenged chunk _physicalDataSize should be the same as _logicalDataSize // for scavenged chunk _logicalDataSize should be at least the same as _physicalDataSize if ((!ChunkHeader.IsScavenged && _logicalDataSize != _physicalDataSize) || (ChunkHeader.IsScavenged && _logicalDataSize < _physicalDataSize)) { throw new Exception(string.Format("Data sizes violation. Chunk: {0}, IsScavenged: {1}, LogicalDataSize: {2}, PhysicalDataSize: {3}.", FileName, ChunkHeader.IsScavenged, _logicalDataSize, _physicalDataSize)); } return RecordWriteResult.Successful(oldPosition, _physicalDataSize); }
public DbResult(TFChunkDb db, LogRecord[][] recs, Dictionary<string, StreamInfo> streams) { Ensure.NotNull(db, "db"); Ensure.NotNull(recs, "recs"); Ensure.NotNull(streams, "streams"); Db = db; Recs = recs; Streams = streams; }
protected static bool TryReadBackwardInternal(ReaderWorkItem workItem, int actualPosition, out int length, out LogRecord record) { length = -1; record = null; if (actualPosition < 2 * sizeof(int)) // no space even for length prefix and suffix return false; var realPos = GetRealPosition(actualPosition); workItem.Stream.Position = realPos - sizeof(int); length = workItem.Reader.ReadInt32(); if (length <= 0) { throw new ArgumentException( string.Format("Log record that ends at actual pos {0} has non-positive length: {1}. " + "Something is seriously wrong.", actualPosition, length)); } if (length > TFConsts.MaxLogRecordSize) { throw new ArgumentException( string.Format("Log record that ends at actual pos {0} has too large length: {1} bytes, " + "while limit is {2} bytes.", actualPosition, length, TFConsts.MaxLogRecordSize)); } if (actualPosition < length + 2 * sizeof(int)) // no space for record + length prefix and suffix throw new UnableToReadPastEndOfStreamException( string.Format("There is not enough space to read full record (record length according to length suffix: {0}).", length)); workItem.Stream.Position = realPos - length - sizeof(int); record = LogRecord.ReadFrom(workItem.Reader); #if DEBUG workItem.Stream.Position = realPos - length - 2 * sizeof(int); var prefixLength = workItem.Reader.ReadInt32(); Debug.Assert(prefixLength == length); #endif return true; }
private static bool IsCommitAlike(LogRecord rec) { return rec.RecordType == LogRecordType.Commit || (rec.RecordType == LogRecordType.Prepare && ((PrepareLogRecord)rec).Flags.HasAnyOf(PrepareFlags.IsCommitted)); }
public bool Write(LogRecord record, out long newPos) { _buffer.SetLength(4); _buffer.Position = 4; record.WriteTo(_bufferWriter); var length = (int) _buffer.Length - 4; _bufferWriter.Write(length); // length suffix _buffer.Position = 0; _bufferWriter.Write(length); // length prefix var written = 0; var leftToWrite = length + 8; while (leftToWrite > 0) { var leftInFile = (int)(_segmentSize - _writerLocalCheck); var amountToWrite = leftToWrite > leftInFile ? leftInFile : leftToWrite; _fileStream.Write(_buffer.GetBuffer(), written, amountToWrite); leftToWrite -= amountToWrite; written += amountToWrite; _writerCheck += amountToWrite; _writerLocalCheck += amountToWrite; if (_writerLocalCheck >= _segmentSize) { _writerLocalCheck -= _segmentSize; _currentSegment++; CreateSegment(_currentSegment); } } _writerCheckpoint.Write(_writerCheck); newPos = _writerCheck; return true; }
public RecordWriteResult TryAppend(LogRecord record) { if (_isReadonly) throw new InvalidOperationException("Cannot write to a read-only block."); var workItem = _writerWorkItem; var buffer = workItem.Buffer; var bufferWriter = workItem.BufferWriter; var stream = workItem.Stream; buffer.SetLength(4); buffer.Position = 4; record.WriteTo(bufferWriter); buffer.Position = 0; var toWriteLength = (int) buffer.Length; bufferWriter.Write(toWriteLength - 4); if (!VerifyStreamLength(stream, toWriteLength)) return RecordWriteResult.Failed(GetLogicalPosition(workItem)); var oldPosition = WriteRawData(buffer); _actualDataSize = GetLogicalPosition(workItem); return RecordWriteResult.Successful(oldPosition, _actualDataSize); }
private bool TryReadRecordInternal(ReaderWorkItem workItem, int actualPosition, out int length, out LogRecord record) { length = -1; record = null; workItem.Stream.Position = GetRealPosition(actualPosition, workItem.IsMemory); if (!VerifyStreamLength(workItem.Stream, 4)) return false; length = workItem.Reader.ReadInt32(); CheckLength(workItem, length, actualPosition); record = LogRecord.ReadFrom(workItem.Reader); return true; }
public RecordReadResult(bool success, LogRecord logRecord, long nextPosition) { Success = success; LogRecord = logRecord; NextPosition = nextPosition; }
protected bool TryReadForwardInternal(ReaderWorkItem workItem, int actualPosition, out int length, out LogRecord record) { length = -1; record = null; workItem.Stream.Position = GetRealPosition(actualPosition); if (actualPosition + 2*sizeof(int) > Chunk.ActualDataSize) // no space even for length prefix and suffix return false; length = workItem.Reader.ReadInt32(); if (length <= 0) { throw new ArgumentException( string.Format("Log record at actual pos {0} has non-positive length: {1}. " + "Something is seriously wrong in chunk {2}-{3} ({4}).", actualPosition, length, Chunk.ChunkHeader.ChunkStartNumber, Chunk.ChunkHeader.ChunkEndNumber, Chunk.FileName)); } if (length > TFConsts.MaxLogRecordSize) { throw new ArgumentException( string.Format("Log record at actual pos {0} has too large length: {1} bytes, " + "while limit is {2} bytes. Something is seriously wrong in chunk {3}-{4} ({5}).", actualPosition, length, TFConsts.MaxLogRecordSize, Chunk.ChunkHeader.ChunkStartNumber, Chunk.ChunkHeader.ChunkEndNumber, Chunk.FileName)); } if (actualPosition + length + 2*sizeof(int) > Chunk.ActualDataSize) throw new UnableToReadPastEndOfStreamException( string.Format("There is not enough space to read full record (record length according to length prefix: {0}).", length)); record = LogRecord.ReadFrom(workItem.Reader); #if DEBUG // verify suffix length == prefix length int suffixLength = workItem.Reader.ReadInt32(); Debug.Assert(suffixLength == length, "Prefix/suffix length inconsistency ", "Prefix length({0}) != suffix length ({1})", length, suffixLength); #endif return true; }
protected bool TryReadForwardInternal(ReaderWorkItem workItem, long actualPosition, out int length, out LogRecord record) { length = -1; record = null; workItem.Stream.Position = GetRawPosition(actualPosition); if (actualPosition + 2*sizeof(int) > Chunk.PhysicalDataSize) // no space even for length prefix and suffix return false; length = workItem.Reader.ReadInt32(); if (length <= 0) { throw new InvalidReadException( string.Format("Log record at actual pos {0} has non-positive length: {1}. " + " in chunk.", actualPosition, length, Chunk)); } if (length > TFConsts.MaxLogRecordSize) { throw new InvalidReadException( string.Format("Log record at actual pos {0} has too large length: {1} bytes, " + "while limit is {2} bytes. In chunk {3}.", actualPosition, length, TFConsts.MaxLogRecordSize, Chunk)); } if (actualPosition + length + 2 * sizeof(int) > Chunk.PhysicalDataSize) { throw new UnableToReadPastEndOfStreamException( string.Format("There is not enough space to read full record (length prefix: {0}). " + "Actual pre-position: {1}. Something is seriously wrong in chunk {2}.", length, actualPosition, Chunk)); } record = LogRecord.ReadFrom(workItem.Reader); // verify suffix length == prefix length int suffixLength = workItem.Reader.ReadInt32(); if (suffixLength != length) { throw new Exception( string.Format("Prefix/suffix length inconsistency: prefix length({0}) != suffix length ({1}).\n" + "Actual pre-position: {2}. Something is seriously wrong in chunk {3}.", length, suffixLength, actualPosition, Chunk)); } return true; }
public bool TryReadNext(out LogRecord record) { var res = TryReadNext(); record = res.LogRecord; return res.Success; }
protected bool TryReadBackwardInternal(ReaderWorkItem workItem, long actualPosition, out int length, out LogRecord record) { length = -1; record = null; if (actualPosition < 2 * sizeof(int)) // no space even for length prefix and suffix return false; var realPos = GetRawPosition(actualPosition); workItem.Stream.Position = realPos - sizeof(int); length = workItem.Reader.ReadInt32(); if (length <= 0) { throw new ArgumentException( string.Format("Log record that ends at actual pos {0} has non-positive length: {1}. " + "Something is seriously wrong in chunk {2}.", actualPosition, length, Chunk)); } if (length > TFConsts.MaxLogRecordSize) { throw new ArgumentException( string.Format("Log record that ends at actual pos {0} has too large length: {1} bytes, " + "while limit is {2} bytes. Something is seriously wrong in chunk {3}.", actualPosition, length, TFConsts.MaxLogRecordSize, Chunk)); } if (actualPosition < length + 2 * sizeof(int)) // no space for record + length prefix and suffix { throw new UnableToReadPastEndOfStreamException( string.Format("There is not enough space to read full record (length suffix: {0}). " + "Actual post-position: {1}. Something is seriously wrong in chunk {2}.", length, actualPosition, Chunk)); } workItem.Stream.Position = realPos - length - 2*sizeof(int); // verify suffix length == prefix length int prefixLength = workItem.Reader.ReadInt32(); if (prefixLength != length) { throw new Exception( string.Format("Prefix/suffix length inconsistency: prefix length({0}) != suffix length ({1})" + "Actual post-position: {2}. Something is seriously wrong in chunk {3}.", prefixLength, length, actualPosition, Chunk)); } record = LogRecord.ReadFrom(workItem.Reader); return true; }
public DbResult CreateDb() { var records = new LogRecord[_chunkRecs.Count][]; for (int i = 0; i < records.Length; ++i) { records[i] = new LogRecord[_chunkRecs[i].Length]; } var transactions = new Dictionary<int, TransactionInfo>(); var streams = new Dictionary<string, StreamInfo>(); var streamUncommitedVersion = new Dictionary<string, int>(); for (int i = 0; i < _chunkRecs.Count; ++i) { for (int j = 0; j < _chunkRecs[i].Length; ++j) { var rec = _chunkRecs[i][j]; TransactionInfo transInfo; bool transCreate = transactions.TryGetValue(rec.Transaction, out transInfo); if (!transCreate) { if (rec.Type == Rec.RecType.Commit) throw new Exception("Commit for non-existing transaction."); transactions[rec.Transaction] = transInfo = new TransactionInfo(rec.StreamId, rec.Id, rec.Id); streams[rec.StreamId] = new StreamInfo(-1); streamUncommitedVersion[rec.StreamId] = -1; } else { if (rec.Type == Rec.RecType.TransStart) throw new Exception(string.Format("Not expected record type: {0}.", rec.Type)); } if (transInfo.StreamId != rec.StreamId) { throw new Exception(string.Format("Wrong stream id for transaction. Transaction StreamId: {0}, record StreamId: {1}.", transInfo.StreamId, rec.StreamId)); } if (rec.Type != Rec.RecType.Commit && transInfo.IsDelete) throw new Exception("Transaction with records after delete record."); if (rec.Type == Rec.RecType.Delete) transInfo.IsDelete = true; transInfo.LastPrepareId = rec.Id; } } for (int i = 0; i < _chunkRecs.Count; ++i) { var chunk = i == 0 ? _db.Manager.GetChunk(0) : _db.Manager.AddNewChunk(); _db.Config.WriterCheckpoint.Write(i * (long)_db.Config.ChunkSize); for (int j = 0; j < _chunkRecs[i].Length; ++j) { var rec = _chunkRecs[i][j]; var transInfo = transactions[rec.Transaction]; var logPos = _db.Config.WriterCheckpoint.ReadNonFlushed(); int streamVersion = streamUncommitedVersion[rec.StreamId]; if (streamVersion == -1 && rec.Type != Rec.RecType.TransStart && rec.Type != Rec.RecType.Create) throw new Exception(string.Format("Stream {0} is empty and we are not creating it with stream created.", rec.StreamId)); if (streamVersion == EventNumber.DeletedStream && rec.Type != Rec.RecType.Commit) throw new Exception(string.Format("Stream {0} was deleted, but we need to write some more prepares.", rec.StreamId)); if (transInfo.FirstPrepareId == rec.Id) { transInfo.TransactionPosition = logPos; transInfo.TransactionEventNumber = streamVersion + 1; transInfo.TransactionOffset = 0; } LogRecord record; var expectedVersion = transInfo.FirstPrepareId == rec.Id ? streamVersion : ExpectedVersion.Any; switch (rec.Type) { case Rec.RecType.Prepare: case Rec.RecType.Create: { int transOffset = transInfo.TransactionOffset; transInfo.TransactionOffset += 1; record = LogRecord.Prepare(logPos, Guid.NewGuid(), rec.Id, transInfo.TransactionPosition, transOffset, rec.StreamId, expectedVersion, PrepareFlags.Data | (transInfo.FirstPrepareId == rec.Id ? PrepareFlags.TransactionBegin : PrepareFlags.None) | (transInfo.LastPrepareId == rec.Id ? PrepareFlags.TransactionEnd : PrepareFlags.None), rec.EventType, rec.Id.ToByteArray(), null, rec.TimeStamp); if (rec.Type == Rec.RecType.Create) transInfo.StreamMetadata = rec.Metadata; streamUncommitedVersion[rec.StreamId] += 1; break; } case Rec.RecType.Delete: { int transOffset = transInfo.TransactionOffset; transInfo.TransactionOffset += 1; record = LogRecord.Prepare(logPos, Guid.NewGuid(), rec.Id, transInfo.TransactionPosition, transOffset, rec.StreamId, expectedVersion, PrepareFlags.StreamDelete | (transInfo.FirstPrepareId == rec.Id ? PrepareFlags.TransactionBegin : PrepareFlags.None) | (transInfo.LastPrepareId == rec.Id ? PrepareFlags.TransactionEnd : PrepareFlags.None), rec.EventType, LogRecord.NoData, null, rec.TimeStamp); streamUncommitedVersion[rec.StreamId] = EventNumber.DeletedStream; break; } case Rec.RecType.TransStart: case Rec.RecType.TransEnd: { record = LogRecord.Prepare(logPos, Guid.NewGuid(), rec.Id, transInfo.TransactionPosition, -1, rec.StreamId, expectedVersion, (transInfo.FirstPrepareId == rec.Id ? PrepareFlags.TransactionBegin : PrepareFlags.None) | (transInfo.LastPrepareId == rec.Id ? PrepareFlags.TransactionEnd : PrepareFlags.None), rec.EventType, LogRecord.NoData, null, rec.TimeStamp); break; } case Rec.RecType.Commit: { record = LogRecord.Commit(logPos, Guid.NewGuid(), transInfo.TransactionPosition, transInfo.TransactionEventNumber); if (transInfo.StreamMetadata.HasValue) streams[rec.StreamId].StreamMetadata = transInfo.StreamMetadata.Value; if (transInfo.IsDelete) streams[rec.StreamId].StreamVersion = EventNumber.DeletedStream; else streams[rec.StreamId].StreamVersion = transInfo.TransactionEventNumber + transInfo.TransactionOffset - 1; break; } default: throw new ArgumentOutOfRangeException(); } var writerRes = chunk.TryAppend(record); if (!writerRes.Success) throw new Exception(string.Format("Couldn't write log record: {0}", record)); _db.Config.WriterCheckpoint.Write(i * (long)_db.Config.ChunkSize + writerRes.NewPosition); records[i][j] = record; } if (i < _chunkRecs.Count-1 || (_completeLast && i == _chunkRecs.Count - 1)) chunk.Complete(); } return new DbResult(_db, records, streams); }
private static PosMap WriteRecord(TFChunk newChunk, LogRecord record) { var writeResult = newChunk.TryAppend(record); if (!writeResult.Success) { throw new Exception(string.Format( "Unable to append record during scavenging. Scavenge position: {0}, Record: {1}.", writeResult.OldPosition, record)); } int logPos = (int) (record.Position%newChunk.ChunkHeader.ChunkSize); int actualPos = (int) writeResult.OldPosition; return new PosMap(logPos, actualPos); }