public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
16 * 1024,
0,
new InMemoryCheckpoint(),
new ICheckpoint[0]));
_db.OpenVerifyAndClean();
var chunk = _db.Manager.GetChunk(0);
_rec1 = LogRecord.SingleWrite(0, Guid.NewGuid(), Guid.NewGuid(), "es1", ExpectedVersion.Any, "et1",
new byte[] { 0, 1, 2 }, new byte[] { 5, 7 });
_res1 = chunk.TryAppend(_rec1);
_rec2 = LogRecord.SingleWrite(_res1.NewPosition,
Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1",
new byte[] { 0, 1, 2 }, new byte[] { 5, 7 });
_res2 = chunk.TryAppend(_rec2);
_rec3 = LogRecord.SingleWrite(_res2.NewPosition,
Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1",
new byte[] { 0, 1, 2 }, new byte[] { 5, 7 });
_res3 = chunk.TryAppend(_rec3);
chunk.Complete();
var scavenger = new TFChunkScavenger(_db, new FakeReadIndex(x => x == "es-to-scavenge"));
scavenger.Scavenge(alwaysKeepScavenged: true);
_scavengedChunk = _db.Manager.GetChunk(0);
}
public RecordReadResult(bool success, long nextPosition, LogRecord logRecord, int recordLength)
{
Success = success;
LogRecord = logRecord;
NextPosition = nextPosition;
RecordLength = recordLength;
}
public bool Write(LogRecord record, out long newPos)
{
var result = _currentChunk.TryAppend(record);
if (result.Success)
_writerCheckpoint.Write(result.NewPosition + _currentChunk.ChunkHeader.ChunkStartPosition);
else
CompleteChunk(); // complete updates checkpoint internally
newPos = _writerCheckpoint.ReadNonFlushed();
return result.Success;
}
public SeqReadResult(bool success,
bool eof,
LogRecord logRecord,
int recordLength,
long recordPrePosition,
long recordPostPosition)
{
Success = success;
Eof = eof;
LogRecord = logRecord;
RecordLength = recordLength;
RecordPrePosition = recordPrePosition;
RecordPostPosition = recordPostPosition;
}
public bool Write(LogRecord record, out long newPos)
{
var chunkNum = (int)(_writerPos / _db.Config.ChunkSize);
var chunkPos = _writerPos % _db.Config.ChunkSize;
var result = _writerChunk.TryAppend(record);
if (result.Success)
{
Debug.Assert(result.OldPosition == chunkPos);
_writerPos = chunkNum * (long)_db.Config.ChunkSize + result.NewPosition;
_writerCheckpoint.Write(_writerPos);
}
else
{
CompleteChunk(); // complete updates checkpoint internally
}
newPos = _writerPos;
return result.Success;
}
public bool Write(LogRecord record, out long newPos)
{
var chunkNum = (int)(_writerPos / _db.Config.ChunkSize);
var chunkPos = _writerPos % _db.Config.ChunkSize;
var result = _writerChunk.TryAppend(record);
if (result.Success)
{
Debug.Assert(result.OldPosition == chunkPos);
_writerPos = chunkNum * (long)_db.Config.ChunkSize + result.NewPosition;
}
else
{
_writerChunk.Flush();
_writerChunk.Complete();
_writerCheckpoint.Flush(); //flush our checkpoint
_writerChunk = _db.Manager.AddNewChunk();
_writerPos = (chunkNum + 1) * (long)_db.Config.ChunkSize; // we just moved to a new chunk at pos 0
}
_writerCheckpoint.Write(_writerPos);
newPos = _writerPos;
return result.Success;
}
public RecordWriteResult TryAppend(LogRecord record)
{
if (_isReadonly)
throw new InvalidOperationException("Cannot write to a read-only block.");
var workItem = _writerWorkItem;
var buffer = workItem.Buffer;
var bufferWriter = workItem.BufferWriter;
var stream = workItem.Stream;
buffer.SetLength(4);
buffer.Position = 4;
record.WriteTo(bufferWriter);
var length = (int) buffer.Length - 4;
bufferWriter.Write(length); // length suffix
buffer.Position = 0;
bufferWriter.Write(length); // length prefix
if (stream.Position + length + 8 > ChunkHeader.Size + _chunkHeader.ChunkSize)
return RecordWriteResult.Failed(GetLogicalPosition(workItem));
var oldPosition = WriteRawData(buffer);
_actualDataSize = GetLogicalPosition(workItem);
return RecordWriteResult.Successful(oldPosition, _actualDataSize);
}
private bool TryReadForwardInternal(ReaderWorkItem workItem, int actualPosition, out int length, out LogRecord record)
{
length = -1;
record = null;
workItem.Stream.Position = GetRealPosition(actualPosition, workItem.IsMemory);
if (!VerifyDataLengthForward(workItem, 2*sizeof(int)))
return false;
length = workItem.Reader.ReadInt32();
if (length <= 0)
{
throw new ArgumentException(
string.Format("Log record at actual pos {0} has non-positive length: {1}. "
+ "Something is seriously wrong.",
actualPosition,
length));
}
if (length > TFConsts.MaxLogRecordSize)
{
throw new ArgumentException(
string.Format("Log record at actual pos {0} has too large length: {1} bytes, "
+ "while limit is {2} bytes.",
actualPosition,
length,
TFConsts.MaxLogRecordSize));
}
if (!VerifyDataLengthForward(workItem, length + sizeof(int) /*suffix*/))
throw new UnableToReadPastEndOfStreamException();
record = LogRecord.ReadFrom(workItem.Reader);
Debug.Assert(workItem.Reader.ReadInt32() == length); // verify suffix length == prefix length
return true;
}
public RecordWriteResult TryAppend(LogRecord record)
{
if (_isReadOnly)
throw new InvalidOperationException("Cannot write to a read-only block.");
var workItem = _writerWorkItem;
var buffer = workItem.Buffer;
var bufferWriter = workItem.BufferWriter;
buffer.SetLength(4);
buffer.Position = 4;
record.WriteTo(bufferWriter);
var length = (int) buffer.Length - 4;
bufferWriter.Write(length); // length suffix
buffer.Position = 0;
bufferWriter.Write(length); // length prefix
if (workItem.StreamPosition + length + 2*sizeof(int) > ChunkHeader.Size + _chunkHeader.ChunkSize)
return RecordWriteResult.Failed(GetDataPosition(workItem));
var oldPosition = WriteRawData(workItem, buffer);
_physicalDataSize = (int)GetDataPosition(workItem); // should fit 32 bits
_logicalDataSize = ChunkHeader.GetLocalLogPosition(record.LogPosition + length + 2*sizeof(int));
// for non-scavenged chunk _physicalDataSize should be the same as _logicalDataSize
// for scavenged chunk _logicalDataSize should be at least the same as _physicalDataSize
if ((!ChunkHeader.IsScavenged && _logicalDataSize != _physicalDataSize)
|| (ChunkHeader.IsScavenged && _logicalDataSize < _physicalDataSize))
{
throw new Exception(string.Format("Data sizes violation. Chunk: {0}, IsScavenged: {1}, LogicalDataSize: {2}, PhysicalDataSize: {3}.",
FileName, ChunkHeader.IsScavenged, _logicalDataSize, _physicalDataSize));
}
return RecordWriteResult.Successful(oldPosition, _physicalDataSize);
}
public DbResult(TFChunkDb db, LogRecord[][] recs, Dictionary<string, StreamInfo> streams)
{
Ensure.NotNull(db, "db");
Ensure.NotNull(recs, "recs");
Ensure.NotNull(streams, "streams");
Db = db;
Recs = recs;
Streams = streams;
}
protected static bool TryReadBackwardInternal(ReaderWorkItem workItem, int actualPosition, out int length, out LogRecord record)
{
length = -1;
record = null;
if (actualPosition < 2 * sizeof(int)) // no space even for length prefix and suffix
return false;
var realPos = GetRealPosition(actualPosition);
workItem.Stream.Position = realPos - sizeof(int);
length = workItem.Reader.ReadInt32();
if (length <= 0)
{
throw new ArgumentException(
string.Format("Log record that ends at actual pos {0} has non-positive length: {1}. "
+ "Something is seriously wrong.",
actualPosition,
length));
}
if (length > TFConsts.MaxLogRecordSize)
{
throw new ArgumentException(
string.Format("Log record that ends at actual pos {0} has too large length: {1} bytes, "
+ "while limit is {2} bytes.",
actualPosition,
length,
TFConsts.MaxLogRecordSize));
}
if (actualPosition < length + 2 * sizeof(int)) // no space for record + length prefix and suffix
throw new UnableToReadPastEndOfStreamException(
string.Format("There is not enough space to read full record (record length according to length suffix: {0}).", length));
workItem.Stream.Position = realPos - length - sizeof(int);
record = LogRecord.ReadFrom(workItem.Reader);
#if DEBUG
workItem.Stream.Position = realPos - length - 2 * sizeof(int);
var prefixLength = workItem.Reader.ReadInt32();
Debug.Assert(prefixLength == length);
#endif
return true;
}
private static bool IsCommitAlike(LogRecord rec)
{
return rec.RecordType == LogRecordType.Commit
|| (rec.RecordType == LogRecordType.Prepare && ((PrepareLogRecord)rec).Flags.HasAnyOf(PrepareFlags.IsCommitted));
}
public bool Write(LogRecord record, out long newPos)
{
_buffer.SetLength(4);
_buffer.Position = 4;
record.WriteTo(_bufferWriter);
var length = (int) _buffer.Length - 4;
_bufferWriter.Write(length); // length suffix
_buffer.Position = 0;
_bufferWriter.Write(length); // length prefix
var written = 0;
var leftToWrite = length + 8;
while (leftToWrite > 0)
{
var leftInFile = (int)(_segmentSize - _writerLocalCheck);
var amountToWrite = leftToWrite > leftInFile ? leftInFile : leftToWrite;
_fileStream.Write(_buffer.GetBuffer(), written, amountToWrite);
leftToWrite -= amountToWrite;
written += amountToWrite;
_writerCheck += amountToWrite;
_writerLocalCheck += amountToWrite;
if (_writerLocalCheck >= _segmentSize)
{
_writerLocalCheck -= _segmentSize;
_currentSegment++;
CreateSegment(_currentSegment);
}
}
_writerCheckpoint.Write(_writerCheck);
newPos = _writerCheck;
return true;
}
public RecordWriteResult TryAppend(LogRecord record)
{
if (_isReadonly)
throw new InvalidOperationException("Cannot write to a read-only block.");
var workItem = _writerWorkItem;
var buffer = workItem.Buffer;
var bufferWriter = workItem.BufferWriter;
var stream = workItem.Stream;
buffer.SetLength(4);
buffer.Position = 4;
record.WriteTo(bufferWriter);
buffer.Position = 0;
var toWriteLength = (int) buffer.Length;
bufferWriter.Write(toWriteLength - 4);
if (!VerifyStreamLength(stream, toWriteLength))
return RecordWriteResult.Failed(GetLogicalPosition(workItem));
var oldPosition = WriteRawData(buffer);
_actualDataSize = GetLogicalPosition(workItem);
return RecordWriteResult.Successful(oldPosition, _actualDataSize);
}
private bool TryReadRecordInternal(ReaderWorkItem workItem, int actualPosition, out int length, out LogRecord record)
{
length = -1;
record = null;
workItem.Stream.Position = GetRealPosition(actualPosition, workItem.IsMemory);
if (!VerifyStreamLength(workItem.Stream, 4))
return false;
length = workItem.Reader.ReadInt32();
CheckLength(workItem, length, actualPosition);
record = LogRecord.ReadFrom(workItem.Reader);
return true;
}
public RecordReadResult(bool success, LogRecord logRecord, long nextPosition)
{
Success = success;
LogRecord = logRecord;
NextPosition = nextPosition;
}
protected bool TryReadForwardInternal(ReaderWorkItem workItem, int actualPosition, out int length, out LogRecord record)
{
length = -1;
record = null;
workItem.Stream.Position = GetRealPosition(actualPosition);
if (actualPosition + 2*sizeof(int) > Chunk.ActualDataSize) // no space even for length prefix and suffix
return false;
length = workItem.Reader.ReadInt32();
if (length <= 0)
{
throw new ArgumentException(
string.Format("Log record at actual pos {0} has non-positive length: {1}. "
+ "Something is seriously wrong in chunk {2}-{3} ({4}).",
actualPosition,
length,
Chunk.ChunkHeader.ChunkStartNumber,
Chunk.ChunkHeader.ChunkEndNumber,
Chunk.FileName));
}
if (length > TFConsts.MaxLogRecordSize)
{
throw new ArgumentException(
string.Format("Log record at actual pos {0} has too large length: {1} bytes, "
+ "while limit is {2} bytes. Something is seriously wrong in chunk {3}-{4} ({5}).",
actualPosition,
length,
TFConsts.MaxLogRecordSize,
Chunk.ChunkHeader.ChunkStartNumber,
Chunk.ChunkHeader.ChunkEndNumber,
Chunk.FileName));
}
if (actualPosition + length + 2*sizeof(int) > Chunk.ActualDataSize)
throw new UnableToReadPastEndOfStreamException(
string.Format("There is not enough space to read full record (record length according to length prefix: {0}).", length));
record = LogRecord.ReadFrom(workItem.Reader);
#if DEBUG
// verify suffix length == prefix length
int suffixLength = workItem.Reader.ReadInt32();
Debug.Assert(suffixLength == length,
"Prefix/suffix length inconsistency ",
"Prefix length({0}) != suffix length ({1})",
length,
suffixLength);
#endif
return true;
}
protected bool TryReadForwardInternal(ReaderWorkItem workItem, long actualPosition, out int length, out LogRecord record)
{
length = -1;
record = null;
workItem.Stream.Position = GetRawPosition(actualPosition);
if (actualPosition + 2*sizeof(int) > Chunk.PhysicalDataSize) // no space even for length prefix and suffix
return false;
length = workItem.Reader.ReadInt32();
if (length <= 0)
{
throw new InvalidReadException(
string.Format("Log record at actual pos {0} has non-positive length: {1}. "
+ " in chunk.", actualPosition, length, Chunk));
}
if (length > TFConsts.MaxLogRecordSize)
{
throw new InvalidReadException(
string.Format("Log record at actual pos {0} has too large length: {1} bytes, "
+ "while limit is {2} bytes. In chunk {3}.",
actualPosition, length, TFConsts.MaxLogRecordSize, Chunk));
}
if (actualPosition + length + 2 * sizeof(int) > Chunk.PhysicalDataSize)
{
throw new UnableToReadPastEndOfStreamException(
string.Format("There is not enough space to read full record (length prefix: {0}). "
+ "Actual pre-position: {1}. Something is seriously wrong in chunk {2}.",
length, actualPosition, Chunk));
}
record = LogRecord.ReadFrom(workItem.Reader);
// verify suffix length == prefix length
int suffixLength = workItem.Reader.ReadInt32();
if (suffixLength != length)
{
throw new Exception(
string.Format("Prefix/suffix length inconsistency: prefix length({0}) != suffix length ({1}).\n"
+ "Actual pre-position: {2}. Something is seriously wrong in chunk {3}.",
length, suffixLength, actualPosition, Chunk));
}
return true;
}
public bool TryReadNext(out LogRecord record)
{
var res = TryReadNext();
record = res.LogRecord;
return res.Success;
}
protected bool TryReadBackwardInternal(ReaderWorkItem workItem, long actualPosition, out int length, out LogRecord record)
{
length = -1;
record = null;
if (actualPosition < 2 * sizeof(int)) // no space even for length prefix and suffix
return false;
var realPos = GetRawPosition(actualPosition);
workItem.Stream.Position = realPos - sizeof(int);
length = workItem.Reader.ReadInt32();
if (length <= 0)
{
throw new ArgumentException(
string.Format("Log record that ends at actual pos {0} has non-positive length: {1}. "
+ "Something is seriously wrong in chunk {2}.",
actualPosition, length, Chunk));
}
if (length > TFConsts.MaxLogRecordSize)
{
throw new ArgumentException(
string.Format("Log record that ends at actual pos {0} has too large length: {1} bytes, "
+ "while limit is {2} bytes. Something is seriously wrong in chunk {3}.",
actualPosition, length, TFConsts.MaxLogRecordSize, Chunk));
}
if (actualPosition < length + 2 * sizeof(int)) // no space for record + length prefix and suffix
{
throw new UnableToReadPastEndOfStreamException(
string.Format("There is not enough space to read full record (length suffix: {0}). "
+ "Actual post-position: {1}. Something is seriously wrong in chunk {2}.",
length, actualPosition, Chunk));
}
workItem.Stream.Position = realPos - length - 2*sizeof(int);
// verify suffix length == prefix length
int prefixLength = workItem.Reader.ReadInt32();
if (prefixLength != length)
{
throw new Exception(
string.Format("Prefix/suffix length inconsistency: prefix length({0}) != suffix length ({1})"
+ "Actual post-position: {2}. Something is seriously wrong in chunk {3}.",
prefixLength, length, actualPosition, Chunk));
}
record = LogRecord.ReadFrom(workItem.Reader);
return true;
}
public DbResult CreateDb()
{
var records = new LogRecord[_chunkRecs.Count][];
for (int i = 0; i < records.Length; ++i)
{
records[i] = new LogRecord[_chunkRecs[i].Length];
}
var transactions = new Dictionary<int, TransactionInfo>();
var streams = new Dictionary<string, StreamInfo>();
var streamUncommitedVersion = new Dictionary<string, int>();
for (int i = 0; i < _chunkRecs.Count; ++i)
{
for (int j = 0; j < _chunkRecs[i].Length; ++j)
{
var rec = _chunkRecs[i][j];
TransactionInfo transInfo;
bool transCreate = transactions.TryGetValue(rec.Transaction, out transInfo);
if (!transCreate)
{
if (rec.Type == Rec.RecType.Commit)
throw new Exception("Commit for non-existing transaction.");
transactions[rec.Transaction] = transInfo = new TransactionInfo(rec.StreamId, rec.Id, rec.Id);
streams[rec.StreamId] = new StreamInfo(-1);
streamUncommitedVersion[rec.StreamId] = -1;
}
else
{
if (rec.Type == Rec.RecType.TransStart)
throw new Exception(string.Format("Not expected record type: {0}.", rec.Type));
}
if (transInfo.StreamId != rec.StreamId)
{
throw new Exception(string.Format("Wrong stream id for transaction. Transaction StreamId: {0}, record StreamId: {1}.",
transInfo.StreamId,
rec.StreamId));
}
if (rec.Type != Rec.RecType.Commit && transInfo.IsDelete)
throw new Exception("Transaction with records after delete record.");
if (rec.Type == Rec.RecType.Delete)
transInfo.IsDelete = true;
transInfo.LastPrepareId = rec.Id;
}
}
for (int i = 0; i < _chunkRecs.Count; ++i)
{
var chunk = i == 0 ? _db.Manager.GetChunk(0) : _db.Manager.AddNewChunk();
_db.Config.WriterCheckpoint.Write(i * (long)_db.Config.ChunkSize);
for (int j = 0; j < _chunkRecs[i].Length; ++j)
{
var rec = _chunkRecs[i][j];
var transInfo = transactions[rec.Transaction];
var logPos = _db.Config.WriterCheckpoint.ReadNonFlushed();
int streamVersion = streamUncommitedVersion[rec.StreamId];
if (streamVersion == -1 && rec.Type != Rec.RecType.TransStart && rec.Type != Rec.RecType.Create)
throw new Exception(string.Format("Stream {0} is empty and we are not creating it with stream created.", rec.StreamId));
if (streamVersion == EventNumber.DeletedStream && rec.Type != Rec.RecType.Commit)
throw new Exception(string.Format("Stream {0} was deleted, but we need to write some more prepares.", rec.StreamId));
if (transInfo.FirstPrepareId == rec.Id)
{
transInfo.TransactionPosition = logPos;
transInfo.TransactionEventNumber = streamVersion + 1;
transInfo.TransactionOffset = 0;
}
LogRecord record;
var expectedVersion = transInfo.FirstPrepareId == rec.Id ? streamVersion : ExpectedVersion.Any;
switch (rec.Type)
{
case Rec.RecType.Prepare:
case Rec.RecType.Create:
{
int transOffset = transInfo.TransactionOffset;
transInfo.TransactionOffset += 1;
record = LogRecord.Prepare(logPos,
Guid.NewGuid(),
rec.Id,
transInfo.TransactionPosition,
transOffset,
rec.StreamId,
expectedVersion,
PrepareFlags.Data
| (transInfo.FirstPrepareId == rec.Id ? PrepareFlags.TransactionBegin : PrepareFlags.None)
| (transInfo.LastPrepareId == rec.Id ? PrepareFlags.TransactionEnd : PrepareFlags.None),
rec.EventType,
rec.Id.ToByteArray(),
null,
rec.TimeStamp);
if (rec.Type == Rec.RecType.Create)
transInfo.StreamMetadata = rec.Metadata;
streamUncommitedVersion[rec.StreamId] += 1;
break;
}
case Rec.RecType.Delete:
{
int transOffset = transInfo.TransactionOffset;
transInfo.TransactionOffset += 1;
record = LogRecord.Prepare(logPos,
Guid.NewGuid(),
rec.Id,
transInfo.TransactionPosition,
transOffset,
rec.StreamId,
expectedVersion,
PrepareFlags.StreamDelete
| (transInfo.FirstPrepareId == rec.Id ? PrepareFlags.TransactionBegin : PrepareFlags.None)
| (transInfo.LastPrepareId == rec.Id ? PrepareFlags.TransactionEnd : PrepareFlags.None),
rec.EventType,
LogRecord.NoData,
null,
rec.TimeStamp);
streamUncommitedVersion[rec.StreamId] = EventNumber.DeletedStream;
break;
}
case Rec.RecType.TransStart:
case Rec.RecType.TransEnd:
{
record = LogRecord.Prepare(logPos,
Guid.NewGuid(),
rec.Id,
transInfo.TransactionPosition,
-1,
rec.StreamId,
expectedVersion,
(transInfo.FirstPrepareId == rec.Id ? PrepareFlags.TransactionBegin : PrepareFlags.None)
| (transInfo.LastPrepareId == rec.Id ? PrepareFlags.TransactionEnd : PrepareFlags.None),
rec.EventType,
LogRecord.NoData,
null,
rec.TimeStamp);
break;
}
case Rec.RecType.Commit:
{
record = LogRecord.Commit(logPos, Guid.NewGuid(), transInfo.TransactionPosition, transInfo.TransactionEventNumber);
if (transInfo.StreamMetadata.HasValue)
streams[rec.StreamId].StreamMetadata = transInfo.StreamMetadata.Value;
if (transInfo.IsDelete)
streams[rec.StreamId].StreamVersion = EventNumber.DeletedStream;
else
streams[rec.StreamId].StreamVersion = transInfo.TransactionEventNumber + transInfo.TransactionOffset - 1;
break;
}
default:
throw new ArgumentOutOfRangeException();
}
var writerRes = chunk.TryAppend(record);
if (!writerRes.Success)
throw new Exception(string.Format("Couldn't write log record: {0}", record));
_db.Config.WriterCheckpoint.Write(i * (long)_db.Config.ChunkSize + writerRes.NewPosition);
records[i][j] = record;
}
if (i < _chunkRecs.Count-1 || (_completeLast && i == _chunkRecs.Count - 1))
chunk.Complete();
}
return new DbResult(_db, records, streams);
}
private static PosMap WriteRecord(TFChunk newChunk, LogRecord record)
{
var writeResult = newChunk.TryAppend(record);
if (!writeResult.Success)
{
throw new Exception(string.Format(
"Unable to append record during scavenging. Scavenge position: {0}, Record: {1}.",
writeResult.OldPosition,
record));
}
int logPos = (int) (record.Position%newChunk.ChunkHeader.ChunkSize);
int actualPos = (int) writeResult.OldPosition;
return new PosMap(logPos, actualPos);
}
