IndexReadEventResult IIndexReader.ReadEvent(string streamId, long eventNumber)
{
Ensure.NotNullOrEmpty(streamId, "streamId");
if (eventNumber < -1)
{
throw new ArgumentOutOfRangeException("eventNumber");
}
using (var reader = _backend.BorrowReader()) {
return(ReadEventInternal(reader, streamId, eventNumber));
}
}
private IndexReadAllResult ReadAllEventsForwardInternal(TFPos pos, int maxCount, int maxSearchWindow,
IEventFilter eventFilter)
{
var records = new List <CommitEventRecord>();
var nextPos = pos;
// in case we are at position after which there is no commit at all, in that case we have to force
// PreparePosition to long.MaxValue, so if you decide to read backwards from PrevPos,
// you will receive all prepares.
var prevPos = new TFPos(pos.CommitPosition, long.MaxValue);
var consideredEventsCount = 0L;
var firstCommit = true;
var reachedEndOfStream = false;
using (var reader = _backend.BorrowReader()) {
long nextCommitPos = pos.CommitPosition;
while (records.Count < maxCount && consideredEventsCount < maxSearchWindow)
{
if (nextCommitPos > _indexCommitter.LastIndexedPosition)
{
reachedEndOfStream = true;
break;
}
reader.Reposition(nextCommitPos);
SeqReadResult result;
while ((result = reader.TryReadNext()).Success && !IsCommitAlike(result.LogRecord))
{
// skip until commit
}
if (!result.Success) // no more records in TF
{
break;
}
nextCommitPos = result.RecordPostPosition;
switch (result.LogRecord.RecordType)
{
case LogRecordType.Prepare:
case LogRecordType.Stream: {
var prepare = (IPrepareLogRecord <TStreamId>)result.LogRecord;
if (firstCommit)
{
firstCommit = false;
prevPos = new TFPos(result.RecordPrePosition, result.RecordPrePosition);
}
if (prepare.Flags.HasAnyOf(PrepareFlags.Data | PrepareFlags.StreamDelete) &&
new TFPos(prepare.LogPosition, prepare.LogPosition) >= pos)
{
var streamName = _streamNames.LookupName(prepare.EventStreamId);
var eventRecord = new EventRecord(eventNumber: prepare.ExpectedVersion + 1,
prepare, streamName);
consideredEventsCount++;
if (eventFilter.IsEventAllowed(eventRecord))
{
records.Add(new CommitEventRecord(eventRecord, prepare.LogPosition));
}
nextPos = new TFPos(result.RecordPostPosition, 0);
}
break;
}
case LogRecordType.Commit: {
var commit = (CommitLogRecord)result.LogRecord;
if (firstCommit)
{
firstCommit = false;
// for backward pass we want to allow read the same commit and skip read prepares,
// so we put post-position of commit and post-position of prepare as TFPos for backward pass
prevPos = new TFPos(result.RecordPostPosition, pos.PreparePosition);
}
reader.Reposition(commit.TransactionPosition);
while (consideredEventsCount < maxCount)
{
result = reader.TryReadNext();
if (!result.Success) // no more records in TF
{
break;
}
// prepare with TransactionEnd could be scavenged already
// so we could reach the same commit record. In that case have to stop
if (result.LogRecord.LogPosition >= commit.LogPosition)
{
break;
}
if (result.LogRecord.RecordType != LogRecordType.Prepare)
{
continue;
}
var prepare = (IPrepareLogRecord <TStreamId>)result.LogRecord;
if (prepare.TransactionPosition != commit.TransactionPosition) // wrong prepare
{
continue;
}
// prepare with useful data or delete tombstone
if (prepare.Flags.HasAnyOf(PrepareFlags.Data | PrepareFlags.StreamDelete) &&
new TFPos(commit.LogPosition, prepare.LogPosition) >= pos)
{
var streamName = _streamNames.LookupName(prepare.EventStreamId);
var eventRecord =
new EventRecord(commit.FirstEventNumber + prepare.TransactionOffset,
prepare, streamName);
consideredEventsCount++;
if (eventFilter.IsEventAllowed(eventRecord))
{
records.Add(new CommitEventRecord(eventRecord, commit.LogPosition));
}
// for forward pass position is inclusive,
// so we put pre-position of commit and post-position of prepare
nextPos = new TFPos(commit.LogPosition, result.RecordPostPosition);
}
if (prepare.Flags.HasAnyOf(PrepareFlags.TransactionEnd))
{
break;
}
}
break;
}
default:
throw new Exception(string.Format("Unexpected log record type: {0}.",
result.LogRecord.RecordType));
}
}
return(new IndexReadAllResult(records, pos, nextPos, prevPos, reachedEndOfStream, consideredEventsCount));
}
}
public void Init(long buildToPosition)
{
Log.Info("TableIndex initialization...");
_tableIndex.Initialize(buildToPosition);
_persistedPreparePos = _tableIndex.PrepareCheckpoint;
_persistedCommitPos = _tableIndex.CommitCheckpoint;
_lastCommitPosition = _tableIndex.CommitCheckpoint;
if (_lastCommitPosition >= buildToPosition)
{
throw new Exception(string.Format("_lastCommitPosition {0} >= buildToPosition {1}", _lastCommitPosition, buildToPosition));
}
var startTime = DateTime.UtcNow;
var lastTime = DateTime.UtcNow;
var reportPeriod = TimeSpan.FromSeconds(5);
Log.Info("ReadIndex building...");
_indexRebuild = true;
using (var reader = _backend.BorrowReader())
{
var startPosition = Math.Max(0, _persistedCommitPos);
reader.Reposition(startPosition);
var commitedPrepares = new List <PrepareLogRecord>();
long processed = 0;
SeqReadResult result;
while ((result = reader.TryReadNext()).Success && result.LogRecord.LogPosition < buildToPosition)
{
switch (result.LogRecord.RecordType)
{
case LogRecordType.Prepare:
{
var prepare = (PrepareLogRecord)result.LogRecord;
if (prepare.Flags.HasAnyOf(PrepareFlags.IsCommitted))
{
if (prepare.Flags.HasAnyOf(PrepareFlags.Data | PrepareFlags.StreamDelete))
{
commitedPrepares.Add(prepare);
}
if (prepare.Flags.HasAnyOf(PrepareFlags.TransactionEnd))
{
Commit(commitedPrepares);
commitedPrepares.Clear();
}
}
break;
}
case LogRecordType.Commit:
Commit((CommitLogRecord)result.LogRecord);
break;
case LogRecordType.System:
break;
default:
throw new Exception(string.Format("Unknown RecordType: {0}", result.LogRecord.RecordType));
}
processed += 1;
if (DateTime.UtcNow - lastTime > reportPeriod || processed % 100000 == 0)
{
Log.Debug("ReadIndex Rebuilding: processed {0} records ({1:0.0}%).",
processed, (result.RecordPostPosition - startPosition) * 100.0 / (buildToPosition - startPosition));
lastTime = DateTime.UtcNow;
}
}
Log.Debug("ReadIndex rebuilding done: total processed {0} records, time elapsed: {1}.", processed, DateTime.UtcNow - startTime);
_backend.SetSystemSettings(GetSystemSettings());
}
_indexRebuild = false;
}
public void Init(long buildToPosition)
{
Log.Information("TableIndex initialization...");
_tableIndex.Initialize(buildToPosition);
_persistedPreparePos = _tableIndex.PrepareCheckpoint;
_persistedCommitPos = _tableIndex.CommitCheckpoint;
//todo(clc) determin if this needs to move into the TableIndex re:project-io
_indexChk.Write(_tableIndex.CommitCheckpoint);
_indexChk.Flush();
if (_indexChk.Read() >= buildToPosition)
{
throw new Exception(string.Format("_lastCommitPosition {0} >= buildToPosition {1}", _indexChk.Read(),
buildToPosition));
}
var startTime = DateTime.UtcNow;
var lastTime = DateTime.UtcNow;
var reportPeriod = TimeSpan.FromSeconds(5);
Log.Information("ReadIndex building...");
_indexRebuild = true;
using (var reader = _backend.BorrowReader()) {
var startPosition = Math.Max(0, _persistedCommitPos);
reader.Reposition(startPosition);
var commitedPrepares = new List <PrepareLogRecord>();
long processed = 0;
SeqReadResult result;
while ((result = reader.TryReadNext()).Success && result.LogRecord.LogPosition < buildToPosition)
{
switch (result.LogRecord.RecordType)
{
case LogRecordType.Prepare: {
var prepare = (PrepareLogRecord)result.LogRecord;
if (prepare.Flags.HasAnyOf(PrepareFlags.IsCommitted))
{
if (prepare.Flags.HasAnyOf(PrepareFlags.SingleWrite))
{
Commit(commitedPrepares, false, false);
commitedPrepares.Clear();
Commit(new[] { prepare }, result.Eof, false);
}
else
{
if (prepare.Flags.HasAnyOf(PrepareFlags.Data | PrepareFlags.StreamDelete))
{
commitedPrepares.Add(prepare);
}
if (prepare.Flags.HasAnyOf(PrepareFlags.TransactionEnd))
{
Commit(commitedPrepares, result.Eof, false);
commitedPrepares.Clear();
}
}
}
break;
}
case LogRecordType.Commit:
Commit((CommitLogRecord)result.LogRecord, result.Eof, false);
break;
case LogRecordType.System:
break;
default:
throw new Exception(string.Format("Unknown RecordType: {0}", result.LogRecord.RecordType));
}
processed += 1;
if (DateTime.UtcNow - lastTime > reportPeriod || processed % 100000 == 0)
{
Log.Debug("ReadIndex Rebuilding: processed {processed} records ({resultPosition:0.0}%).",
processed,
(result.RecordPostPosition - startPosition) * 100.0 / (buildToPosition - startPosition));
lastTime = DateTime.UtcNow;
}
}
Log.Debug("ReadIndex rebuilding done: total processed {processed} records, time elapsed: {elapsed}.",
processed, DateTime.UtcNow - startTime);
_bus.Publish(new StorageMessage.TfEofAtNonCommitRecord());
_backend.SetSystemSettings(GetSystemSettings());
}
_indexRebuild = false;
}