public void Build()
{
_tableIndex.Initialize();
_persistedPrepareCheckpoint = _tableIndex.PrepareCheckpoint;
_persistedCommitCheckpoint = _tableIndex.CommitCheckpoint.ReadNonFlushed();
foreach (var rdr in _readers)
{
rdr.Open();
}
long pos = Math.Max(0, _persistedCommitCheckpoint);
long processed = 0;
var chaser = _chaserFactory(pos);
RecordReadResult result;
while ((result = chaser.TryReadNext()).Success)
{
//Debug.WriteLine(result.LogRecord);
switch (result.LogRecord.RecordType)
{
case LogRecordType.Prepare:
{
//Prepare((PrepareLogRecord) result.LogRecord);
break;
}
case LogRecordType.Commit:
{
Commit((CommitLogRecord)result.LogRecord);
break;
}
default:
throw new ArgumentOutOfRangeException();
}
processed += 1;
if (processed % 100000 == 0)
{
Log.Debug("ReadIndex Rebuilding: processed {0} records.", processed);
}
}
}
public void Build()
{
_tableIndex.Initialize();
_persistedPrepareCheckpoint = _tableIndex.PrepareCheckpoint;
_persistedCommitCheckpoint = _tableIndex.CommitCheckpoint;
_lastCommitPosition = _tableIndex.CommitCheckpoint;
foreach (var rdr in _readers)
{
rdr.Open();
}
foreach (var seqRdr in _seqReaders)
{
seqRdr.Open();
}
var seqReader = GetSeqReader();
try
{
seqReader.Reposition(Math.Max(0, _persistedCommitCheckpoint));
long processed = 0;
SeqReadResult result;
while ((result = seqReader.TryReadNext()).Success)
{
if (result.LogRecord.RecordType == LogRecordType.Commit)
{
Commit((CommitLogRecord)result.LogRecord);
}
processed += 1;
if (processed % 100000 == 0)
{
Log.Debug("ReadIndex Rebuilding: processed {0} records.", processed);
}
}
}
finally
{
ReturnSeqReader(seqReader);
}
_indexRebuild = false;
}
public void Init(long buildToPosition)
{
Log.Info("TableIndex initialization...");
_tableIndex.Initialize(buildToPosition);
_persistedPreparePos = _tableIndex.PrepareCheckpoint;
_persistedCommitPos = _tableIndex.CommitCheckpoint;
_lastCommitPosition = _tableIndex.CommitCheckpoint;
if (_lastCommitPosition >= buildToPosition)
{
throw new Exception(string.Format("_lastCommitPosition {0} >= buildToPosition {1}", _lastCommitPosition, buildToPosition));
}
var startTime = DateTime.UtcNow;
var lastTime = DateTime.UtcNow;
var reportPeriod = TimeSpan.FromSeconds(5);
Log.Info("ReadIndex building...");
_indexRebuild = true;
using (var reader = _backend.BorrowReader())
{
var startPosition = Math.Max(0, _persistedCommitPos);
reader.Reposition(startPosition);
var commitedPrepares = new List <PrepareLogRecord>();
long processed = 0;
SeqReadResult result;
while ((result = reader.TryReadNext()).Success && result.LogRecord.LogPosition < buildToPosition)
{
switch (result.LogRecord.RecordType)
{
case LogRecordType.Prepare:
{
var prepare = (PrepareLogRecord)result.LogRecord;
if (prepare.Flags.HasAnyOf(PrepareFlags.IsCommitted))
{
if (prepare.Flags.HasAnyOf(PrepareFlags.Data | PrepareFlags.StreamDelete))
{
commitedPrepares.Add(prepare);
}
if (prepare.Flags.HasAnyOf(PrepareFlags.TransactionEnd))
{
Commit(commitedPrepares);
commitedPrepares.Clear();
}
}
break;
}
case LogRecordType.Commit:
Commit((CommitLogRecord)result.LogRecord);
break;
case LogRecordType.System:
break;
default:
throw new Exception(string.Format("Unknown RecordType: {0}", result.LogRecord.RecordType));
}
processed += 1;
if (DateTime.UtcNow - lastTime > reportPeriod || processed % 100000 == 0)
{
Log.Debug("ReadIndex Rebuilding: processed {0} records ({1:0.0}%).",
processed, (result.RecordPostPosition - startPosition) * 100.0 / (buildToPosition - startPosition));
lastTime = DateTime.UtcNow;
}
}
Log.Debug("ReadIndex rebuilding done: total processed {0} records, time elapsed: {1}.", processed, DateTime.UtcNow - startTime);
_backend.SetSystemSettings(GetSystemSettings());
}
_indexRebuild = false;
}
public void Init(long buildToPosition)
{
Log.Information("TableIndex initialization...");
_tableIndex.Initialize(buildToPosition);
_persistedPreparePos = _tableIndex.PrepareCheckpoint;
_persistedCommitPos = _tableIndex.CommitCheckpoint;
//todo(clc) determin if this needs to move into the TableIndex re:project-io
_indexChk.Write(_tableIndex.CommitCheckpoint);
_indexChk.Flush();
if (_indexChk.Read() >= buildToPosition)
{
throw new Exception(string.Format("_lastCommitPosition {0} >= buildToPosition {1}", _indexChk.Read(),
buildToPosition));
}
var startTime = DateTime.UtcNow;
var lastTime = DateTime.UtcNow;
var reportPeriod = TimeSpan.FromSeconds(5);
Log.Information("ReadIndex building...");
_indexRebuild = true;
using (var reader = _backend.BorrowReader()) {
var startPosition = Math.Max(0, _persistedCommitPos);
reader.Reposition(startPosition);
var commitedPrepares = new List <PrepareLogRecord>();
long processed = 0;
SeqReadResult result;
while ((result = reader.TryReadNext()).Success && result.LogRecord.LogPosition < buildToPosition)
{
switch (result.LogRecord.RecordType)
{
case LogRecordType.Prepare: {
var prepare = (PrepareLogRecord)result.LogRecord;
if (prepare.Flags.HasAnyOf(PrepareFlags.IsCommitted))
{
if (prepare.Flags.HasAnyOf(PrepareFlags.SingleWrite))
{
Commit(commitedPrepares, false, false);
commitedPrepares.Clear();
Commit(new[] { prepare }, result.Eof, false);
}
else
{
if (prepare.Flags.HasAnyOf(PrepareFlags.Data | PrepareFlags.StreamDelete))
{
commitedPrepares.Add(prepare);
}
if (prepare.Flags.HasAnyOf(PrepareFlags.TransactionEnd))
{
Commit(commitedPrepares, result.Eof, false);
commitedPrepares.Clear();
}
}
}
break;
}
case LogRecordType.Commit:
Commit((CommitLogRecord)result.LogRecord, result.Eof, false);
break;
case LogRecordType.System:
break;
default:
throw new Exception(string.Format("Unknown RecordType: {0}", result.LogRecord.RecordType));
}
processed += 1;
if (DateTime.UtcNow - lastTime > reportPeriod || processed % 100000 == 0)
{
Log.Debug("ReadIndex Rebuilding: processed {processed} records ({resultPosition:0.0}%).",
processed,
(result.RecordPostPosition - startPosition) * 100.0 / (buildToPosition - startPosition));
lastTime = DateTime.UtcNow;
}
}
Log.Debug("ReadIndex rebuilding done: total processed {processed} records, time elapsed: {elapsed}.",
processed, DateTime.UtcNow - startTime);
_bus.Publish(new StorageMessage.TfEofAtNonCommitRecord());
_backend.SetSystemSettings(GetSystemSettings());
}
_indexRebuild = false;
}