public void Test_TruncateHead(
TruncateHeadLogRecord record,
IndexingLogRecord headRecord)
{
this.LastLinkedPhysicalRecord = record;
this.CurrentLogHeadRecord = headRecord;
this.LastInProgressTruncateHeadRecord = record;
this.LogManager.PhysicalLogWriter.InsertBufferedRecord(record);
}
internal static void Assert(bool condition, string format, TruncateHeadLogRecord param1, LogRecord param2)
{
if (condition == false)
{
var failFastMessage = string.Format(System.Globalization.CultureInfo.InvariantCulture, format, param1, param2);
FailFast(failFastMessage);
// AMW - Force break into debugger for ease of debugging
Debugger.Break();
}
}
/// <summary>
/// Do a check if our log matches truncate head criteria, and append truncate head record
/// </summary>
public TruncateHeadLogRecord TruncateHead(
bool isStable,
long lastPeriodicTruncationTimeTicks,
Func <IndexingLogRecord, bool> isGoodLogHeadCandidateCalculator)
{
lock (lsnOrderingLock)
{
var earliestRecord = this.GetEarliestRecordCallerHoldsLock();
IndexingLogRecord previousIndexingRecord = null;
PhysicalLogRecord previousPhysicalRecord = this.LastCompletedBeginCheckpointRecord;
TruncateHeadLogRecord record = null;
do
{
// Search for the last Indexing Log Record
do
{
previousPhysicalRecord = previousPhysicalRecord.PreviousPhysicalRecord;
} while (!(previousPhysicalRecord is IndexingLogRecord));
previousIndexingRecord = previousPhysicalRecord as IndexingLogRecord;
// This index record is not before the ealiest pending transaction record
if (previousIndexingRecord.RecordPosition >= earliestRecord.RecordPosition)
{
continue;
}
// We reached log head, so do not continue to look for last index log record
if (previousIndexingRecord == this.CurrentLogHeadRecord)
{
return(null);
}
if (isGoodLogHeadCandidateCalculator(previousIndexingRecord))
{
break;
}
} while (true);
record = new TruncateHeadLogRecord(
previousIndexingRecord,
this.CurrentLogTailLsn,
this.LastLinkedPhysicalRecord,
isStable,
lastPeriodicTruncationTimeTicks);
Test_TruncateHead(record, previousIndexingRecord);
return(record);
}
}
/// <summary>
/// Process the truncate head log record.
/// </summary>
/// <param name="truncateHeadLogRecord">The truncate head record to be processed.</param>
/// <param name="isRecoverableRecord">Is this a recoverable record.</param>
private void ProcessLogRecord(TruncateHeadLogRecord truncateHeadLogRecord, out bool isRecoverableRecord)
{
if (this.mode == Mode.Restore)
{
truncateHeadLogRecord.HeadRecord = this.firstIndexingLogRecord;
}
this.LastLinkedPhysicalRecord = truncateHeadLogRecord;
isRecoverableRecord = false;
this.LastPeriodicTruncationTimeTicks = truncateHeadLogRecord.PeriodicTruncationTimestampTicks;
}
internal Task ProcessLogHeadTruncationAsync(TruncateHeadLogRecord truncateHeadRecord)
{
var tcs = new TaskCompletionSource <object>();
var context = new PendingLogHeadTruncationContext(
tcs,
truncateHeadRecord);
lock (this.readersLock)
{
if (this.earliestLogReader.StartingRecordPosition < truncateHeadRecord.HeadRecord.RecordPosition)
{
Utility.Assert(
this.pendingLogHeadTruncationContext == null,
"Pending Log Head Truncation should be null");
this.pendingLogHeadTruncationContext = context;
FabricEvents.Events.LogManager(
this.Tracer.Type,
"ProcessLogHeadTruncationAsync: Could not truncate log head immediately due to reader: " + this.earliestLogReader.ReaderName);
return(tcs.Task);
}
else
{
this.LogHeadRecordPosition = truncateHeadRecord.HeadRecord.RecordPosition;
FabricEvents.Events.LogManager(
this.Tracer.Type,
"ProcessLogHeadTruncationAsync: Initiating log head truncation to position: " + this.LogHeadRecordPosition);
Task.Factory.StartNew(this.PerformLogHeadTruncation, context).IgnoreExceptionVoid();
return(tcs.Task);
}
}
}
public async Task <RecoveryInformation> OpenAsync(
ReplicaOpenMode openMode,
TransactionalReplicatorSettings replicatorSettings,
ITracer tracer,
LogManager logManager,
bool shouldLocalStateBeRemoved,
PhysicalLogReader recoveryReader,
bool isRestoring)
{
this.RecoveryLogsReader = recoveryReader;
this.tracer = tracer;
PhysicalLogRecord recoveredLinkedPhysicalRecord;
TruncateHeadLogRecord recoveredLastTruncateHeadLogRecord = null;
CompleteCheckpointLogRecord recoveredCompleteCheckpointRecord = null;
this.logCompleteCheckpointAfterRecovery = true;
this.tailRecordAtStart = await this.RecoveryLogsReader.SeekToLastRecord().ConfigureAwait(false);
var logUsage = logManager.Length;
var trace = "OpenAsync: Log Usage: " + logUsage + " Recovered Replica Tail Record Type: "
+ tailRecordAtStart.RecordType + " LSN: " + tailRecordAtStart.Lsn.LSN + " PSN: "
+ tailRecordAtStart.Psn.PSN + " Position: " + tailRecordAtStart.RecordPosition;
FabricEvents.Events.Api(tracer.Type, trace);
var lastPhysicalRecord = tailRecordAtStart as PhysicalLogRecord;
if (lastPhysicalRecord == null)
{
lastPhysicalRecord = await this.RecoveryLogsReader.GetPreviousPhysicalRecord(tailRecordAtStart).ConfigureAwait(false);
}
trace = "OpenAsync: Recovered last physical record Type: " + lastPhysicalRecord.RecordType + " LSN: "
+ lastPhysicalRecord.Lsn.LSN + " PSN: " + lastPhysicalRecord.Psn.PSN + " Position: "
+ lastPhysicalRecord.RecordPosition;
FabricEvents.Events.Api(tracer.Type, trace);
if (lastPhysicalRecord.RecordType == LogRecordType.Information)
{
var removingStateRecord = lastPhysicalRecord as InformationLogRecord;
if (removingStateRecord.InformationEvent == InformationEvent.RemovingState)
{
Utility.Assert(
lastPhysicalRecord == tailRecordAtStart,
"Last Physical Record {0} should be same as tail record at start {1}",
lastPhysicalRecord, tailRecordAtStart);
FabricEvents.Events.Api(tracer.Type, "OpenAsync: Skipping Recovery due to pending RemovingState");
this.IsRemovingStateAfterOpen = true;
return(new RecoveryInformation(shouldLocalStateBeRemoved));
}
}
if (lastPhysicalRecord.RecordType == LogRecordType.TruncateHead)
{
recoveredLastTruncateHeadLogRecord = lastPhysicalRecord as TruncateHeadLogRecord;
}
else if (lastPhysicalRecord.RecordType == LogRecordType.EndCheckpoint)
{
this.RecoveredLastEndCheckpointRecord = lastPhysicalRecord as EndCheckpointLogRecord;
}
else if (lastPhysicalRecord.RecordType == LogRecordType.CompleteCheckpoint)
{
recoveredCompleteCheckpointRecord = lastPhysicalRecord as CompleteCheckpointLogRecord;
}
else
{
recoveredLinkedPhysicalRecord = await this.RecoveryLogsReader.GetLinkedPhysicalRecord(lastPhysicalRecord).ConfigureAwait(false);
Utility.Assert(
(recoveredLinkedPhysicalRecord != null) &&
((recoveredLinkedPhysicalRecord.RecordType == LogRecordType.TruncateHead) ||
(recoveredLinkedPhysicalRecord.RecordType == LogRecordType.EndCheckpoint) ||
(recoveredLinkedPhysicalRecord.RecordType == LogRecordType.CompleteCheckpoint)),
"Record type should be truncate head or end checkpoint or complete checkpoint. Record type is : {0}",
recoveredLinkedPhysicalRecord.RecordType);
trace = "OpenAsync: RecoveredLinkedPhysicalRecord: " + recoveredLinkedPhysicalRecord.RecordType
+ " LSN: " + recoveredLinkedPhysicalRecord.Lsn.LSN + " PSN: "
+ recoveredLinkedPhysicalRecord.Psn.PSN + " Position: "
+ recoveredLinkedPhysicalRecord.RecordPosition;
FabricEvents.Events.Api(tracer.Type, trace);
if (recoveredLinkedPhysicalRecord.RecordType == LogRecordType.TruncateHead)
{
recoveredLastTruncateHeadLogRecord = recoveredLinkedPhysicalRecord as TruncateHeadLogRecord;
}
else if (recoveredLinkedPhysicalRecord.RecordType == LogRecordType.EndCheckpoint)
{
this.RecoveredLastEndCheckpointRecord = recoveredLinkedPhysicalRecord as EndCheckpointLogRecord;
}
else
{
recoveredCompleteCheckpointRecord = recoveredLinkedPhysicalRecord as CompleteCheckpointLogRecord;
}
}
ulong logHeadPosition = 0;
long logHeadLsn = 0;
if (recoveredLastTruncateHeadLogRecord != null)
{
logHeadPosition = recoveredLastTruncateHeadLogRecord.LogHeadRecordPosition;
logHeadLsn = recoveredLastTruncateHeadLogRecord.LogHeadLsn.LSN;
trace = "OpenAsync: Recovered last truncate head record Type: "
+ recoveredLastTruncateHeadLogRecord.RecordType + " LSN: "
+ recoveredLastTruncateHeadLogRecord.Lsn.LSN + " PSN: "
+ recoveredLastTruncateHeadLogRecord.Psn.PSN + " Position: "
+ recoveredLastTruncateHeadLogRecord.RecordPosition + " LogHeadLSN: "
+ recoveredLastTruncateHeadLogRecord.LogHeadLsn.LSN + " LogHeadPosition: "
+ recoveredLastTruncateHeadLogRecord.LogHeadRecordPosition;
FabricEvents.Events.Api(tracer.Type, trace);
recoveredLinkedPhysicalRecord = recoveredLastTruncateHeadLogRecord;
do
{
recoveredLinkedPhysicalRecord = await this.RecoveryLogsReader.GetLinkedPhysicalRecord(recoveredLinkedPhysicalRecord).ConfigureAwait(false);
Utility.Assert(
(recoveredLinkedPhysicalRecord != null) &&
((recoveredLinkedPhysicalRecord.RecordType == LogRecordType.TruncateHead) ||
(recoveredLinkedPhysicalRecord.RecordType == LogRecordType.EndCheckpoint) ||
(recoveredLinkedPhysicalRecord.RecordType == LogRecordType.CompleteCheckpoint)),
"Record type should be truncate head or end checkpoint or complete checkpoint. Record type is : {0}",
recoveredLinkedPhysicalRecord.RecordType);
this.RecoveredLastEndCheckpointRecord = recoveredLinkedPhysicalRecord as EndCheckpointLogRecord;
if (recoveredLinkedPhysicalRecord.RecordType == LogRecordType.CompleteCheckpoint)
{
recoveredCompleteCheckpointRecord = recoveredLinkedPhysicalRecord as CompleteCheckpointLogRecord;
}
} while (this.RecoveredLastEndCheckpointRecord == null);
}
else if (recoveredCompleteCheckpointRecord != null)
{
logHeadPosition = recoveredCompleteCheckpointRecord.LogHeadRecordPosition;
logHeadLsn = recoveredCompleteCheckpointRecord.LogHeadLsn.LSN;
trace = "OpenAsync: Recovered last complete checkpoint record Type: "
+ recoveredCompleteCheckpointRecord.RecordType + " LSN: "
+ recoveredCompleteCheckpointRecord.Lsn.LSN + " PSN: "
+ recoveredCompleteCheckpointRecord.Psn.PSN + " Position: "
+ recoveredCompleteCheckpointRecord.RecordPosition + " LogHeadLsn: "
+ recoveredCompleteCheckpointRecord.LogHeadLsn.LSN + " LogHeadPosition: "
+ recoveredCompleteCheckpointRecord.LogHeadRecordPosition;
FabricEvents.Events.Api(tracer.Type, trace);
recoveredLinkedPhysicalRecord = recoveredCompleteCheckpointRecord;
do
{
recoveredLinkedPhysicalRecord = await this.RecoveryLogsReader.GetLinkedPhysicalRecord(recoveredLinkedPhysicalRecord).ConfigureAwait(false);
Utility.Assert(
(recoveredLinkedPhysicalRecord != null) &&
((recoveredLinkedPhysicalRecord.RecordType == LogRecordType.TruncateHead) ||
(recoveredLinkedPhysicalRecord.RecordType == LogRecordType.EndCheckpoint) ||
(recoveredLinkedPhysicalRecord.RecordType == LogRecordType.CompleteCheckpoint)),
"Record type should be truncate head or end checkpoint or complete checkpoint. Record type is : {0}",
recoveredLinkedPhysicalRecord.RecordType);
this.RecoveredLastEndCheckpointRecord = recoveredLinkedPhysicalRecord as EndCheckpointLogRecord;
} while (this.RecoveredLastEndCheckpointRecord == null);
}
else
{
logHeadPosition = this.RecoveredLastEndCheckpointRecord.LogHeadRecordPosition;
logHeadLsn = this.RecoveredLastEndCheckpointRecord.LogHeadLsn.LSN;
}
trace = "OpenAsync: Recovered last end checkpoint record Type: "
+ this.RecoveredLastEndCheckpointRecord.RecordType + " LSN: " + this.RecoveredLastEndCheckpointRecord.Lsn.LSN
+ " PSN: " + this.RecoveredLastEndCheckpointRecord.Psn.PSN + " Position: "
+ this.RecoveredLastEndCheckpointRecord.RecordPosition + " LogHeadLSN: "
+ this.RecoveredLastEndCheckpointRecord.LogHeadLsn.LSN + " LogHeadPosition: "
+ this.RecoveredLastEndCheckpointRecord.LogHeadRecordPosition;
FabricEvents.Events.Api(tracer.Type, trace);
if (recoveredCompleteCheckpointRecord != null)
{
this.logCompleteCheckpointAfterRecovery = false;
// This is fine since both the begin and end records guarantee all the SP's have been renamed
trace = "OpenAsync: " + " LogHeadPosition: " + this.RecoveredLastEndCheckpointRecord.LogHeadRecordPosition
+ Environment.NewLine + " LogHeadLSN: " + this.RecoveredLastEndCheckpointRecord.LogHeadLsn.LSN
+ Environment.NewLine + " CompleteCheckpoint record found with LSN: "
+ recoveredCompleteCheckpointRecord.Lsn.LSN + " PSN: "
+ recoveredCompleteCheckpointRecord.Psn.PSN + " Position: "
+ recoveredCompleteCheckpointRecord.RecordPosition;
FabricEvents.Events.Api(tracer.Type, trace);
}
else
{
this.logCompleteCheckpointAfterRecovery = true;
trace = "OpenAsync: " + " LogHeadPosition: " + this.RecoveredLastEndCheckpointRecord.LogHeadRecordPosition
+ Environment.NewLine + " LogHeadLSN: " + this.RecoveredLastEndCheckpointRecord.LogHeadLsn.LSN
+ Environment.NewLine + " CompleteCheckpoint record missing";
FabricEvents.Events.Api(tracer.Type, trace);
}
// For restore cases, logHeadPosition must be 0.
if (isRestoring == true)
{
Utility.Assert(
logHeadPosition == 0,
"Full backup: LogHead Position ({0}) must be 0",
logHeadPosition);
}
this.RecoveryLogsReader.MoveStartingRecordPosition(
logHeadLsn,
logHeadPosition,
"recovery",
LogManager.LogReaderType.Recovery);
logManager.SetLogHeadRecordPosition(logHeadPosition);
return(new RecoveryInformation(
this.logCompleteCheckpointAfterRecovery,
shouldLocalStateBeRemoved));
}
private static LogRecord ReadRecord(BinaryReader br, ulong recordPosition, bool isPhysicalRead)
{
LogRecord record;
var lsn = LogicalSequenceNumber.InvalidLsn.LSN;
LogRecordType recordType;
// Metadata section.
var startingPosition = br.BaseStream.Position;
var sizeOfSection = br.ReadInt32();
var endPosition = startingPosition + sizeOfSection;
// Read Logical Metadata
recordType = (LogRecordType)br.ReadUInt32();
switch (recordType)
{
case LogRecordType.BeginTransaction:
record = new BeginTransactionOperationLogRecord(recordType, recordPosition, lsn);
break;
case LogRecordType.Operation:
record = new OperationLogRecord(recordType, recordPosition, lsn);
break;
case LogRecordType.EndTransaction:
record = new EndTransactionLogRecord(recordType, recordPosition, lsn);
break;
case LogRecordType.Barrier:
record = new BarrierLogRecord(recordType, recordPosition, lsn);
break;
case LogRecordType.UpdateEpoch:
record = new UpdateEpochLogRecord(recordType, recordPosition, lsn);
break;
case LogRecordType.Backup:
record = new BackupLogRecord(recordType, recordPosition, lsn);
break;
case LogRecordType.BeginCheckpoint:
record = new BeginCheckpointLogRecord(recordType, recordPosition, lsn);
break;
case LogRecordType.EndCheckpoint:
record = new EndCheckpointLogRecord(recordType, recordPosition, lsn);
break;
case LogRecordType.Indexing:
record = new IndexingLogRecord(recordType, recordPosition, lsn);
break;
case LogRecordType.TruncateHead:
record = new TruncateHeadLogRecord(recordType, recordPosition, lsn);
break;
case LogRecordType.TruncateTail:
record = new TruncateTailLogRecord(recordType, recordPosition, lsn);
break;
case LogRecordType.Information:
record = new InformationLogRecord(recordType, recordPosition, lsn);
break;
case LogRecordType.CompleteCheckpoint:
record = new CompleteCheckpointLogRecord(recordType, recordPosition, lsn);
break;
default:
Utility.CodingError("Unexpected record type {0}", recordType);
return(null);
}
record.lsn = new LogicalSequenceNumber(br.ReadInt64());
// Jump to the end of the section ignoring fields that are not understood.
Utility.Assert(endPosition >= br.BaseStream.Position, "Could not have read more than section size.");
br.BaseStream.Position = endPosition;
record.Read(br, isPhysicalRead);
return(record);
}
public PendingLogHeadTruncationContext(TaskCompletionSource <object> tcs, TruncateHeadLogRecord newHeadRecord)
{
this.Tcs = tcs;
this.truncateHeadLogRecord = newHeadRecord;
}
