public CopyContextParameters(ProgressVector pv, Epoch logHeadEpoch, LogicalSequenceNumber logHeadLsn, LogicalSequenceNumber logTailLsn)
{
this.ProgressVector = pv;
this.LogHeadEpoch = logHeadEpoch;
this.LogHeadLsn = logHeadLsn;
this.LogTailLsn = logTailLsn;
}
internal BeginCheckpointLogRecord(LogRecordType recordType, ulong recordPosition, long lsn)
: base(recordType, recordPosition, lsn)
{
Utility.Assert(
recordType == LogRecordType.BeginCheckpoint,
"Record type is expected to be begin checkpoint but the record type is : {0}",
recordType);
this.IsFirstCheckpointOnFullCopy = false;
this.progressVector = null;
this.earliestPendingTransactionOffset = LogicalLogRecord.InvalidLogicalRecordOffset;
this.earliestPendingTransaction = BeginTransactionOperationLogRecord.InvalidBeginTransactionLogRecord;
this.checkpointState = CheckpointState.Invalid;
this.lastStableLsn = LogicalSequenceNumber.InvalidLsn;
this.epoch = LogicalSequenceNumber.InvalidEpoch;
// Initializes the backup information to invalid.
this.backupId = BackupLogRecord.InvalidBackupId;
this.highestBackedUpEpoch = new Epoch(
LogicalSequenceNumber.InvalidLsn.LSN,
LogicalSequenceNumber.InvalidLsn.LSN);
this.highestBackedUpLsn = LogicalSequenceNumber.InvalidLsn;
// Uint.MaxValue is used to indicate invalid. 4,294,967,295 log records, 4.294967295 TB.
this.backupLogRecordCount = uint.MaxValue;
this.backupLogSize = uint.MaxValue;
this.earliestPendingTransactionInvalidated = 0;
this.lastPeriodicCheckpointTimeTicks = 0;
this.lastPeriodicTruncationTimeTicks = 0;
}
/// <summary>
/// Initializes a new instance of the BeginCheckpointLogRecord class.
/// </summary>
/// <remarks>Only used for generating invalid BeginCheckpointLogRecord.</remarks>
private BeginCheckpointLogRecord()
{
this.IsFirstCheckpointOnFullCopy = false;
this.progressVector = null;
this.earliestPendingTransactionOffset = LogicalLogRecord.InvalidLogicalRecordOffset;
this.earliestPendingTransaction = BeginTransactionOperationLogRecord.InvalidBeginTransactionLogRecord;
this.checkpointState = CheckpointState.Invalid;
this.lastStableLsn = LogicalSequenceNumber.InvalidLsn;
this.epoch = LogicalSequenceNumber.InvalidEpoch;
// Initializes the backup information to invalid.
this.highestBackedUpEpoch = new Epoch(
LogicalSequenceNumber.InvalidLsn.LSN,
LogicalSequenceNumber.InvalidLsn.LSN);
this.highestBackedUpLsn = LogicalSequenceNumber.InvalidLsn;
// Uint.MaxValue is used to indicate invalid. 4,294,967,295 log records, 4.294967295 TB.
this.backupLogRecordCount = uint.MaxValue;
this.backupLogSize = uint.MaxValue;
this.earliestPendingTransactionInvalidated = 0;
this.lastPeriodicCheckpointTimeTicks = 0;
this.lastPeriodicTruncationTimeTicks = 0;
}
internal LoggingReplicatorCopyStream(
ReplicatedLogManager replicatedLogManager,
IStateManager stateManager,
CheckpointManager checkpointManager,
Func <LogicalSequenceNumber, Task> waitForLogFlushUptoLsn,
long replicaId,
LogicalSequenceNumber uptoLsn,
IOperationDataStream copyContext,
ITracer tracer)
{
this.stateManager = stateManager;
this.checkpointManager = checkpointManager;
this.replicatedLogManager = replicatedLogManager;
this.replicaId = replicaId;
this.waitForLogFlushUptoLsn = waitForLogFlushUptoLsn;
this.uptoLsn = uptoLsn;
this.copyContext = copyContext;
this.targetReplicaId = 0;
this.targetProgressVector = null;
this.targetLogHeadEpoch = LogicalSequenceNumber.InvalidEpoch;
this.targetLogHeadLsn = LogicalSequenceNumber.InvalidLsn;
this.currentTargetLsn = LogicalSequenceNumber.InvalidLsn;
this.copyStage = CopyStage.CopyMetadata;
this.copyStateStream = null;
this.copiedRecordNumber = 0;
this.sourceStartingLsn = LogicalSequenceNumber.InvalidLsn;
this.targetStartingLsn = LogicalSequenceNumber.InvalidLsn;
this.logRecordsToCopy = null;
this.beginCheckpointRecord = null;
this.bw = new BinaryWriter(new MemoryStream());
this.isDisposed = false;
this.tracer = tracer;
}
/// <summary>
/// The read from operation data.
/// </summary>
/// <param name="operationData">
/// The operation data.
/// </param>
/// <returns>
/// The <see cref="CopyMetadata"/>.
/// </returns>
public static CopyMetadata ReadFromOperationData(OperationData operationData)
{
var copiedProgressVector = new ProgressVector();
using (var br = new BinaryReader(new MemoryStream(operationData[0].Array, operationData[0].Offset, operationData[0].Count)))
{
var copyStateMetadataVersion = br.ReadInt32();
copiedProgressVector.Read(br, false);
var startingEpoch = new Epoch(br.ReadInt64(), br.ReadInt64());
var startingLsn = new LogicalSequenceNumber(br.ReadInt64());
var checkpointLsn = new LogicalSequenceNumber(br.ReadInt64());
var uptoLsn = new LogicalSequenceNumber(br.ReadInt64());
var highestStateProviderCopiedLsn = new LogicalSequenceNumber(br.ReadInt64());
// Note that if version is 1, then the size must be exactly as expected.
// Else the rest of the data is expected to be not required.
Utility.Assert(
copyStateMetadataVersion != 1 || operationData[0].Array.Length == br.BaseStream.Position,
"Unexpected copy state metadata size. Version {0} logMetadata.Length {1} Position {2}",
copyStateMetadataVersion, operationData[0].Array.Length, br.BaseStream.Position);
return(new CopyMetadata(
copyStateMetadataVersion,
copiedProgressVector,
startingEpoch,
startingLsn,
checkpointLsn,
uptoLsn,
highestStateProviderCopiedLsn));
}
}
internal static ProgressVector Clone(ProgressVector originalProgressVector, uint progressVectorMaxEntries, Epoch highestBackedUpEpoch, Epoch headEpoch)
{
//try trimming the progress vector before cloning
originalProgressVector.TrimProgressVectorIfNeeded(highestBackedUpEpoch, headEpoch);
var copiedProgressVector = new ProgressVector();
copiedProgressVector.progressVectorMaxEntries = progressVectorMaxEntries;
foreach (var vector in originalProgressVector.vectors)
{
copiedProgressVector.vectors.Add(vector);
}
return(copiedProgressVector);
}
/// <summary>
/// Initializes a new instance of the <see cref="CopyMetadata"/> class.
/// </summary>
/// <param name="copyStateMetadataVersion">
/// The copy state metadata version.
/// </param>
/// <param name="progressVector">
/// The progress vector.
/// </param>
/// <param name="startingEpoch">
/// The starting epoch.
/// </param>
/// <param name="startingLogicalSequenceNumber">
/// The starting logical sequence number.
/// </param>
/// <param name="checkpointLsn">
/// The checkpoint lsn.
/// </param>
/// <param name="uptoLsn">
/// The upto lsn.
/// </param>
/// <param name="highestStateProviderCopiedLsn">
/// The highest state provider copied lsn.
/// </param>
public CopyMetadata(
int copyStateMetadataVersion,
ProgressVector progressVector,
Epoch startingEpoch,
LogicalSequenceNumber startingLogicalSequenceNumber,
LogicalSequenceNumber checkpointLsn,
LogicalSequenceNumber uptoLsn,
LogicalSequenceNumber highestStateProviderCopiedLsn)
{
this.copyStateMetadataVersion = copyStateMetadataVersion;
this.progressVector = progressVector;
this.startingEpoch = startingEpoch;
this.startingLogicalSequenceNumber = startingLogicalSequenceNumber;
this.checkpointLsn = checkpointLsn;
this.uptoLsn = uptoLsn;
this.highestStateProviderCopiedLsn = highestStateProviderCopiedLsn;
}
private static bool DecrementIndexUntilLeq(ref int index, ProgressVector vector, ProgressVectorEntry comparand)
{
do
{
if (vector.vectors[index] <= comparand)
{
break;
}
// this can happen when the progress vector is trimmed and does not contain the index relative to the comparand
if (index == 0)
{
return(false);
}
--index;
} while (true);
return(true);
}
public void Reuse(
ProgressVector progressVector,
EndCheckpointLogRecord lastCompletedEndCheckpointRecord,
BeginCheckpointLogRecord lastInProgressBeginCheckpointRecord,
PhysicalLogRecord lastLinkedPhysicalRecord,
InformationLogRecord lastInformationRecord,
IndexingLogRecord currentLogHeadRecord,
Epoch tailEpoch,
LogicalSequenceNumber tailLsn)
{
this.LastInProgressCheckpointRecord = lastInProgressBeginCheckpointRecord;
Utility.Assert(this.LastInProgressCheckpointRecord == null, "ReInitialize of ReplicatedLogManager must have null in progress checkpoint");
this.LastInProgressTruncateHeadRecord = null;
this.ProgressVector = progressVector;
this.LastCompletedEndCheckpointRecord = lastCompletedEndCheckpointRecord;
this.LastLinkedPhysicalRecord = lastLinkedPhysicalRecord;
this.CurrentLogHeadRecord = currentLogHeadRecord;
this.LastInformationRecord = lastInformationRecord;
this.CurrentLogTailEpoch = tailEpoch;
this.CurrentLogTailLsn = tailLsn;
}
/// <summary>
/// Initializes a new instance of the BeginCheckpointLogRecord class.
/// </summary>
/// <param name="dummy">Used to indicate that this is not an Invalid BeginCheckpointLogRecord.</param>
private BeginCheckpointLogRecord(bool dummy)
: base(LogRecordType.BeginCheckpoint, LogicalSequenceNumber.ZeroLsn, null)
{
this.IsFirstCheckpointOnFullCopy = false;
this.progressVector = ProgressVector.Clone(ProgressVector.ZeroProgressVector, 0, LogicalSequenceNumber.ZeroEpoch, LogicalSequenceNumber.ZeroEpoch);
this.earliestPendingTransactionOffset = 0;
this.earliestPendingTransaction = null;
this.checkpointState = CheckpointState.Completed;
this.lastStableLsn = LogicalSequenceNumber.ZeroLsn;
this.epoch = LogicalSequenceNumber.ZeroEpoch;
// Indicates that a full backup has not been taken yet.
this.highestBackedUpEpoch = LogicalSequenceNumber.ZeroEpoch;
this.highestBackedUpLsn = LogicalSequenceNumber.ZeroLsn;
// Indicates that the current backup stream has zero logs and hence 0 KB size.
this.backupLogRecordCount = (uint)0;
this.backupLogSize = (uint)0;
this.earliestPendingTransactionInvalidated = 0;
this.lastPeriodicCheckpointTimeTicks = DateTime.Now.Ticks;
this.lastPeriodicTruncationTimeTicks = this.lastPeriodicCheckpointTimeTicks;
}
public LoggingReplicatorCopyContext(
long replicaId,
ProgressVector progressVector,
IndexingLogRecord logHeadRecord,
LogicalSequenceNumber logTailLsn,
long latestrecoveredAtomicRedoOperationLsn)
{
using (var stream = new MemoryStream())
{
using (var bw = new BinaryWriter(stream))
{
bw.Write(replicaId);
progressVector.Write(bw);
bw.Write(logHeadRecord.CurrentEpoch.DataLossNumber);
bw.Write(logHeadRecord.CurrentEpoch.ConfigurationNumber);
bw.Write(logHeadRecord.Lsn.LSN);
bw.Write(logTailLsn.LSN);
bw.Write(latestrecoveredAtomicRedoOperationLsn);
this.copyData = new OperationData(stream.ToArray());
this.isDone = false;
}
}
}
protected override void Read(BinaryReader br, bool isPhysicalRead)
{
base.Read(br, isPhysicalRead);
var startingPosition = br.BaseStream.Position;
var sizeOfSection = br.ReadInt32();
var endPosition = startingPosition + sizeOfSection;
this.progressVector = new ProgressVector();
this.progressVector.Read(br, isPhysicalRead);
this.earliestPendingTransactionOffset = br.ReadUInt64();
this.epoch = new Epoch(br.ReadInt64(), br.ReadInt64());
// Read the backup information
// Note that if you change this part, backup log record also must be changed.
this.backupId = new Guid(br.ReadBytes(Constants.SizeOfGuidBytes));
this.highestBackedUpEpoch = new Epoch(br.ReadInt64(), br.ReadInt64());
this.highestBackedUpLsn = new LogicalSequenceNumber(br.ReadInt64());
this.backupLogRecordCount = br.ReadUInt32();
this.backupLogSize = br.ReadUInt32();
// Conditionally read periodicCheckpointTimeTicks_
// Ensures compatibility with versions prior to addition of timestamp field
if (br.BaseStream.Position < endPosition)
{
this.lastPeriodicCheckpointTimeTicks = br.ReadInt64();
this.lastPeriodicTruncationTimeTicks = br.ReadInt64();
}
// Jump to the end of the section ignoring fields that are not understood.
Utility.Assert(endPosition >= br.BaseStream.Position, "Could not have read more than section size.");
br.BaseStream.Position = endPosition;
this.UpdateApproximateDiskSize();
}
/// <summary>
/// Initializes a new instance of the LogRecordsMap class for recovery.
/// </summary>
/// <param name="startingLogicalSequenceNumber"></param>
/// <param name="transactionsMap"></param>
/// <param name="currentLogTailEpoch"></param>
/// <param name="lastStableLsn"></param>
/// <param name="progressVector"></param>
/// <param name="recoveredLastCompletedBeginCheckpointRecord"></param>
/// <param name="tracer"></param>
/// <param name="recoveredLastEndCheckpointRecord"></param>
public LogRecordsMap(
LogicalSequenceNumber startingLogicalSequenceNumber,
TransactionMap transactionsMap,
Epoch currentLogTailEpoch,
LogicalSequenceNumber lastStableLsn,
ProgressVector progressVector,
BeginCheckpointLogRecord recoveredLastCompletedBeginCheckpointRecord,
ITracer tracer,
EndCheckpointLogRecord recoveredLastEndCheckpointRecord)
{
this.mode = Mode.Recovery;
this.LastLogicalSequenceNumber = startingLogicalSequenceNumber;
this.TransactionsMap = transactionsMap;
this.CurrentLogTailEpoch = currentLogTailEpoch;
this.LastStableLsn = lastStableLsn;
this.LastPhysicalRecord = PhysicalLogRecord.InvalidPhysicalLogRecord;
this.ProgressVector = progressVector;
this.recoveredLastCompletedBeginCheckpointRecord = recoveredLastCompletedBeginCheckpointRecord;
this.tracer = tracer;
this.recoveredLastEndCheckpointRecord = recoveredLastEndCheckpointRecord;
}
/// <summary>
/// Initializes a new instance of the BeginCheckpointLogRecord class.
/// </summary>
/// <remarks>Called when the replicator decides to checkpoint.</remarks>
internal BeginCheckpointLogRecord(
bool isFirstCheckpointOnFullCopy,
ProgressVector progressVector,
BeginTransactionOperationLogRecord earliestPendingTransaction,
Epoch headEpoch,
Epoch epoch,
LogicalSequenceNumber lsn,
PhysicalLogRecord lastLinkedPhysicalRecord,
BackupLogRecord lastCompletedBackupLogRecord,
uint progressVectorMaxEntries,
long periodicCheckpointTimeTicks,
long periodicTruncationTimeTicks)
: base(LogRecordType.BeginCheckpoint, lsn, lastLinkedPhysicalRecord)
{
this.IsFirstCheckpointOnFullCopy = isFirstCheckpointOnFullCopy;
this.progressVector = ProgressVector.Clone(progressVector, progressVectorMaxEntries, lastCompletedBackupLogRecord.HighestBackedUpEpoch, headEpoch);
this.earliestPendingTransactionOffset = LogicalLogRecord.InvalidLogicalRecordOffset;
this.earliestPendingTransaction = earliestPendingTransaction;
this.checkpointState = CheckpointState.Invalid;
this.lastStableLsn = LogicalSequenceNumber.InvalidLsn;
this.epoch = (earliestPendingTransaction != null) ? earliestPendingTransaction.RecordEpoch : epoch;
// Initialize backup log record fields.
this.highestBackedUpEpoch = lastCompletedBackupLogRecord.HighestBackedUpEpoch;
this.highestBackedUpLsn = lastCompletedBackupLogRecord.HighestBackedUpLsn;
this.backupLogRecordCount = lastCompletedBackupLogRecord.BackupLogRecordCount;
this.backupLogSize = lastCompletedBackupLogRecord.BackupLogSizeInKB;
this.earliestPendingTransactionInvalidated = 0;
this.lastPeriodicCheckpointTimeTicks = periodicCheckpointTimeTicks;
this.lastPeriodicTruncationTimeTicks = periodicTruncationTimeTicks;
this.UpdateApproximateDiskSize();
}
private async Task <OperationData> GetNextAsyncSafe(CancellationToken cancellationToken)
{
OperationData data;
if (this.copyStage == CopyStage.CopyMetadata)
{
data = await this.copyContext.GetNextAsync(CancellationToken.None).ConfigureAwait(false);
Utility.Assert(data.Count == 1, "data.Count == 1");
var bytes = data[0].Array;
using (var br = new BinaryReader(new MemoryStream(bytes, data[0].Offset, data[0].Count)))
{
this.targetReplicaId = br.ReadInt64();
this.targetProgressVector = new ProgressVector();
this.targetProgressVector.Read(br, false);
var targetLogHeadDatalossNumber = br.ReadInt64();
var targetLogHeadConfigurationNumber = br.ReadInt64();
this.targetLogHeadEpoch = new Epoch(targetLogHeadDatalossNumber, targetLogHeadConfigurationNumber);
this.targetLogHeadLsn = new LogicalSequenceNumber(br.ReadInt64());
this.currentTargetLsn = new LogicalSequenceNumber(br.ReadInt64());
this.latestRecoveredAtomicRedoOperationLsn = br.ReadInt64();
}
var message =
string.Format(
CultureInfo.InvariantCulture,
"Target Replica Copy Context: {0}" + Environment.NewLine
+ " TargetLogHeadEpoch: {1},{2} TargetLogHeadLsn: {3} TargetLogTailEpoch: {4},{5} TargetLogTailLSN: {6}"
+ Environment.NewLine
+ " SourceLogHeadEpoch: {7},{8} SourceLogHeadLsn: {9} SourceLogTailEpoch: {10},{11} SourceLogTailLSN: {12}"
+ Environment.NewLine + " Target ProgressVector: {13}" + Environment.NewLine
+ " Source ProgressVector: {14}" + Environment.NewLine
+ " LastRecoveredAtomicRedoOperationLsn: {15}",
this.targetReplicaId,
this.targetLogHeadEpoch.DataLossNumber,
this.targetLogHeadEpoch.ConfigurationNumber,
this.targetLogHeadLsn.LSN,
this.targetProgressVector.LastProgressVectorEntry.Epoch.DataLossNumber,
this.targetProgressVector.LastProgressVectorEntry.Epoch.ConfigurationNumber,
this.currentTargetLsn.LSN,
this.replicatedLogManager.CurrentLogHeadRecord.CurrentEpoch.DataLossNumber,
this.replicatedLogManager.CurrentLogHeadRecord.CurrentEpoch.ConfigurationNumber,
this.replicatedLogManager.CurrentLogHeadRecord.Lsn.LSN,
this.replicatedLogManager.CurrentLogTailEpoch.DataLossNumber,
this.replicatedLogManager.CurrentLogTailEpoch.ConfigurationNumber,
this.replicatedLogManager.CurrentLogTailLsn.LSN,
this.targetProgressVector.ToString(Constants.ProgressVectorMaxStringSizeInKb / 2),
this.replicatedLogManager.ProgressVector.ToString(Constants.ProgressVectorMaxStringSizeInKb / 2),
this.latestRecoveredAtomicRedoOperationLsn);
FabricEvents.Events.CopyStreamMetadata(tracer.Type, message);
await this.waitForLogFlushUptoLsn(this.uptoLsn).ConfigureAwait(false);
CopyMode copyMode;
await this.checkpointManager.AcquireBackupAndCopyConsistencyLockAsync("Copy").ConfigureAwait(false);
try
{
copyMode = this.checkpointManager.GetLogRecordsToCopy(
this.targetProgressVector,
this.targetLogHeadEpoch,
this.targetLogHeadLsn,
this.currentTargetLsn,
this.latestRecoveredAtomicRedoOperationLsn,
this.targetReplicaId,
out this.sourceStartingLsn,
out this.targetStartingLsn,
out this.logRecordsToCopy,
out this.beginCheckpointRecord);
}
finally
{
this.checkpointManager.ReleaseBackupAndCopyConsistencyLock("Copy");
}
CopyStage copyStageToWrite;
if (copyMode == CopyMode.None)
{
this.copyStage = CopyStage.CopyNone;
copyStageToWrite = CopyStage.CopyNone;
}
else if (copyMode == CopyMode.Full)
{
this.copyStage = CopyStage.CopyState;
copyStageToWrite = CopyStage.CopyState;
this.copyStateStream = this.stateManager.GetCurrentState();
}
else if ((copyMode & CopyMode.FalseProgress) != 0)
{
Utility.Assert(
this.sourceStartingLsn <= this.targetStartingLsn,
"this.sourceStartingLsn ({0}) <= this.targetStartingLsn ({1})",
this.sourceStartingLsn, this.targetStartingLsn);
this.copyStage = CopyStage.CopyFalseProgress;
copyStageToWrite = CopyStage.CopyFalseProgress;
}
else
{
this.copyStage = CopyStage.CopyScanToStartingLsn;
copyStageToWrite = CopyStage.CopyLog;
}
return(this.GetCopyMetadata(copyStageToWrite));
}
if (this.copyStage == CopyStage.CopyState)
{
data = await this.copyStateStream.GetNextAsync(cancellationToken).ConfigureAwait(false);
if (data != null)
{
data.Add(CopyStateOperation);
FabricEvents.Events.CopyStreamGetNextNoise(
tracer.Type,
"Copying State Operation: " + this.copiedRecordNumber);
++this.copiedRecordNumber;
return(data);
}
var disposableCopyStateStream = this.copyStateStream as IDisposable;
if (disposableCopyStateStream != null)
{
disposableCopyStateStream.Dispose();
}
if (this.uptoLsn < beginCheckpointRecord.LastStableLsn)
{
// Ensure we copy records up to the stable LSN of the begin checkpoint record as part of the copy stream
// so that the idle can initiate a checkpoint and apply it as part of the copy pump itself
// Not doing the above will mean that the idle can get promoted to active even before the checkpoint is completed
// uptolsn is inclusive LSN
// BUG: RDBug #9269022:Replicator must rename copy log before changing role to active secondary during full builds
this.uptoLsn = beginCheckpointRecord.LastStableLsn;
var trace = "FullCopy: Copying Logs from " + this.sourceStartingLsn.LSN
+ " to BC.LastStableLsn: " + beginCheckpointRecord.LastStableLsn.LSN + " as the upto lsn: " + this.uptoLsn + " was smaller than the checkpoint stable lsn";
FabricEvents.Events.CopyStreamGetNext(tracer.Type, trace);
}
data = this.GetCopyStateMetadata();
this.copyStage = CopyStage.CopyLog;
this.copiedRecordNumber = 0;
return(data);
}
if (this.copyStage == CopyStage.CopyFalseProgress)
{
this.copyStage = CopyStage.CopyScanToStartingLsn;
this.bw.BaseStream.Position = 0;
this.bw.Write(this.sourceStartingLsn.LSN);
var stream = this.bw.BaseStream as MemoryStream;
data = new OperationData(new ArraySegment <byte>(stream.GetBuffer(), 0, (int)stream.Position));
data.Add(CopyFalseProgressOperation);
var trace = "Copying False Progress. SourceStartingLSN: " + this.sourceStartingLsn.LSN
+ " TargetStartingLSN: " + this.targetStartingLsn.LSN;
FabricEvents.Events.CopyStreamGetNext(tracer.Type, trace);
return(data);
}
LogRecord record;
if (this.copyStage == CopyStage.CopyScanToStartingLsn)
{
var startingLsn = (this.sourceStartingLsn < this.targetStartingLsn)
? this.sourceStartingLsn
: this.targetStartingLsn;
do
{
var hasMoved = await this.logRecordsToCopy.MoveNextAsync(cancellationToken).ConfigureAwait(false);
if (hasMoved == false)
{
goto Finish;
}
record = this.logRecordsToCopy.Current;
Utility.Assert(record.Lsn <= startingLsn, "record.Lsn <= startingLsn");
} while (record.Lsn < startingLsn);
// The log stream is positioned at the end of startingLsn at this point. The enumerator Current is pointing to the "startingLsn"
this.copyStage = CopyStage.CopyLog;
}
if (this.copyStage == CopyStage.CopyLog)
{
do
{
// This will start copying after the startingLsn record as expected
var hasMoved = await this.logRecordsToCopy.MoveNextAsync(cancellationToken).ConfigureAwait(false);
if (hasMoved == false)
{
goto Finish;
}
record = this.logRecordsToCopy.Current;
} while (record is PhysicalLogRecord);
if (record.Lsn > this.uptoLsn)
{
goto Finish;
}
var logicalLogRecord = record as LogicalLogRecord;
Utility.Assert(logicalLogRecord != null, "Must be a logical log record");
data = logicalLogRecord.ToOperationData(this.bw);
data.Add(CopyLogOperation);
var trace = "Copying log record. LogRecordNumber: " + this.copiedRecordNumber + "Record Type: "
+ record.RecordType + " LSN: " + record.Lsn.LSN;
FabricEvents.Events.CopyStreamGetNextNoise(tracer.Type, trace);
++this.copiedRecordNumber;
return(data);
}
Finish:
if (this.logRecordsToCopy != null)
{
this.logRecordsToCopy.Dispose();
}
FabricEvents.Events.CopyStreamFinished(tracer.Type, this.copiedRecordNumber);
return(null);
}
internal static SharedProgressVectorEntry FindSharedVector(
ProgressVector sourceProgressVector,
ProgressVector targetProgressVector)
{
// Initialize indices
var sourceIndex = sourceProgressVector.vectors.Count - 1;
var targetIndex = targetProgressVector.vectors.Count - 1;
Utility.Assert(
targetProgressVector.vectors[targetIndex].Epoch.DataLossNumber <= sourceProgressVector.vectors[sourceIndex].Epoch.DataLossNumber,
"targetDataLossNumber ({0}) <= sourceDataLossNumber ({1})",
targetProgressVector.vectors[targetIndex].Epoch.DataLossNumber, sourceProgressVector.vectors[sourceIndex].Epoch.DataLossNumber);
SharedProgressVectorEntry progrssVectorTrimmedResult = new SharedProgressVectorEntry()
{
FullCopyReason = FullCopyReason.ProgressVectorTrimmed
};
do
{
bool decrementIndexSuccess = DecrementIndexUntilLeq(
ref targetIndex,
targetProgressVector,
sourceProgressVector.vectors[sourceIndex]);
if (!decrementIndexSuccess)
{
return(progrssVectorTrimmedResult);
}
decrementIndexSuccess = DecrementIndexUntilLeq(
ref sourceIndex,
sourceProgressVector,
targetProgressVector.vectors[targetIndex]);
if (!decrementIndexSuccess)
{
return(progrssVectorTrimmedResult);
}
} while (sourceProgressVector.vectors[sourceIndex] != targetProgressVector.vectors[targetIndex]);
var sourceProgressVectorEntry = sourceProgressVector.vectors[sourceIndex];
var targetProgressVectorEntry = targetProgressVector.vectors[targetIndex];
// WITHOUT MINI RECONFIG ENABLED, it is not possible for the target to make VALID progress TWICE from the shared epoch
// Note that this check is only valid in the absence of dataloss and restore.
// E.g
/*
* Target Replica Copy Context: 131383889527030885
* TargetLogHeadEpoch: 0,0 TargetLogHeadLsn: 0 TargetLogTailEpoch: 131383889406148342,64424509440 TargetLogTailLSN: 73
* SourceLogHeadEpoch: 0,0 SourceLogHeadLsn: 0 SourceLogTailEpoch: 131383889406148342,68719476736 SourceLogTailLSN: 68
* Target ProgressVector: [(131383889406148342,64424509440),73,131383889527030885,2017-05-04T16:32:22]
* [(131383889406148342,55834574848),68,131383889527030885,2017-05-04T16:31:54]
* [(131383889406148342,51539607552),62,131383889416461059,2017-05-04T16:31:37]
* [(131383889406148342,47244640256),56,131383889527030885,2017-05-04T16:31:20]
* [(0,0),0,0,2017-05-04T16:29:07]
* Source ProgressVector: [(131383889406148342,68719476736),68,131383890970495929,2017-05-04T16:32:33]
* [(131383889406148342,60129542144),68,131383890970495929,2017-05-04T16:32:09]
* [(131383889406148342,51539607552),62,131383889416461059,2017-05-04T16:31:37]
* [(131383889406148342,47244640256),56,131383889527030885,2017-05-04T16:31:20]
* [(0,0),0,0,2017-05-04T16:31:33]
* LastRecoveredAtomicRedoOperationLsn: 0.
*
*
* Target Replica Copy Context: 131399770096882719
* TargetLogHeadEpoch: 0,0 TargetLogHeadLsn: 0 TargetLogTailEpoch: 131399758616683034,390842023936 TargetLogTailLSN: 374
* SourceLogHeadEpoch: 0,0 SourceLogHeadLsn: 0 SourceLogTailEpoch: 131399758616683034,395136991232 SourceLogTailLSN: 369
* Target ProgressVector:
* [(131399758616683034,390842023936),374,131399770096882719,2017-05-23T01:42:03]
* [(131399758616683034,382252089344),374,131399770096882719,2017-05-23T01:41:29]
* [(131399758616683034,377957122048),369,131399770096882719,2017-05-23T01:41:00]
* [(131399758616683034,373662154752),338,131399758632647497,2017-05-23T01:40:31]
* Source ProgressVector:
* [(131399758616683034,395136991232),369,131399758701866693,2017-05-23T01:42:19]
* [(131399758616683034,386547056640),369,131399758701866693,2017-05-23T01:41:44]
* [(131399758616683034,373662154752),338,131399758632647497,2017-05-23T01:40:31]
* LastRecoveredAtomicRedoOperationLsn: 0
*/
string failureMsg;
if (targetProgressVector.LastProgressVectorEntry.Epoch.DataLossNumber
== sourceProgressVector.LastProgressVectorEntry.Epoch.DataLossNumber &&
targetIndex < targetProgressVector.vectors.Count - 1 &&
targetProgressVector.LastProgressVectorEntry.Epoch.DataLossNumber == targetProgressVector.vectors[targetIndex].Epoch.DataLossNumber)
{
// Target replica could only have repeatedly attempted to become a
// primary without ever making progress at the shared dataloss number
// We can do "double progress check" only if there has NOT been any other data loss happening on the target after shared point. Having a differnt dataloss numbers in shared vector and target's tail
// means that target has been a valid primary and progressed in LSN until hitting a dataloss and then a very old secondary can become primary and invalidate all the progress made by the target.
var failureLsn = targetProgressVector.vectors[targetIndex + 1].Lsn;
var failureLsnIncremented = 0;
for (var n = targetIndex + 2; n <= targetProgressVector.vectors.Count - 1; n++)
{
var vector = targetProgressVector.vectors[n];
if (vector.Epoch.DataLossNumber != targetProgressVectorEntry.Epoch.DataLossNumber)
{
break;
}
if (vector.Lsn != failureLsn)
{
// Update failureLsn to ensure we don't assert if there are multiple entries for same LSN like in 2nd example above
failureLsn = vector.Lsn;
failureLsnIncremented++;
}
}
failureMsg = string.Format(
"FailureLsn incremented must be <= 1. It is {0}",
failureLsnIncremented);
if (!ValidateIfDebugEnabled(
failureLsnIncremented <= 1,
failureMsg))
{
return(new SharedProgressVectorEntry
{
SourceIndex = sourceIndex,
TargetIndex = targetIndex,
SourceProgressVectorEntry = sourceProgressVectorEntry,
TargetProgressVectorEntry = targetProgressVectorEntry,
FullCopyReason = FullCopyReason.ValidationFailed,
FailedValidationMessage = failureMsg
});
}
}
// Sanity check that source and target progress vectors are always in
// agreement upto above determined ProgressVectorEntry shared by both of them
var i = sourceIndex;
var sourceEntry = sourceProgressVectorEntry;
var j = targetIndex;
var targetEntry = targetProgressVectorEntry;
failureMsg = "sourceEntry == targetEntry";
do
{
sourceEntry = DecrementIndexUntilLsnIsEqual(ref i, sourceProgressVector, sourceEntry.Lsn.LSN);
if (i == 0)
{
break;
}
targetEntry = DecrementIndexUntilLsnIsEqual(ref j, targetProgressVector, targetEntry.Lsn.LSN);
if (j == 0)
{
break;
}
if (!ValidateIfDebugEnabled(
sourceEntry == targetEntry,
failureMsg))
{
return(new SharedProgressVectorEntry
{
SourceIndex = sourceIndex,
TargetIndex = targetIndex,
SourceProgressVectorEntry = sourceProgressVectorEntry,
TargetProgressVectorEntry = targetProgressVectorEntry,
FullCopyReason = FullCopyReason.ValidationFailed,
FailedValidationMessage = failureMsg
});
}
} while (true);
Utility.Assert(sourceProgressVectorEntry != null && targetProgressVectorEntry != null, "Source and target expected to be non null");
return(new SharedProgressVectorEntry
{
SourceIndex = sourceIndex,
TargetIndex = targetIndex,
SourceProgressVectorEntry = sourceProgressVectorEntry,
TargetProgressVectorEntry = targetProgressVectorEntry
});
}
private static ProgressVectorEntry DecrementIndexUntilLsnIsEqual(ref int index, ProgressVector vector, long lsn)
{
while (index > 0)
{
var previousSourceVector = vector.vectors[index - 1];
if (previousSourceVector.Lsn.LSN != lsn)
{
return(previousSourceVector);
}
--index;
}
return(null);
}
private static CopyModeResult FindCopyModePrivate(
CopyContextParameters sourceParamaters,
CopyContextParameters targetParameters,
long lastRecoveredAtomicRedoOperationLsnAtTarget)
{
// We can perform the check for CASE 1(Shown Below) before finding the shared ProgressVectorEntry.
// However, we do not do that because the shared ProgressVectorEntry method provides additional checks of epoch history
var sharedProgressVectorEntry = FindSharedVector(
sourceParamaters.ProgressVector,
targetParameters.ProgressVector);
// If we detect that the shared progress vector cannot be obtained because of a trimmed progress vector, we decide to do a full copy
if (sharedProgressVectorEntry.FullCopyReason == FullCopyReason.ProgressVectorTrimmed)
{
return(CopyModeResult.CreateFullCopyResult(null, FullCopyReason.ProgressVectorTrimmed));
}
// Validation failed, perform full copy
if (sharedProgressVectorEntry.FullCopyReason == FullCopyReason.ValidationFailed)
{
return(CopyModeResult.CreateFullCopyResult(sharedProgressVectorEntry, FullCopyReason.ValidationFailed));
}
// *******************************************CASE 1 - COPY_NONE***********************************************************
//
// The last entry in the source and the target vectors are same with the same tail.
// This can happen if target goes down and comes back up and the primary made no progress
if (sourceParamaters.ProgressVector.LastProgressVectorEntry == targetParameters.ProgressVector.LastProgressVectorEntry &&
sourceParamaters.LogTailLsn == targetParameters.LogTailLsn)
{
// Note that copy none covers only cases where nothing needs to be copied (including the UpdateEpochLogRecord)
// Hence, service creation scenario is Partial copy, not None.
return(CopyModeResult.CreateCopyNoneResult(sharedProgressVectorEntry));
}
// *******************************************CASE 2a - COPY_FULL - Due to Data Loss******************************************
//
// Check for any dataloss in the source or target log
// If there was data loss, perform full copy
// No full copy with DataLoss reason, if the target is a brand new replica
if (!ProgressVector.IsBrandNewReplica(targetParameters))
{
if (sharedProgressVectorEntry.SourceProgressVectorEntry.Epoch.DataLossNumber != sourceParamaters.ProgressVector.LastProgressVectorEntry.Epoch.DataLossNumber ||
sharedProgressVectorEntry.TargetProgressVectorEntry.Epoch.DataLossNumber != targetParameters.ProgressVector.LastProgressVectorEntry.Epoch.DataLossNumber ||
sourceParamaters.LogHeadEpoch.DataLossNumber > sharedProgressVectorEntry.TargetProgressVectorEntry.Epoch.DataLossNumber ||
targetParameters.LogHeadEpoch.DataLossNumber > sharedProgressVectorEntry.SourceProgressVectorEntry.Epoch.DataLossNumber)
{
return(CopyModeResult.CreateFullCopyResult(sharedProgressVectorEntry, FullCopyReason.DataLoss));
}
}
var sourceStartingLsn = (sharedProgressVectorEntry.SourceIndex == (sourceParamaters.ProgressVector.vectors.Count - 1))
? sourceParamaters.LogTailLsn
: sourceParamaters.ProgressVector.vectors[sharedProgressVectorEntry.SourceIndex + 1].Lsn;
var targetStartingLsn = (sharedProgressVectorEntry.TargetIndex == (targetParameters.ProgressVector.vectors.Count - 1))
? targetParameters.LogTailLsn
: targetParameters.ProgressVector.vectors[sharedProgressVectorEntry.TargetIndex + 1].Lsn;
// *******************************************CASE 2b - COPY_FULL - Due to Insufficient Loss******************************************
//
// Since there was no data loss, check if we can perform partial copy and there are sufficient logs in the source
// and target to do so
if (sourceParamaters.LogHeadLsn > targetParameters.LogTailLsn ||
sourceStartingLsn < sourceParamaters.LogHeadLsn ||
targetParameters.LogHeadLsn > sourceStartingLsn)
{
return(CopyModeResult.CreateFullCopyResult(sharedProgressVectorEntry, FullCopyReason.InsufficientLogs));
}
// *******************************************CASE 3a - FALSE_PROGRESS -Basic Case******************************************
//
// The simple case where target made more progress in an epoch than the primary
// Source = (1,1,0) (1,3,10) (1,4,15) Tail = 16
// Target = (1,1,0) (1,3,10) Tail = 20
if (sourceStartingLsn < targetStartingLsn)
{
return(CopyModeResult.CreateFalseProgressResult(
lastRecoveredAtomicRedoOperationLsnAtTarget,
sharedProgressVectorEntry,
sourceStartingLsn,
targetStartingLsn));
}
// *******************************************CASE 3b - FALSE_PROGRESS -Advanced Case******************************************
//
// The cases where target has made progress in an epoch that the primary has no knowledge of
// Series of crashes could lead to the target having made false progress in an epoch that the primary DOES NOT KNOW about
// Source = (1,1,0) (1,3,10) Tail = 15
// Target = (1,1,0) (1,2,10) Tail = 12
// Source = (1,1,0) (1,3,10) Tail = 15
// Target = (1,1,0) (1,2,10) Tail = 10
// Source = (1,3,10) (1,5,17) Tail = 18
// Target = (1,3,10) (1,4,15) Tail = 17
if (targetStartingLsn != targetParameters.LogTailLsn)
{
sourceStartingLsn = targetStartingLsn;
return(CopyModeResult.CreateFalseProgressResult(
lastRecoveredAtomicRedoOperationLsnAtTarget,
sharedProgressVectorEntry,
sourceStartingLsn,
targetStartingLsn));
}
// *******************************************CASE 4 - Work Around Case to be able to assert******************************************
// RDBUG : 3374739
// Source = (1,1,0) (1,3,10) Tail >= 10
// Target = (1,1,0) (1,2,5) Tail = 5
// Target got UpdateEpoch(e5) at 5, while source got UpdateEpoch(e6) at 10 (without having an entry for e5)
// Source should detect this and do a full build - (We cannot undo false progress since the process at the target until 5 could be true progress and could be checkpointed)
if (sourceParamaters.ProgressVector.vectors[sharedProgressVectorEntry.SourceIndex].Epoch < targetParameters.ProgressVector.LastProgressVectorEntry.Epoch)
{
return(CopyModeResult.CreateFullCopyResult(sharedProgressVectorEntry, FullCopyReason.Other));
}
return(CopyModeResult.CreatePartialCopyResult(sharedProgressVectorEntry, sourceStartingLsn, targetStartingLsn));
}
