public CopyContextParameters(ProgressVector pv, Epoch logHeadEpoch, LogicalSequenceNumber logHeadLsn, LogicalSequenceNumber logTailLsn) { this.ProgressVector = pv; this.LogHeadEpoch = logHeadEpoch; this.LogHeadLsn = logHeadLsn; this.LogTailLsn = logTailLsn; }
internal BeginCheckpointLogRecord(LogRecordType recordType, ulong recordPosition, long lsn) : base(recordType, recordPosition, lsn) { Utility.Assert( recordType == LogRecordType.BeginCheckpoint, "Record type is expected to be begin checkpoint but the record type is : {0}", recordType); this.IsFirstCheckpointOnFullCopy = false; this.progressVector = null; this.earliestPendingTransactionOffset = LogicalLogRecord.InvalidLogicalRecordOffset; this.earliestPendingTransaction = BeginTransactionOperationLogRecord.InvalidBeginTransactionLogRecord; this.checkpointState = CheckpointState.Invalid; this.lastStableLsn = LogicalSequenceNumber.InvalidLsn; this.epoch = LogicalSequenceNumber.InvalidEpoch; // Initializes the backup information to invalid. this.backupId = BackupLogRecord.InvalidBackupId; this.highestBackedUpEpoch = new Epoch( LogicalSequenceNumber.InvalidLsn.LSN, LogicalSequenceNumber.InvalidLsn.LSN); this.highestBackedUpLsn = LogicalSequenceNumber.InvalidLsn; // Uint.MaxValue is used to indicate invalid. 4,294,967,295 log records, 4.294967295 TB. this.backupLogRecordCount = uint.MaxValue; this.backupLogSize = uint.MaxValue; this.earliestPendingTransactionInvalidated = 0; this.lastPeriodicCheckpointTimeTicks = 0; this.lastPeriodicTruncationTimeTicks = 0; }
/// <summary> /// Initializes a new instance of the BeginCheckpointLogRecord class. /// </summary> /// <remarks>Only used for generating invalid BeginCheckpointLogRecord.</remarks> private BeginCheckpointLogRecord() { this.IsFirstCheckpointOnFullCopy = false; this.progressVector = null; this.earliestPendingTransactionOffset = LogicalLogRecord.InvalidLogicalRecordOffset; this.earliestPendingTransaction = BeginTransactionOperationLogRecord.InvalidBeginTransactionLogRecord; this.checkpointState = CheckpointState.Invalid; this.lastStableLsn = LogicalSequenceNumber.InvalidLsn; this.epoch = LogicalSequenceNumber.InvalidEpoch; // Initializes the backup information to invalid. this.highestBackedUpEpoch = new Epoch( LogicalSequenceNumber.InvalidLsn.LSN, LogicalSequenceNumber.InvalidLsn.LSN); this.highestBackedUpLsn = LogicalSequenceNumber.InvalidLsn; // Uint.MaxValue is used to indicate invalid. 4,294,967,295 log records, 4.294967295 TB. this.backupLogRecordCount = uint.MaxValue; this.backupLogSize = uint.MaxValue; this.earliestPendingTransactionInvalidated = 0; this.lastPeriodicCheckpointTimeTicks = 0; this.lastPeriodicTruncationTimeTicks = 0; }
internal LoggingReplicatorCopyStream( ReplicatedLogManager replicatedLogManager, IStateManager stateManager, CheckpointManager checkpointManager, Func <LogicalSequenceNumber, Task> waitForLogFlushUptoLsn, long replicaId, LogicalSequenceNumber uptoLsn, IOperationDataStream copyContext, ITracer tracer) { this.stateManager = stateManager; this.checkpointManager = checkpointManager; this.replicatedLogManager = replicatedLogManager; this.replicaId = replicaId; this.waitForLogFlushUptoLsn = waitForLogFlushUptoLsn; this.uptoLsn = uptoLsn; this.copyContext = copyContext; this.targetReplicaId = 0; this.targetProgressVector = null; this.targetLogHeadEpoch = LogicalSequenceNumber.InvalidEpoch; this.targetLogHeadLsn = LogicalSequenceNumber.InvalidLsn; this.currentTargetLsn = LogicalSequenceNumber.InvalidLsn; this.copyStage = CopyStage.CopyMetadata; this.copyStateStream = null; this.copiedRecordNumber = 0; this.sourceStartingLsn = LogicalSequenceNumber.InvalidLsn; this.targetStartingLsn = LogicalSequenceNumber.InvalidLsn; this.logRecordsToCopy = null; this.beginCheckpointRecord = null; this.bw = new BinaryWriter(new MemoryStream()); this.isDisposed = false; this.tracer = tracer; }
/// <summary> /// The read from operation data. /// </summary> /// <param name="operationData"> /// The operation data. /// </param> /// <returns> /// The <see cref="CopyMetadata"/>. /// </returns> public static CopyMetadata ReadFromOperationData(OperationData operationData) { var copiedProgressVector = new ProgressVector(); using (var br = new BinaryReader(new MemoryStream(operationData[0].Array, operationData[0].Offset, operationData[0].Count))) { var copyStateMetadataVersion = br.ReadInt32(); copiedProgressVector.Read(br, false); var startingEpoch = new Epoch(br.ReadInt64(), br.ReadInt64()); var startingLsn = new LogicalSequenceNumber(br.ReadInt64()); var checkpointLsn = new LogicalSequenceNumber(br.ReadInt64()); var uptoLsn = new LogicalSequenceNumber(br.ReadInt64()); var highestStateProviderCopiedLsn = new LogicalSequenceNumber(br.ReadInt64()); // Note that if version is 1, then the size must be exactly as expected. // Else the rest of the data is expected to be not required. Utility.Assert( copyStateMetadataVersion != 1 || operationData[0].Array.Length == br.BaseStream.Position, "Unexpected copy state metadata size. Version {0} logMetadata.Length {1} Position {2}", copyStateMetadataVersion, operationData[0].Array.Length, br.BaseStream.Position); return(new CopyMetadata( copyStateMetadataVersion, copiedProgressVector, startingEpoch, startingLsn, checkpointLsn, uptoLsn, highestStateProviderCopiedLsn)); } }
internal static ProgressVector Clone(ProgressVector originalProgressVector, uint progressVectorMaxEntries, Epoch highestBackedUpEpoch, Epoch headEpoch) { //try trimming the progress vector before cloning originalProgressVector.TrimProgressVectorIfNeeded(highestBackedUpEpoch, headEpoch); var copiedProgressVector = new ProgressVector(); copiedProgressVector.progressVectorMaxEntries = progressVectorMaxEntries; foreach (var vector in originalProgressVector.vectors) { copiedProgressVector.vectors.Add(vector); } return(copiedProgressVector); }
/// <summary> /// Initializes a new instance of the <see cref="CopyMetadata"/> class. /// </summary> /// <param name="copyStateMetadataVersion"> /// The copy state metadata version. /// </param> /// <param name="progressVector"> /// The progress vector. /// </param> /// <param name="startingEpoch"> /// The starting epoch. /// </param> /// <param name="startingLogicalSequenceNumber"> /// The starting logical sequence number. /// </param> /// <param name="checkpointLsn"> /// The checkpoint lsn. /// </param> /// <param name="uptoLsn"> /// The upto lsn. /// </param> /// <param name="highestStateProviderCopiedLsn"> /// The highest state provider copied lsn. /// </param> public CopyMetadata( int copyStateMetadataVersion, ProgressVector progressVector, Epoch startingEpoch, LogicalSequenceNumber startingLogicalSequenceNumber, LogicalSequenceNumber checkpointLsn, LogicalSequenceNumber uptoLsn, LogicalSequenceNumber highestStateProviderCopiedLsn) { this.copyStateMetadataVersion = copyStateMetadataVersion; this.progressVector = progressVector; this.startingEpoch = startingEpoch; this.startingLogicalSequenceNumber = startingLogicalSequenceNumber; this.checkpointLsn = checkpointLsn; this.uptoLsn = uptoLsn; this.highestStateProviderCopiedLsn = highestStateProviderCopiedLsn; }
private static bool DecrementIndexUntilLeq(ref int index, ProgressVector vector, ProgressVectorEntry comparand) { do { if (vector.vectors[index] <= comparand) { break; } // this can happen when the progress vector is trimmed and does not contain the index relative to the comparand if (index == 0) { return(false); } --index; } while (true); return(true); }
public void Reuse( ProgressVector progressVector, EndCheckpointLogRecord lastCompletedEndCheckpointRecord, BeginCheckpointLogRecord lastInProgressBeginCheckpointRecord, PhysicalLogRecord lastLinkedPhysicalRecord, InformationLogRecord lastInformationRecord, IndexingLogRecord currentLogHeadRecord, Epoch tailEpoch, LogicalSequenceNumber tailLsn) { this.LastInProgressCheckpointRecord = lastInProgressBeginCheckpointRecord; Utility.Assert(this.LastInProgressCheckpointRecord == null, "ReInitialize of ReplicatedLogManager must have null in progress checkpoint"); this.LastInProgressTruncateHeadRecord = null; this.ProgressVector = progressVector; this.LastCompletedEndCheckpointRecord = lastCompletedEndCheckpointRecord; this.LastLinkedPhysicalRecord = lastLinkedPhysicalRecord; this.CurrentLogHeadRecord = currentLogHeadRecord; this.LastInformationRecord = lastInformationRecord; this.CurrentLogTailEpoch = tailEpoch; this.CurrentLogTailLsn = tailLsn; }
/// <summary> /// Initializes a new instance of the BeginCheckpointLogRecord class. /// </summary> /// <param name="dummy">Used to indicate that this is not an Invalid BeginCheckpointLogRecord.</param> private BeginCheckpointLogRecord(bool dummy) : base(LogRecordType.BeginCheckpoint, LogicalSequenceNumber.ZeroLsn, null) { this.IsFirstCheckpointOnFullCopy = false; this.progressVector = ProgressVector.Clone(ProgressVector.ZeroProgressVector, 0, LogicalSequenceNumber.ZeroEpoch, LogicalSequenceNumber.ZeroEpoch); this.earliestPendingTransactionOffset = 0; this.earliestPendingTransaction = null; this.checkpointState = CheckpointState.Completed; this.lastStableLsn = LogicalSequenceNumber.ZeroLsn; this.epoch = LogicalSequenceNumber.ZeroEpoch; // Indicates that a full backup has not been taken yet. this.highestBackedUpEpoch = LogicalSequenceNumber.ZeroEpoch; this.highestBackedUpLsn = LogicalSequenceNumber.ZeroLsn; // Indicates that the current backup stream has zero logs and hence 0 KB size. this.backupLogRecordCount = (uint)0; this.backupLogSize = (uint)0; this.earliestPendingTransactionInvalidated = 0; this.lastPeriodicCheckpointTimeTicks = DateTime.Now.Ticks; this.lastPeriodicTruncationTimeTicks = this.lastPeriodicCheckpointTimeTicks; }
public LoggingReplicatorCopyContext( long replicaId, ProgressVector progressVector, IndexingLogRecord logHeadRecord, LogicalSequenceNumber logTailLsn, long latestrecoveredAtomicRedoOperationLsn) { using (var stream = new MemoryStream()) { using (var bw = new BinaryWriter(stream)) { bw.Write(replicaId); progressVector.Write(bw); bw.Write(logHeadRecord.CurrentEpoch.DataLossNumber); bw.Write(logHeadRecord.CurrentEpoch.ConfigurationNumber); bw.Write(logHeadRecord.Lsn.LSN); bw.Write(logTailLsn.LSN); bw.Write(latestrecoveredAtomicRedoOperationLsn); this.copyData = new OperationData(stream.ToArray()); this.isDone = false; } } }
protected override void Read(BinaryReader br, bool isPhysicalRead) { base.Read(br, isPhysicalRead); var startingPosition = br.BaseStream.Position; var sizeOfSection = br.ReadInt32(); var endPosition = startingPosition + sizeOfSection; this.progressVector = new ProgressVector(); this.progressVector.Read(br, isPhysicalRead); this.earliestPendingTransactionOffset = br.ReadUInt64(); this.epoch = new Epoch(br.ReadInt64(), br.ReadInt64()); // Read the backup information // Note that if you change this part, backup log record also must be changed. this.backupId = new Guid(br.ReadBytes(Constants.SizeOfGuidBytes)); this.highestBackedUpEpoch = new Epoch(br.ReadInt64(), br.ReadInt64()); this.highestBackedUpLsn = new LogicalSequenceNumber(br.ReadInt64()); this.backupLogRecordCount = br.ReadUInt32(); this.backupLogSize = br.ReadUInt32(); // Conditionally read periodicCheckpointTimeTicks_ // Ensures compatibility with versions prior to addition of timestamp field if (br.BaseStream.Position < endPosition) { this.lastPeriodicCheckpointTimeTicks = br.ReadInt64(); this.lastPeriodicTruncationTimeTicks = br.ReadInt64(); } // Jump to the end of the section ignoring fields that are not understood. Utility.Assert(endPosition >= br.BaseStream.Position, "Could not have read more than section size."); br.BaseStream.Position = endPosition; this.UpdateApproximateDiskSize(); }
/// <summary> /// Initializes a new instance of the LogRecordsMap class for recovery. /// </summary> /// <param name="startingLogicalSequenceNumber"></param> /// <param name="transactionsMap"></param> /// <param name="currentLogTailEpoch"></param> /// <param name="lastStableLsn"></param> /// <param name="progressVector"></param> /// <param name="recoveredLastCompletedBeginCheckpointRecord"></param> /// <param name="tracer"></param> /// <param name="recoveredLastEndCheckpointRecord"></param> public LogRecordsMap( LogicalSequenceNumber startingLogicalSequenceNumber, TransactionMap transactionsMap, Epoch currentLogTailEpoch, LogicalSequenceNumber lastStableLsn, ProgressVector progressVector, BeginCheckpointLogRecord recoveredLastCompletedBeginCheckpointRecord, ITracer tracer, EndCheckpointLogRecord recoveredLastEndCheckpointRecord) { this.mode = Mode.Recovery; this.LastLogicalSequenceNumber = startingLogicalSequenceNumber; this.TransactionsMap = transactionsMap; this.CurrentLogTailEpoch = currentLogTailEpoch; this.LastStableLsn = lastStableLsn; this.LastPhysicalRecord = PhysicalLogRecord.InvalidPhysicalLogRecord; this.ProgressVector = progressVector; this.recoveredLastCompletedBeginCheckpointRecord = recoveredLastCompletedBeginCheckpointRecord; this.tracer = tracer; this.recoveredLastEndCheckpointRecord = recoveredLastEndCheckpointRecord; }
/// <summary> /// Initializes a new instance of the BeginCheckpointLogRecord class. /// </summary> /// <remarks>Called when the replicator decides to checkpoint.</remarks> internal BeginCheckpointLogRecord( bool isFirstCheckpointOnFullCopy, ProgressVector progressVector, BeginTransactionOperationLogRecord earliestPendingTransaction, Epoch headEpoch, Epoch epoch, LogicalSequenceNumber lsn, PhysicalLogRecord lastLinkedPhysicalRecord, BackupLogRecord lastCompletedBackupLogRecord, uint progressVectorMaxEntries, long periodicCheckpointTimeTicks, long periodicTruncationTimeTicks) : base(LogRecordType.BeginCheckpoint, lsn, lastLinkedPhysicalRecord) { this.IsFirstCheckpointOnFullCopy = isFirstCheckpointOnFullCopy; this.progressVector = ProgressVector.Clone(progressVector, progressVectorMaxEntries, lastCompletedBackupLogRecord.HighestBackedUpEpoch, headEpoch); this.earliestPendingTransactionOffset = LogicalLogRecord.InvalidLogicalRecordOffset; this.earliestPendingTransaction = earliestPendingTransaction; this.checkpointState = CheckpointState.Invalid; this.lastStableLsn = LogicalSequenceNumber.InvalidLsn; this.epoch = (earliestPendingTransaction != null) ? earliestPendingTransaction.RecordEpoch : epoch; // Initialize backup log record fields. this.highestBackedUpEpoch = lastCompletedBackupLogRecord.HighestBackedUpEpoch; this.highestBackedUpLsn = lastCompletedBackupLogRecord.HighestBackedUpLsn; this.backupLogRecordCount = lastCompletedBackupLogRecord.BackupLogRecordCount; this.backupLogSize = lastCompletedBackupLogRecord.BackupLogSizeInKB; this.earliestPendingTransactionInvalidated = 0; this.lastPeriodicCheckpointTimeTicks = periodicCheckpointTimeTicks; this.lastPeriodicTruncationTimeTicks = periodicTruncationTimeTicks; this.UpdateApproximateDiskSize(); }
private async Task <OperationData> GetNextAsyncSafe(CancellationToken cancellationToken) { OperationData data; if (this.copyStage == CopyStage.CopyMetadata) { data = await this.copyContext.GetNextAsync(CancellationToken.None).ConfigureAwait(false); Utility.Assert(data.Count == 1, "data.Count == 1"); var bytes = data[0].Array; using (var br = new BinaryReader(new MemoryStream(bytes, data[0].Offset, data[0].Count))) { this.targetReplicaId = br.ReadInt64(); this.targetProgressVector = new ProgressVector(); this.targetProgressVector.Read(br, false); var targetLogHeadDatalossNumber = br.ReadInt64(); var targetLogHeadConfigurationNumber = br.ReadInt64(); this.targetLogHeadEpoch = new Epoch(targetLogHeadDatalossNumber, targetLogHeadConfigurationNumber); this.targetLogHeadLsn = new LogicalSequenceNumber(br.ReadInt64()); this.currentTargetLsn = new LogicalSequenceNumber(br.ReadInt64()); this.latestRecoveredAtomicRedoOperationLsn = br.ReadInt64(); } var message = string.Format( CultureInfo.InvariantCulture, "Target Replica Copy Context: {0}" + Environment.NewLine + " TargetLogHeadEpoch: {1},{2} TargetLogHeadLsn: {3} TargetLogTailEpoch: {4},{5} TargetLogTailLSN: {6}" + Environment.NewLine + " SourceLogHeadEpoch: {7},{8} SourceLogHeadLsn: {9} SourceLogTailEpoch: {10},{11} SourceLogTailLSN: {12}" + Environment.NewLine + " Target ProgressVector: {13}" + Environment.NewLine + " Source ProgressVector: {14}" + Environment.NewLine + " LastRecoveredAtomicRedoOperationLsn: {15}", this.targetReplicaId, this.targetLogHeadEpoch.DataLossNumber, this.targetLogHeadEpoch.ConfigurationNumber, this.targetLogHeadLsn.LSN, this.targetProgressVector.LastProgressVectorEntry.Epoch.DataLossNumber, this.targetProgressVector.LastProgressVectorEntry.Epoch.ConfigurationNumber, this.currentTargetLsn.LSN, this.replicatedLogManager.CurrentLogHeadRecord.CurrentEpoch.DataLossNumber, this.replicatedLogManager.CurrentLogHeadRecord.CurrentEpoch.ConfigurationNumber, this.replicatedLogManager.CurrentLogHeadRecord.Lsn.LSN, this.replicatedLogManager.CurrentLogTailEpoch.DataLossNumber, this.replicatedLogManager.CurrentLogTailEpoch.ConfigurationNumber, this.replicatedLogManager.CurrentLogTailLsn.LSN, this.targetProgressVector.ToString(Constants.ProgressVectorMaxStringSizeInKb / 2), this.replicatedLogManager.ProgressVector.ToString(Constants.ProgressVectorMaxStringSizeInKb / 2), this.latestRecoveredAtomicRedoOperationLsn); FabricEvents.Events.CopyStreamMetadata(tracer.Type, message); await this.waitForLogFlushUptoLsn(this.uptoLsn).ConfigureAwait(false); CopyMode copyMode; await this.checkpointManager.AcquireBackupAndCopyConsistencyLockAsync("Copy").ConfigureAwait(false); try { copyMode = this.checkpointManager.GetLogRecordsToCopy( this.targetProgressVector, this.targetLogHeadEpoch, this.targetLogHeadLsn, this.currentTargetLsn, this.latestRecoveredAtomicRedoOperationLsn, this.targetReplicaId, out this.sourceStartingLsn, out this.targetStartingLsn, out this.logRecordsToCopy, out this.beginCheckpointRecord); } finally { this.checkpointManager.ReleaseBackupAndCopyConsistencyLock("Copy"); } CopyStage copyStageToWrite; if (copyMode == CopyMode.None) { this.copyStage = CopyStage.CopyNone; copyStageToWrite = CopyStage.CopyNone; } else if (copyMode == CopyMode.Full) { this.copyStage = CopyStage.CopyState; copyStageToWrite = CopyStage.CopyState; this.copyStateStream = this.stateManager.GetCurrentState(); } else if ((copyMode & CopyMode.FalseProgress) != 0) { Utility.Assert( this.sourceStartingLsn <= this.targetStartingLsn, "this.sourceStartingLsn ({0}) <= this.targetStartingLsn ({1})", this.sourceStartingLsn, this.targetStartingLsn); this.copyStage = CopyStage.CopyFalseProgress; copyStageToWrite = CopyStage.CopyFalseProgress; } else { this.copyStage = CopyStage.CopyScanToStartingLsn; copyStageToWrite = CopyStage.CopyLog; } return(this.GetCopyMetadata(copyStageToWrite)); } if (this.copyStage == CopyStage.CopyState) { data = await this.copyStateStream.GetNextAsync(cancellationToken).ConfigureAwait(false); if (data != null) { data.Add(CopyStateOperation); FabricEvents.Events.CopyStreamGetNextNoise( tracer.Type, "Copying State Operation: " + this.copiedRecordNumber); ++this.copiedRecordNumber; return(data); } var disposableCopyStateStream = this.copyStateStream as IDisposable; if (disposableCopyStateStream != null) { disposableCopyStateStream.Dispose(); } if (this.uptoLsn < beginCheckpointRecord.LastStableLsn) { // Ensure we copy records up to the stable LSN of the begin checkpoint record as part of the copy stream // so that the idle can initiate a checkpoint and apply it as part of the copy pump itself // Not doing the above will mean that the idle can get promoted to active even before the checkpoint is completed // uptolsn is inclusive LSN // BUG: RDBug #9269022:Replicator must rename copy log before changing role to active secondary during full builds this.uptoLsn = beginCheckpointRecord.LastStableLsn; var trace = "FullCopy: Copying Logs from " + this.sourceStartingLsn.LSN + " to BC.LastStableLsn: " + beginCheckpointRecord.LastStableLsn.LSN + " as the upto lsn: " + this.uptoLsn + " was smaller than the checkpoint stable lsn"; FabricEvents.Events.CopyStreamGetNext(tracer.Type, trace); } data = this.GetCopyStateMetadata(); this.copyStage = CopyStage.CopyLog; this.copiedRecordNumber = 0; return(data); } if (this.copyStage == CopyStage.CopyFalseProgress) { this.copyStage = CopyStage.CopyScanToStartingLsn; this.bw.BaseStream.Position = 0; this.bw.Write(this.sourceStartingLsn.LSN); var stream = this.bw.BaseStream as MemoryStream; data = new OperationData(new ArraySegment <byte>(stream.GetBuffer(), 0, (int)stream.Position)); data.Add(CopyFalseProgressOperation); var trace = "Copying False Progress. SourceStartingLSN: " + this.sourceStartingLsn.LSN + " TargetStartingLSN: " + this.targetStartingLsn.LSN; FabricEvents.Events.CopyStreamGetNext(tracer.Type, trace); return(data); } LogRecord record; if (this.copyStage == CopyStage.CopyScanToStartingLsn) { var startingLsn = (this.sourceStartingLsn < this.targetStartingLsn) ? this.sourceStartingLsn : this.targetStartingLsn; do { var hasMoved = await this.logRecordsToCopy.MoveNextAsync(cancellationToken).ConfigureAwait(false); if (hasMoved == false) { goto Finish; } record = this.logRecordsToCopy.Current; Utility.Assert(record.Lsn <= startingLsn, "record.Lsn <= startingLsn"); } while (record.Lsn < startingLsn); // The log stream is positioned at the end of startingLsn at this point. The enumerator Current is pointing to the "startingLsn" this.copyStage = CopyStage.CopyLog; } if (this.copyStage == CopyStage.CopyLog) { do { // This will start copying after the startingLsn record as expected var hasMoved = await this.logRecordsToCopy.MoveNextAsync(cancellationToken).ConfigureAwait(false); if (hasMoved == false) { goto Finish; } record = this.logRecordsToCopy.Current; } while (record is PhysicalLogRecord); if (record.Lsn > this.uptoLsn) { goto Finish; } var logicalLogRecord = record as LogicalLogRecord; Utility.Assert(logicalLogRecord != null, "Must be a logical log record"); data = logicalLogRecord.ToOperationData(this.bw); data.Add(CopyLogOperation); var trace = "Copying log record. LogRecordNumber: " + this.copiedRecordNumber + "Record Type: " + record.RecordType + " LSN: " + record.Lsn.LSN; FabricEvents.Events.CopyStreamGetNextNoise(tracer.Type, trace); ++this.copiedRecordNumber; return(data); } Finish: if (this.logRecordsToCopy != null) { this.logRecordsToCopy.Dispose(); } FabricEvents.Events.CopyStreamFinished(tracer.Type, this.copiedRecordNumber); return(null); }
internal static SharedProgressVectorEntry FindSharedVector( ProgressVector sourceProgressVector, ProgressVector targetProgressVector) { // Initialize indices var sourceIndex = sourceProgressVector.vectors.Count - 1; var targetIndex = targetProgressVector.vectors.Count - 1; Utility.Assert( targetProgressVector.vectors[targetIndex].Epoch.DataLossNumber <= sourceProgressVector.vectors[sourceIndex].Epoch.DataLossNumber, "targetDataLossNumber ({0}) <= sourceDataLossNumber ({1})", targetProgressVector.vectors[targetIndex].Epoch.DataLossNumber, sourceProgressVector.vectors[sourceIndex].Epoch.DataLossNumber); SharedProgressVectorEntry progrssVectorTrimmedResult = new SharedProgressVectorEntry() { FullCopyReason = FullCopyReason.ProgressVectorTrimmed }; do { bool decrementIndexSuccess = DecrementIndexUntilLeq( ref targetIndex, targetProgressVector, sourceProgressVector.vectors[sourceIndex]); if (!decrementIndexSuccess) { return(progrssVectorTrimmedResult); } decrementIndexSuccess = DecrementIndexUntilLeq( ref sourceIndex, sourceProgressVector, targetProgressVector.vectors[targetIndex]); if (!decrementIndexSuccess) { return(progrssVectorTrimmedResult); } } while (sourceProgressVector.vectors[sourceIndex] != targetProgressVector.vectors[targetIndex]); var sourceProgressVectorEntry = sourceProgressVector.vectors[sourceIndex]; var targetProgressVectorEntry = targetProgressVector.vectors[targetIndex]; // WITHOUT MINI RECONFIG ENABLED, it is not possible for the target to make VALID progress TWICE from the shared epoch // Note that this check is only valid in the absence of dataloss and restore. // E.g /* * Target Replica Copy Context: 131383889527030885 * TargetLogHeadEpoch: 0,0 TargetLogHeadLsn: 0 TargetLogTailEpoch: 131383889406148342,64424509440 TargetLogTailLSN: 73 * SourceLogHeadEpoch: 0,0 SourceLogHeadLsn: 0 SourceLogTailEpoch: 131383889406148342,68719476736 SourceLogTailLSN: 68 * Target ProgressVector: [(131383889406148342,64424509440),73,131383889527030885,2017-05-04T16:32:22] * [(131383889406148342,55834574848),68,131383889527030885,2017-05-04T16:31:54] * [(131383889406148342,51539607552),62,131383889416461059,2017-05-04T16:31:37] * [(131383889406148342,47244640256),56,131383889527030885,2017-05-04T16:31:20] * [(0,0),0,0,2017-05-04T16:29:07] * Source ProgressVector: [(131383889406148342,68719476736),68,131383890970495929,2017-05-04T16:32:33] * [(131383889406148342,60129542144),68,131383890970495929,2017-05-04T16:32:09] * [(131383889406148342,51539607552),62,131383889416461059,2017-05-04T16:31:37] * [(131383889406148342,47244640256),56,131383889527030885,2017-05-04T16:31:20] * [(0,0),0,0,2017-05-04T16:31:33] * LastRecoveredAtomicRedoOperationLsn: 0. * * * Target Replica Copy Context: 131399770096882719 * TargetLogHeadEpoch: 0,0 TargetLogHeadLsn: 0 TargetLogTailEpoch: 131399758616683034,390842023936 TargetLogTailLSN: 374 * SourceLogHeadEpoch: 0,0 SourceLogHeadLsn: 0 SourceLogTailEpoch: 131399758616683034,395136991232 SourceLogTailLSN: 369 * Target ProgressVector: * [(131399758616683034,390842023936),374,131399770096882719,2017-05-23T01:42:03] * [(131399758616683034,382252089344),374,131399770096882719,2017-05-23T01:41:29] * [(131399758616683034,377957122048),369,131399770096882719,2017-05-23T01:41:00] * [(131399758616683034,373662154752),338,131399758632647497,2017-05-23T01:40:31] * Source ProgressVector: * [(131399758616683034,395136991232),369,131399758701866693,2017-05-23T01:42:19] * [(131399758616683034,386547056640),369,131399758701866693,2017-05-23T01:41:44] * [(131399758616683034,373662154752),338,131399758632647497,2017-05-23T01:40:31] * LastRecoveredAtomicRedoOperationLsn: 0 */ string failureMsg; if (targetProgressVector.LastProgressVectorEntry.Epoch.DataLossNumber == sourceProgressVector.LastProgressVectorEntry.Epoch.DataLossNumber && targetIndex < targetProgressVector.vectors.Count - 1 && targetProgressVector.LastProgressVectorEntry.Epoch.DataLossNumber == targetProgressVector.vectors[targetIndex].Epoch.DataLossNumber) { // Target replica could only have repeatedly attempted to become a // primary without ever making progress at the shared dataloss number // We can do "double progress check" only if there has NOT been any other data loss happening on the target after shared point. Having a differnt dataloss numbers in shared vector and target's tail // means that target has been a valid primary and progressed in LSN until hitting a dataloss and then a very old secondary can become primary and invalidate all the progress made by the target. var failureLsn = targetProgressVector.vectors[targetIndex + 1].Lsn; var failureLsnIncremented = 0; for (var n = targetIndex + 2; n <= targetProgressVector.vectors.Count - 1; n++) { var vector = targetProgressVector.vectors[n]; if (vector.Epoch.DataLossNumber != targetProgressVectorEntry.Epoch.DataLossNumber) { break; } if (vector.Lsn != failureLsn) { // Update failureLsn to ensure we don't assert if there are multiple entries for same LSN like in 2nd example above failureLsn = vector.Lsn; failureLsnIncremented++; } } failureMsg = string.Format( "FailureLsn incremented must be <= 1. It is {0}", failureLsnIncremented); if (!ValidateIfDebugEnabled( failureLsnIncremented <= 1, failureMsg)) { return(new SharedProgressVectorEntry { SourceIndex = sourceIndex, TargetIndex = targetIndex, SourceProgressVectorEntry = sourceProgressVectorEntry, TargetProgressVectorEntry = targetProgressVectorEntry, FullCopyReason = FullCopyReason.ValidationFailed, FailedValidationMessage = failureMsg }); } } // Sanity check that source and target progress vectors are always in // agreement upto above determined ProgressVectorEntry shared by both of them var i = sourceIndex; var sourceEntry = sourceProgressVectorEntry; var j = targetIndex; var targetEntry = targetProgressVectorEntry; failureMsg = "sourceEntry == targetEntry"; do { sourceEntry = DecrementIndexUntilLsnIsEqual(ref i, sourceProgressVector, sourceEntry.Lsn.LSN); if (i == 0) { break; } targetEntry = DecrementIndexUntilLsnIsEqual(ref j, targetProgressVector, targetEntry.Lsn.LSN); if (j == 0) { break; } if (!ValidateIfDebugEnabled( sourceEntry == targetEntry, failureMsg)) { return(new SharedProgressVectorEntry { SourceIndex = sourceIndex, TargetIndex = targetIndex, SourceProgressVectorEntry = sourceProgressVectorEntry, TargetProgressVectorEntry = targetProgressVectorEntry, FullCopyReason = FullCopyReason.ValidationFailed, FailedValidationMessage = failureMsg }); } } while (true); Utility.Assert(sourceProgressVectorEntry != null && targetProgressVectorEntry != null, "Source and target expected to be non null"); return(new SharedProgressVectorEntry { SourceIndex = sourceIndex, TargetIndex = targetIndex, SourceProgressVectorEntry = sourceProgressVectorEntry, TargetProgressVectorEntry = targetProgressVectorEntry }); }
private static ProgressVectorEntry DecrementIndexUntilLsnIsEqual(ref int index, ProgressVector vector, long lsn) { while (index > 0) { var previousSourceVector = vector.vectors[index - 1]; if (previousSourceVector.Lsn.LSN != lsn) { return(previousSourceVector); } --index; } return(null); }
private static CopyModeResult FindCopyModePrivate( CopyContextParameters sourceParamaters, CopyContextParameters targetParameters, long lastRecoveredAtomicRedoOperationLsnAtTarget) { // We can perform the check for CASE 1(Shown Below) before finding the shared ProgressVectorEntry. // However, we do not do that because the shared ProgressVectorEntry method provides additional checks of epoch history var sharedProgressVectorEntry = FindSharedVector( sourceParamaters.ProgressVector, targetParameters.ProgressVector); // If we detect that the shared progress vector cannot be obtained because of a trimmed progress vector, we decide to do a full copy if (sharedProgressVectorEntry.FullCopyReason == FullCopyReason.ProgressVectorTrimmed) { return(CopyModeResult.CreateFullCopyResult(null, FullCopyReason.ProgressVectorTrimmed)); } // Validation failed, perform full copy if (sharedProgressVectorEntry.FullCopyReason == FullCopyReason.ValidationFailed) { return(CopyModeResult.CreateFullCopyResult(sharedProgressVectorEntry, FullCopyReason.ValidationFailed)); } // *******************************************CASE 1 - COPY_NONE*********************************************************** // // The last entry in the source and the target vectors are same with the same tail. // This can happen if target goes down and comes back up and the primary made no progress if (sourceParamaters.ProgressVector.LastProgressVectorEntry == targetParameters.ProgressVector.LastProgressVectorEntry && sourceParamaters.LogTailLsn == targetParameters.LogTailLsn) { // Note that copy none covers only cases where nothing needs to be copied (including the UpdateEpochLogRecord) // Hence, service creation scenario is Partial copy, not None. return(CopyModeResult.CreateCopyNoneResult(sharedProgressVectorEntry)); } // *******************************************CASE 2a - COPY_FULL - Due to Data Loss****************************************** // // Check for any dataloss in the source or target log // If there was data loss, perform full copy // No full copy with DataLoss reason, if the target is a brand new replica if (!ProgressVector.IsBrandNewReplica(targetParameters)) { if (sharedProgressVectorEntry.SourceProgressVectorEntry.Epoch.DataLossNumber != sourceParamaters.ProgressVector.LastProgressVectorEntry.Epoch.DataLossNumber || sharedProgressVectorEntry.TargetProgressVectorEntry.Epoch.DataLossNumber != targetParameters.ProgressVector.LastProgressVectorEntry.Epoch.DataLossNumber || sourceParamaters.LogHeadEpoch.DataLossNumber > sharedProgressVectorEntry.TargetProgressVectorEntry.Epoch.DataLossNumber || targetParameters.LogHeadEpoch.DataLossNumber > sharedProgressVectorEntry.SourceProgressVectorEntry.Epoch.DataLossNumber) { return(CopyModeResult.CreateFullCopyResult(sharedProgressVectorEntry, FullCopyReason.DataLoss)); } } var sourceStartingLsn = (sharedProgressVectorEntry.SourceIndex == (sourceParamaters.ProgressVector.vectors.Count - 1)) ? sourceParamaters.LogTailLsn : sourceParamaters.ProgressVector.vectors[sharedProgressVectorEntry.SourceIndex + 1].Lsn; var targetStartingLsn = (sharedProgressVectorEntry.TargetIndex == (targetParameters.ProgressVector.vectors.Count - 1)) ? targetParameters.LogTailLsn : targetParameters.ProgressVector.vectors[sharedProgressVectorEntry.TargetIndex + 1].Lsn; // *******************************************CASE 2b - COPY_FULL - Due to Insufficient Loss****************************************** // // Since there was no data loss, check if we can perform partial copy and there are sufficient logs in the source // and target to do so if (sourceParamaters.LogHeadLsn > targetParameters.LogTailLsn || sourceStartingLsn < sourceParamaters.LogHeadLsn || targetParameters.LogHeadLsn > sourceStartingLsn) { return(CopyModeResult.CreateFullCopyResult(sharedProgressVectorEntry, FullCopyReason.InsufficientLogs)); } // *******************************************CASE 3a - FALSE_PROGRESS -Basic Case****************************************** // // The simple case where target made more progress in an epoch than the primary // Source = (1,1,0) (1,3,10) (1,4,15) Tail = 16 // Target = (1,1,0) (1,3,10) Tail = 20 if (sourceStartingLsn < targetStartingLsn) { return(CopyModeResult.CreateFalseProgressResult( lastRecoveredAtomicRedoOperationLsnAtTarget, sharedProgressVectorEntry, sourceStartingLsn, targetStartingLsn)); } // *******************************************CASE 3b - FALSE_PROGRESS -Advanced Case****************************************** // // The cases where target has made progress in an epoch that the primary has no knowledge of // Series of crashes could lead to the target having made false progress in an epoch that the primary DOES NOT KNOW about // Source = (1,1,0) (1,3,10) Tail = 15 // Target = (1,1,0) (1,2,10) Tail = 12 // Source = (1,1,0) (1,3,10) Tail = 15 // Target = (1,1,0) (1,2,10) Tail = 10 // Source = (1,3,10) (1,5,17) Tail = 18 // Target = (1,3,10) (1,4,15) Tail = 17 if (targetStartingLsn != targetParameters.LogTailLsn) { sourceStartingLsn = targetStartingLsn; return(CopyModeResult.CreateFalseProgressResult( lastRecoveredAtomicRedoOperationLsnAtTarget, sharedProgressVectorEntry, sourceStartingLsn, targetStartingLsn)); } // *******************************************CASE 4 - Work Around Case to be able to assert****************************************** // RDBUG : 3374739 // Source = (1,1,0) (1,3,10) Tail >= 10 // Target = (1,1,0) (1,2,5) Tail = 5 // Target got UpdateEpoch(e5) at 5, while source got UpdateEpoch(e6) at 10 (without having an entry for e5) // Source should detect this and do a full build - (We cannot undo false progress since the process at the target until 5 could be true progress and could be checkpointed) if (sourceParamaters.ProgressVector.vectors[sharedProgressVectorEntry.SourceIndex].Epoch < targetParameters.ProgressVector.LastProgressVectorEntry.Epoch) { return(CopyModeResult.CreateFullCopyResult(sharedProgressVectorEntry, FullCopyReason.Other)); } return(CopyModeResult.CreatePartialCopyResult(sharedProgressVectorEntry, sourceStartingLsn, targetStartingLsn)); }