internal LoggingReplicatorCopyStream(
ReplicatedLogManager replicatedLogManager,
IStateManager stateManager,
CheckpointManager checkpointManager,
Func <LogicalSequenceNumber, Task> waitForLogFlushUptoLsn,
long replicaId,
LogicalSequenceNumber uptoLsn,
IOperationDataStream copyContext,
ITracer tracer)
{
this.stateManager = stateManager;
this.checkpointManager = checkpointManager;
this.replicatedLogManager = replicatedLogManager;
this.replicaId = replicaId;
this.waitForLogFlushUptoLsn = waitForLogFlushUptoLsn;
this.uptoLsn = uptoLsn;
this.copyContext = copyContext;
this.targetReplicaId = 0;
this.targetProgressVector = null;
this.targetLogHeadEpoch = LogicalSequenceNumber.InvalidEpoch;
this.targetLogHeadLsn = LogicalSequenceNumber.InvalidLsn;
this.currentTargetLsn = LogicalSequenceNumber.InvalidLsn;
this.copyStage = CopyStage.CopyMetadata;
this.copyStateStream = null;
this.copiedRecordNumber = 0;
this.sourceStartingLsn = LogicalSequenceNumber.InvalidLsn;
this.targetStartingLsn = LogicalSequenceNumber.InvalidLsn;
this.logRecordsToCopy = null;
this.beginCheckpointRecord = null;
this.bw = new BinaryWriter(new MemoryStream());
this.isDisposed = false;
this.tracer = tracer;
}
private void Dispose(bool disposing)
{
if (!this.disposed)
{
if (disposing)
{
// Trace that copy has completed if we started.
FabricEvents.Events.CopyAsync(this.traceType, "disposing");
if (this.currentFileStream != null)
{
this.currentFileStream.Dispose();
this.currentFileStream = null;
}
if (this.copySnapshotOfMetadataTableEnumerator != null)
{
this.copySnapshotOfMetadataTableEnumerator.Dispose();
this.copySnapshotOfMetadataTableEnumerator = null;
}
if (this.copySnapshotOfMetadataTable != null)
{
this.copySnapshotOfMetadataTable.ReleaseRef();
this.copySnapshotOfMetadataTable = null;
}
}
this.copyStage = CopyStage.None;
}
this.disposed = true;
}
internal static void Assert(bool condition, string format, CopyStage param1)
{
if (condition == false)
{
var failFastMessage = string.Format(System.Globalization.CultureInfo.InvariantCulture, format, param1);
FailFast(failFastMessage);
// AMW - Force break into debugger for ease of debugging
Debugger.Break();
}
}
private OperationData GetCopyMetadata(CopyStage copyStageToWrite)
{
OperationData result;
using (var stream = new MemoryStream(CopyMetadataByteSize))
{
using (var bw = new BinaryWriter(stream))
{
// First 4 bytes are reserved for the version of the metadata.
bw.Write(CopyMetadataVersion);
bw.Write((int)copyStageToWrite);
bw.Write(this.replicaId);
result = new OperationData(new ArraySegment <byte>(stream.GetBuffer(), 0, (int)stream.Position));
}
}
var trace = "Copying Metadata. CopyStage: " + this.copyStage + " SourceStartingLSN: "
+ this.sourceStartingLsn.LSN + " TargetStartingLSN: " + this.targetStartingLsn.LSN;
FabricEvents.Events.CopyStreamGetNext(tracer.Type, trace);
return(result);
}
private async Task <OperationData> GetNextAsyncSafe(CancellationToken cancellationToken)
{
OperationData data;
if (this.copyStage == CopyStage.CopyMetadata)
{
data = await this.copyContext.GetNextAsync(CancellationToken.None).ConfigureAwait(false);
Utility.Assert(data.Count == 1, "data.Count == 1");
var bytes = data[0].Array;
using (var br = new BinaryReader(new MemoryStream(bytes, data[0].Offset, data[0].Count)))
{
this.targetReplicaId = br.ReadInt64();
this.targetProgressVector = new ProgressVector();
this.targetProgressVector.Read(br, false);
var targetLogHeadDatalossNumber = br.ReadInt64();
var targetLogHeadConfigurationNumber = br.ReadInt64();
this.targetLogHeadEpoch = new Epoch(targetLogHeadDatalossNumber, targetLogHeadConfigurationNumber);
this.targetLogHeadLsn = new LogicalSequenceNumber(br.ReadInt64());
this.currentTargetLsn = new LogicalSequenceNumber(br.ReadInt64());
this.latestRecoveredAtomicRedoOperationLsn = br.ReadInt64();
}
var message =
string.Format(
CultureInfo.InvariantCulture,
"Target Replica Copy Context: {0}" + Environment.NewLine
+ " TargetLogHeadEpoch: {1},{2} TargetLogHeadLsn: {3} TargetLogTailEpoch: {4},{5} TargetLogTailLSN: {6}"
+ Environment.NewLine
+ " SourceLogHeadEpoch: {7},{8} SourceLogHeadLsn: {9} SourceLogTailEpoch: {10},{11} SourceLogTailLSN: {12}"
+ Environment.NewLine + " Target ProgressVector: {13}" + Environment.NewLine
+ " Source ProgressVector: {14}" + Environment.NewLine
+ " LastRecoveredAtomicRedoOperationLsn: {15}",
this.targetReplicaId,
this.targetLogHeadEpoch.DataLossNumber,
this.targetLogHeadEpoch.ConfigurationNumber,
this.targetLogHeadLsn.LSN,
this.targetProgressVector.LastProgressVectorEntry.Epoch.DataLossNumber,
this.targetProgressVector.LastProgressVectorEntry.Epoch.ConfigurationNumber,
this.currentTargetLsn.LSN,
this.replicatedLogManager.CurrentLogHeadRecord.CurrentEpoch.DataLossNumber,
this.replicatedLogManager.CurrentLogHeadRecord.CurrentEpoch.ConfigurationNumber,
this.replicatedLogManager.CurrentLogHeadRecord.Lsn.LSN,
this.replicatedLogManager.CurrentLogTailEpoch.DataLossNumber,
this.replicatedLogManager.CurrentLogTailEpoch.ConfigurationNumber,
this.replicatedLogManager.CurrentLogTailLsn.LSN,
this.targetProgressVector.ToString(Constants.ProgressVectorMaxStringSizeInKb / 2),
this.replicatedLogManager.ProgressVector.ToString(Constants.ProgressVectorMaxStringSizeInKb / 2),
this.latestRecoveredAtomicRedoOperationLsn);
FabricEvents.Events.CopyStreamMetadata(tracer.Type, message);
await this.waitForLogFlushUptoLsn(this.uptoLsn).ConfigureAwait(false);
CopyMode copyMode;
await this.checkpointManager.AcquireBackupAndCopyConsistencyLockAsync("Copy").ConfigureAwait(false);
try
{
copyMode = this.checkpointManager.GetLogRecordsToCopy(
this.targetProgressVector,
this.targetLogHeadEpoch,
this.targetLogHeadLsn,
this.currentTargetLsn,
this.latestRecoveredAtomicRedoOperationLsn,
this.targetReplicaId,
out this.sourceStartingLsn,
out this.targetStartingLsn,
out this.logRecordsToCopy,
out this.beginCheckpointRecord);
}
finally
{
this.checkpointManager.ReleaseBackupAndCopyConsistencyLock("Copy");
}
CopyStage copyStageToWrite;
if (copyMode == CopyMode.None)
{
this.copyStage = CopyStage.CopyNone;
copyStageToWrite = CopyStage.CopyNone;
}
else if (copyMode == CopyMode.Full)
{
this.copyStage = CopyStage.CopyState;
copyStageToWrite = CopyStage.CopyState;
this.copyStateStream = this.stateManager.GetCurrentState();
}
else if ((copyMode & CopyMode.FalseProgress) != 0)
{
Utility.Assert(
this.sourceStartingLsn <= this.targetStartingLsn,
"this.sourceStartingLsn ({0}) <= this.targetStartingLsn ({1})",
this.sourceStartingLsn, this.targetStartingLsn);
this.copyStage = CopyStage.CopyFalseProgress;
copyStageToWrite = CopyStage.CopyFalseProgress;
}
else
{
this.copyStage = CopyStage.CopyScanToStartingLsn;
copyStageToWrite = CopyStage.CopyLog;
}
return(this.GetCopyMetadata(copyStageToWrite));
}
if (this.copyStage == CopyStage.CopyState)
{
data = await this.copyStateStream.GetNextAsync(cancellationToken).ConfigureAwait(false);
if (data != null)
{
data.Add(CopyStateOperation);
FabricEvents.Events.CopyStreamGetNextNoise(
tracer.Type,
"Copying State Operation: " + this.copiedRecordNumber);
++this.copiedRecordNumber;
return(data);
}
var disposableCopyStateStream = this.copyStateStream as IDisposable;
if (disposableCopyStateStream != null)
{
disposableCopyStateStream.Dispose();
}
if (this.uptoLsn < beginCheckpointRecord.LastStableLsn)
{
// Ensure we copy records up to the stable LSN of the begin checkpoint record as part of the copy stream
// so that the idle can initiate a checkpoint and apply it as part of the copy pump itself
// Not doing the above will mean that the idle can get promoted to active even before the checkpoint is completed
// uptolsn is inclusive LSN
// BUG: RDBug #9269022:Replicator must rename copy log before changing role to active secondary during full builds
this.uptoLsn = beginCheckpointRecord.LastStableLsn;
var trace = "FullCopy: Copying Logs from " + this.sourceStartingLsn.LSN
+ " to BC.LastStableLsn: " + beginCheckpointRecord.LastStableLsn.LSN + " as the upto lsn: " + this.uptoLsn + " was smaller than the checkpoint stable lsn";
FabricEvents.Events.CopyStreamGetNext(tracer.Type, trace);
}
data = this.GetCopyStateMetadata();
this.copyStage = CopyStage.CopyLog;
this.copiedRecordNumber = 0;
return(data);
}
if (this.copyStage == CopyStage.CopyFalseProgress)
{
this.copyStage = CopyStage.CopyScanToStartingLsn;
this.bw.BaseStream.Position = 0;
this.bw.Write(this.sourceStartingLsn.LSN);
var stream = this.bw.BaseStream as MemoryStream;
data = new OperationData(new ArraySegment <byte>(stream.GetBuffer(), 0, (int)stream.Position));
data.Add(CopyFalseProgressOperation);
var trace = "Copying False Progress. SourceStartingLSN: " + this.sourceStartingLsn.LSN
+ " TargetStartingLSN: " + this.targetStartingLsn.LSN;
FabricEvents.Events.CopyStreamGetNext(tracer.Type, trace);
return(data);
}
LogRecord record;
if (this.copyStage == CopyStage.CopyScanToStartingLsn)
{
var startingLsn = (this.sourceStartingLsn < this.targetStartingLsn)
? this.sourceStartingLsn
: this.targetStartingLsn;
do
{
var hasMoved = await this.logRecordsToCopy.MoveNextAsync(cancellationToken).ConfigureAwait(false);
if (hasMoved == false)
{
goto Finish;
}
record = this.logRecordsToCopy.Current;
Utility.Assert(record.Lsn <= startingLsn, "record.Lsn <= startingLsn");
} while (record.Lsn < startingLsn);
// The log stream is positioned at the end of startingLsn at this point. The enumerator Current is pointing to the "startingLsn"
this.copyStage = CopyStage.CopyLog;
}
if (this.copyStage == CopyStage.CopyLog)
{
do
{
// This will start copying after the startingLsn record as expected
var hasMoved = await this.logRecordsToCopy.MoveNextAsync(cancellationToken).ConfigureAwait(false);
if (hasMoved == false)
{
goto Finish;
}
record = this.logRecordsToCopy.Current;
} while (record is PhysicalLogRecord);
if (record.Lsn > this.uptoLsn)
{
goto Finish;
}
var logicalLogRecord = record as LogicalLogRecord;
Utility.Assert(logicalLogRecord != null, "Must be a logical log record");
data = logicalLogRecord.ToOperationData(this.bw);
data.Add(CopyLogOperation);
var trace = "Copying log record. LogRecordNumber: " + this.copiedRecordNumber + "Record Type: "
+ record.RecordType + " LSN: " + record.Lsn.LSN;
FabricEvents.Events.CopyStreamGetNextNoise(tracer.Type, trace);
++this.copiedRecordNumber;
return(data);
}
Finish:
if (this.logRecordsToCopy != null)
{
this.logRecordsToCopy.Dispose();
}
FabricEvents.Events.CopyStreamFinished(tracer.Type, this.copiedRecordNumber);
return(null);
}
/// <summary>
/// Initializes a new instance of the <see cref="CopyHeader"/> class.
/// </summary>
/// <param name="version">
/// The version.
/// </param>
/// <param name="copyStage">
/// The copy stage.
/// </param>
/// <param name="primaryReplicaId">
/// The primary replica id.
/// </param>
public CopyHeader(int version, CopyStage copyStage, long primaryReplicaId)
{
this.version = version;
this.copyStage = copyStage;
this.primaryReplicaId = primaryReplicaId;
}
/// <summary>
/// Copy operations are returned.
/// </summary>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <OperationData> GetNextAsync(CancellationToken cancellationToken)
{
var directory = this.copyProvider.WorkingDirectory;
// Take a snapshot of the metadata table on first Copy operation.
if (this.copySnapshotOfMetadataTable == null)
{
this.copySnapshotOfMetadataTable = await this.copyProvider.GetMetadataTableAsync().ConfigureAwait(false);
Diagnostics.Assert(
this.copySnapshotOfMetadataTable != null,
this.traceType,
"IStoreCopyProvider.GetMetadataTableAsync() returned a null metadata table.");
this.copySnapshotOfMetadataTableEnumerator = this.copySnapshotOfMetadataTable.Table.GetEnumerator();
}
// Send the copy protocol version first.
if (this.copyStage == CopyStage.Version)
{
FabricEvents.Events.CopyAsync(this.traceType, string.Format(System.Globalization.CultureInfo.InvariantCulture, "starting. directory:{0}", directory));
// Next copy stage.
this.copyStage = CopyStage.MetadataTable;
using (var memoryStream = new MemoryStream(sizeof(int) + sizeof(byte)))
using (var writer = new BinaryWriter(memoryStream))
{
// Write the copy protocol version number.
writer.Write(CopyManager.CopyProtocolVersion);
// Write a byte indicating the operation type is the copy protocol version.
writer.Write((byte)TStoreCopyOperation.Version);
// Send the version operation data.
FabricEvents.Events.CopyAsync(
this.traceType,
string.Format(System.Globalization.CultureInfo.InvariantCulture, "Version. version:{0} bytes:{1}", CopyManager.CopyProtocolVersion, memoryStream.Position));
return(new OperationData(new ArraySegment <byte>(memoryStream.GetBuffer(), 0, checked ((int)memoryStream.Position))));
}
}
// Send the metadata table next.
if (this.copyStage == CopyStage.MetadataTable)
{
// Consistency checks.
Diagnostics.Assert(this.copySnapshotOfMetadataTable != null, this.traceType, "Unexpected copy error. Master table to be copied is null.");
// Next copy stage.
if (this.copySnapshotOfMetadataTableEnumerator.MoveNext())
{
this.copyStage = CopyStage.KeyFile;
}
else
{
this.copyStage = CopyStage.Complete;
}
using (var memoryStream = new MemoryStream())
{
// Write the full metadata table (this will typically be small - even with 1000 tracked files, this will be under 64 KB).
await MetadataManager.WriteAsync(this.copySnapshotOfMetadataTable, memoryStream).ConfigureAwait(false);
using (var writer = new BinaryWriter(memoryStream))
{
// Write a byte indicating the operation type is the full metadata table.
writer.Write((byte)TStoreCopyOperation.MetadataTable);
// Send the metadata table operation data.
FabricEvents.Events.CopyAsync(
this.traceType,
string.Format(System.Globalization.CultureInfo.InvariantCulture, "Metadata table. directory:{0} bytes:{1} Total number of checkpoint files:{2}", directory, memoryStream.Position, this.copySnapshotOfMetadataTable.Table.Count));
return(new OperationData(new ArraySegment <byte>(memoryStream.GetBuffer(), 0, checked ((int)memoryStream.Position))));
}
}
}
// Send the key file.
if (this.copyStage == CopyStage.KeyFile)
{
var bytesRead = 0;
var shortFileName = this.copySnapshotOfMetadataTableEnumerator.Current.Value.FileName;
var dataFileName = Path.Combine(directory, shortFileName);
var keyCheckpointFile = dataFileName + KeyCheckpointFile.FileExtension;
// If we don't have the current file stream opened, this is the first table chunk.
if (this.currentFileStream == null)
{
Diagnostics.Assert(
FabricFile.Exists(keyCheckpointFile),
this.traceType,
"Unexpected copy error. Expected file does not exist: {0}",
keyCheckpointFile);
// Open the file with asynchronous flag and 4096 cache size (C# default).
FabricEvents.Events.CopyAsync(this.traceType, string.Format(System.Globalization.CultureInfo.InvariantCulture, "Opening key file. file:{0}", shortFileName));
// MCoskun: Default IoPriorityHint is used.
// Reason: We do not know whether this copy might be used to build a replica that is or will be required for write quorum.
this.currentFileStream = FabricFile.Open(keyCheckpointFile, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.Asynchronous);
// Send the start of file operation data.
this.perfCounterWriter.StartMeasurement();
bytesRead = await this.currentFileStream.ReadAsync(this.copyDataBuffer, 0, CopyChunkSize, cancellationToken).ConfigureAwait(false);
this.perfCounterWriter.StopMeasurement(bytesRead);
using (var stream = new MemoryStream(this.copyDataBuffer, writable: true))
using (var writer = new BinaryWriter(stream))
{
stream.Position = bytesRead;
writer.Write((int)this.copySnapshotOfMetadataTableEnumerator.Current.Value.FileId);
}
this.copyDataBuffer[bytesRead + sizeof(int)] = (byte)TStoreCopyOperation.StartKeyFile;
FabricEvents.Events.CopyAsync(
this.traceType,
string.Format(System.Globalization.CultureInfo.InvariantCulture, "StartKeyFile. file:{0} bytes:{1} totalFileSize:{2}", shortFileName, bytesRead + sizeof(int) + 1, this.currentFileStream.Length));
return(new OperationData(new ArraySegment <byte>(this.copyDataBuffer, 0, bytesRead + sizeof(int) + 1)));
}
// The start of the current file has been sent. Check if there are more chunks to be sent (if the stream is at the end, this will return zero).
this.perfCounterWriter.StartMeasurement();
bytesRead = await this.currentFileStream.ReadAsync(this.copyDataBuffer, 0, CopyChunkSize, cancellationToken).ConfigureAwait(false);
this.perfCounterWriter.StopMeasurement(bytesRead);
if (bytesRead > 0)
{
// Send the partial table file operation data.
this.copyDataBuffer[bytesRead] = (byte)TStoreCopyOperation.WriteKeyFile;
FabricEvents.Events.CopyAsync(this.traceType, string.Format(System.Globalization.CultureInfo.InvariantCulture, "WriteKeyFile. level:{0} bytes:{1} Position:{2}", shortFileName, bytesRead + 1, this.currentFileStream.Position));
return(new OperationData(new ArraySegment <byte>(this.copyDataBuffer, 0, bytesRead + 1)));
}
// There is no more data in the current file. Send the end of file marker, and prepare for the next copy stage.
this.currentFileStream.Dispose();
this.currentFileStream = null;
// Now move to the value file.
this.copyStage = CopyStage.ValueFile;
// Send the end of file operation data.
var endFileData = new ArraySegment <byte>(new[] { (byte)TStoreCopyOperation.EndKeyFile });
FabricEvents.Events.CopyAsync(this.traceType, string.Format(System.Globalization.CultureInfo.InvariantCulture, "EndKeyFile. file:{0}", shortFileName));
return(new OperationData(endFileData));
}
if (this.copyStage == CopyStage.ValueFile)
{
var bytesRead = 0;
var shortFileName = this.copySnapshotOfMetadataTableEnumerator.Current.Value.FileName;
var dataFileName = Path.Combine(directory, shortFileName);
var valueCheckpointFile = dataFileName + ValueCheckpointFile.FileExtension;
// If we don't have the current file opened, this is the first table chunk.
if (this.currentFileStream == null)
{
Diagnostics.Assert(
FabricFile.Exists(valueCheckpointFile),
this.traceType,
"Unexpected copy error. Expected file does not exist: {0}",
valueCheckpointFile);
// Open the file with asynchronous flag and 4096 cache size (C# default).
FabricEvents.Events.CopyAsync(this.traceType, string.Format(System.Globalization.CultureInfo.InvariantCulture, "Opening value file. directory:{0} file:{1}", directory, shortFileName));
// MCoskun: Default IoPriorityHint is used.
// Reason: We do not know whether this copy might be used to build a replica that is or will be required for write quorum.
this.currentFileStream = FabricFile.Open(
valueCheckpointFile,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
4096,
FileOptions.Asynchronous);
// Send the start of file operation data.
this.perfCounterWriter.StartMeasurement();
bytesRead = await this.currentFileStream.ReadAsync(this.copyDataBuffer, 0, CopyChunkSize, cancellationToken).ConfigureAwait(false);
this.perfCounterWriter.StopMeasurement(bytesRead);
using (var stream = new MemoryStream(this.copyDataBuffer, writable: true))
using (var writer = new BinaryWriter(stream))
{
stream.Position = bytesRead;
writer.Write((int)this.copySnapshotOfMetadataTableEnumerator.Current.Value.FileId);
}
this.copyDataBuffer[bytesRead + sizeof(int)] = (byte)TStoreCopyOperation.StartValueFile;
FabricEvents.Events.CopyAsync(
this.traceType,
string.Format(System.Globalization.CultureInfo.InvariantCulture, "StartValueFile. file:{0} bytes:{1} totalFileSize:{2}", shortFileName, bytesRead + sizeof(int) + 1, this.currentFileStream.Length));
return(new OperationData(new ArraySegment <byte>(this.copyDataBuffer, 0, bytesRead + sizeof(int) + 1)));
}
// The start of the current file was sent. Check if there are more chunks to be sent (if the stream is at the end, this will return zero).
this.perfCounterWriter.StartMeasurement();
bytesRead = await this.currentFileStream.ReadAsync(this.copyDataBuffer, 0, CopyChunkSize, cancellationToken);
this.perfCounterWriter.StopMeasurement(bytesRead);
if (bytesRead > 0)
{
this.copyDataBuffer[bytesRead] = (byte)TStoreCopyOperation.WriteValueFile;
FabricEvents.Events.CopyAsync(this.traceType, string.Format(System.Globalization.CultureInfo.InvariantCulture, "WriteValueFile. file:{0} bytes:{1} Position:{2}", shortFileName, bytesRead + 1, this.currentFileStream.Position));
return(new OperationData(new ArraySegment <byte>(this.copyDataBuffer, 0, bytesRead + 1)));
}
// There is no more data in the current file. Send the end of file marker, and prepare for the next copy stage.
this.currentFileStream.Dispose();
this.currentFileStream = null;
// Check if there are more files.
if (this.copySnapshotOfMetadataTableEnumerator.MoveNext())
{
// More files.
this.copyStage = CopyStage.KeyFile;
}
else
{
// No more files to be sent.
this.copyStage = CopyStage.Complete;
}
// Send the end of file operation data.
var endFileData = new ArraySegment <byte>(new[] { (byte)TStoreCopyOperation.EndValueFile });
FabricEvents.Events.CopyAsync(this.traceType, string.Format(System.Globalization.CultureInfo.InvariantCulture, "EndValueFile. file:{0}", shortFileName));
return(new OperationData(endFileData));
}
// Finally, send the "copy completed" marker.
if (this.copyStage == CopyStage.Complete)
{
this.perfCounterWriter.UpdatePerformanceCounters();
// Next copy stage.
this.copyStage = CopyStage.None;
// Indicate the copy operation is complete.
FabricEvents.Events.CopyAsync(
this.traceType,
string.Format(
System.Globalization.CultureInfo.InvariantCulture,
"completed. directory:{0}, diskread:{1}bytes/sec",
directory,
this.perfCounterWriter.AvgDiskTransferBytesPerSec));
var copyCompleteData = new ArraySegment <byte>(new[] { (byte)TStoreCopyOperation.Complete });
return(new OperationData(copyCompleteData));
}
// Finished copying. Dispose immediately to release resources/locks.
if (!this.disposed)
{
((IDisposable)this).Dispose();
}
return(null);
}
public static void CreateWindow()
{
CopyStage window = (CopyStage)GetWindow(typeof(CopyStage));
window.Show();
}
