private NativeRuntime.IFabricOperationDataStream GetCopyContextHelper()
{
IOperationDataStream copyContext = this.stateProvider.GetCopyContext();
AsyncEnumerateOperationDataBroker comCopyContext = AsyncEnumerateOperationDataBroker.ToNative(copyContext, this.OperationDataFactory);
return(comCopyContext);
}
internal LoggingReplicatorCopyStream(
ReplicatedLogManager replicatedLogManager,
IStateManager stateManager,
CheckpointManager checkpointManager,
Func <LogicalSequenceNumber, Task> waitForLogFlushUptoLsn,
long replicaId,
LogicalSequenceNumber uptoLsn,
IOperationDataStream copyContext,
ITracer tracer)
{
this.stateManager = stateManager;
this.checkpointManager = checkpointManager;
this.replicatedLogManager = replicatedLogManager;
this.replicaId = replicaId;
this.waitForLogFlushUptoLsn = waitForLogFlushUptoLsn;
this.uptoLsn = uptoLsn;
this.copyContext = copyContext;
this.targetReplicaId = 0;
this.targetProgressVector = null;
this.targetLogHeadEpoch = LogicalSequenceNumber.InvalidEpoch;
this.targetLogHeadLsn = LogicalSequenceNumber.InvalidLsn;
this.currentTargetLsn = LogicalSequenceNumber.InvalidLsn;
this.copyStage = CopyStage.CopyMetadata;
this.copyStateStream = null;
this.copiedRecordNumber = 0;
this.sourceStartingLsn = LogicalSequenceNumber.InvalidLsn;
this.targetStartingLsn = LogicalSequenceNumber.InvalidLsn;
this.logRecordsToCopy = null;
this.beginCheckpointRecord = null;
this.bw = new BinaryWriter(new MemoryStream());
this.isDisposed = false;
this.tracer = tracer;
}
private AsyncEnumerateOperationDataBroker(IOperationDataStream enumerator, OperationDataFactoryWrapper operationDataFactory)
{
Requires.Argument("enumerator", enumerator).NotNull();
Requires.Argument("operationDataFactory", operationDataFactory).NotNull();
this.enumerator = enumerator;
this.operationDataFactory = operationDataFactory;
}
private NativeRuntime.IFabricOperationDataStream GetCopyStateHelper(long uptoSequenceNumber, NativeRuntime.IFabricOperationDataStream comCopyContext)
{
IOperationDataStream copyState = null;
AsyncEnumerateOperationData copyContext = comCopyContext == null ? null : new AsyncEnumerateOperationData(comCopyContext);
copyState = this.stateProvider.GetCopyState(uptoSequenceNumber, copyContext);
return(AsyncEnumerateOperationDataBroker.ToNative(copyState, this.OperationDataFactory));
}
public IOperationDataStream GetCopyState(long upToSequenceNumber, IOperationDataStream copyContext)
{
LogMessage(nameof(GetCopyState));
var startData = copyContext.GetNextAsync(CancellationToken.None).GetAwaiter().GetResult();
var startLsn = BitConverter.ToInt64(startData[0].Array, 0);
return(new CopyStateOperationStream(log, startLsn, upToSequenceNumber));
}
/// <summary>
/// Gets operation data from state providers.
/// </summary>
public async Task <OperationData> GetNextAsync(CancellationToken cancellationToken)
{
try
{
while (this.currentIndex < this.operationDataStreamCollection.Count)
{
StateProviderIdentifier stateProviderIdentifier = this.operationDataStreamCollection[this.currentIndex].Item1;
IOperationDataStream operationDataStream = this.operationDataStreamCollection[this.currentIndex].Item2;
OperationData operationData = await operationDataStream.GetNextAsync(cancellationToken).ConfigureAwait(false);
if (operationData == null)
{
++this.currentIndex;
continue;
}
// Reserve the List space with expected size: Version, ID, OperationData
List <ArraySegment <byte> > namedBytes = new List <ArraySegment <byte> >(2 + operationData.Count);
byte[] versionAsBytes = BitConverter.GetBytes(this.version);
namedBytes.Add(new ArraySegment <byte>(versionAsBytes));
long stateProviderId = stateProviderIdentifier.StateProviderId;
byte[] idAsBytes = BitConverter.GetBytes(stateProviderId);
namedBytes.Add(new ArraySegment <byte>(idAsBytes));
foreach (ArraySegment <byte> roll in operationData)
{
namedBytes.Add(roll);
}
return(new OperationData(namedBytes));
}
FabricEvents.Events.NamedOperationDataCollectionGetNext(this.traceType);
return(null);
}
catch (Exception e)
{
string msg = string.Format(
CultureInfo.InvariantCulture,
"OperationDataStreamCollection Count: {0}, CurrentIndex: {1}.",
this.operationDataStreamCollection.Count,
this.currentIndex);
StateManagerStructuredEvents.TraceException(
this.traceType,
"NamedOperationDataCollection.GetNextAsync.",
msg,
e);
throw;
}
}
public static AsyncEnumerateOperationDataBroker ToNative(IOperationDataStream enumerator, OperationDataFactoryWrapper operationDataFactory)
{
if (enumerator == null)
{
return(null);
}
else
{
var native = new AsyncEnumerateOperationDataBroker(enumerator, operationDataFactory);
return(native);
}
}
/// <summary>
/// Gets the copy stream from the primary replica.
/// </summary>
/// <param name="upToSequenceNumber">The upper-bound of the data to return in the copy stream.</param>
/// <param name="copyContext">The copy context from the secondary which will consume the returned copy stream.</param>
/// <returns></returns>
public IOperationDataStream GetCopyState(long upToSequenceNumber, IOperationDataStream copyContext)
{
this.logger.Log($"GetCopyState({upToSequenceNumber})");
return(new CopyStateStream(
upToSequenceNumber,
copyContext,
this.progressVector,
this.operationLog,
this,
this.logger,
this.serializer));
}
public MockStateReplicator(
DataContractSerializer replicationSerializer,
IOperationDataStream copyStateEnumerator,
List <List <ActorStateDataWrapper> > replicationList,
Task replicationStreamReadySignal)
{
this._copyStream = new MockCopyOperationStream(copyStateEnumerator);
this._replicationStream = new MockReplicationOperationStream(
replicationSerializer,
replicationList,
replicationStreamReadySignal);
}
/// <summary>
/// Adds operation data stream to the collection.
/// </summary>
public void Add(StateProviderIdentifier stateProviderIdentifier, IOperationDataStream operationDataStream)
{
Utility.Assert(
stateProviderIdentifier != null,
"{0}: Named operation data cannot be empty in named operation data collection.",
this.traceType);
Utility.Assert(
operationDataStream != null,
"{0}: Operation data stream cannot be null.",
this.traceType);
this.operationDataStreamCollection.Add(
new Tuple <StateProviderIdentifier, IOperationDataStream>(stateProviderIdentifier, operationDataStream));
}
public CopyStateStream(
long upToSequenceNumber,
IOperationDataStream copyContext,
ProgressVector localProgress,
LogFile operationLog,
IReplicationNotifier replicationNotifier,
Logger logger,
Serializer serializer)
{
this.upToSequenceNumber = upToSequenceNumber;
this.copyContext = copyContext;
this.localProgress = localProgress;
this.logger = logger;
this.serializer = serializer;
this.log = operationLog;
this.replicationNotifier = replicationNotifier;
this.records = operationLog.GetAllRecords().GetEnumerator();
}
public CopyStateStream(
long upToSequenceNumber,
IOperationDataStream copyContext,
ProgressVector localProgress,
LogFile operationLog,
IReplicationNotifier replicationNotifier,
Logger logger,
Serializer serializer)
{
this.upToSequenceNumber = upToSequenceNumber;
this.copyContext = copyContext;
this.localProgress = localProgress;
this.logger = logger;
this.serializer = serializer;
this.log = operationLog;
this.replicationNotifier = replicationNotifier;
this.records = operationLog.GetAllRecords().GetEnumerator();
}
/// <summary>
/// Gets copy state.
/// </summary>
IOperationDataStream IStateProvider.GetCopyState(long upToSequenceNumber, IOperationDataStream copyContext)
{
return(this.stateManager.GetCopyState(upToSequenceNumber, copyContext));
}
public IOperationDataStream GetCopyState(long sequenceNumber, IOperationDataStream copyContext)
{
return(null);
}
private async Task <OperationData> GetNextAsyncSafe(CancellationToken cancellationToken)
{
OperationData data;
if (this.copyStage == CopyStage.CopyMetadata)
{
data = await this.copyContext.GetNextAsync(CancellationToken.None).ConfigureAwait(false);
Utility.Assert(data.Count == 1, "data.Count == 1");
var bytes = data[0].Array;
using (var br = new BinaryReader(new MemoryStream(bytes, data[0].Offset, data[0].Count)))
{
this.targetReplicaId = br.ReadInt64();
this.targetProgressVector = new ProgressVector();
this.targetProgressVector.Read(br, false);
var targetLogHeadDatalossNumber = br.ReadInt64();
var targetLogHeadConfigurationNumber = br.ReadInt64();
this.targetLogHeadEpoch = new Epoch(targetLogHeadDatalossNumber, targetLogHeadConfigurationNumber);
this.targetLogHeadLsn = new LogicalSequenceNumber(br.ReadInt64());
this.currentTargetLsn = new LogicalSequenceNumber(br.ReadInt64());
this.latestRecoveredAtomicRedoOperationLsn = br.ReadInt64();
}
var message =
string.Format(
CultureInfo.InvariantCulture,
"Target Replica Copy Context: {0}" + Environment.NewLine
+ " TargetLogHeadEpoch: {1},{2} TargetLogHeadLsn: {3} TargetLogTailEpoch: {4},{5} TargetLogTailLSN: {6}"
+ Environment.NewLine
+ " SourceLogHeadEpoch: {7},{8} SourceLogHeadLsn: {9} SourceLogTailEpoch: {10},{11} SourceLogTailLSN: {12}"
+ Environment.NewLine + " Target ProgressVector: {13}" + Environment.NewLine
+ " Source ProgressVector: {14}" + Environment.NewLine
+ " LastRecoveredAtomicRedoOperationLsn: {15}",
this.targetReplicaId,
this.targetLogHeadEpoch.DataLossNumber,
this.targetLogHeadEpoch.ConfigurationNumber,
this.targetLogHeadLsn.LSN,
this.targetProgressVector.LastProgressVectorEntry.Epoch.DataLossNumber,
this.targetProgressVector.LastProgressVectorEntry.Epoch.ConfigurationNumber,
this.currentTargetLsn.LSN,
this.replicatedLogManager.CurrentLogHeadRecord.CurrentEpoch.DataLossNumber,
this.replicatedLogManager.CurrentLogHeadRecord.CurrentEpoch.ConfigurationNumber,
this.replicatedLogManager.CurrentLogHeadRecord.Lsn.LSN,
this.replicatedLogManager.CurrentLogTailEpoch.DataLossNumber,
this.replicatedLogManager.CurrentLogTailEpoch.ConfigurationNumber,
this.replicatedLogManager.CurrentLogTailLsn.LSN,
this.targetProgressVector.ToString(Constants.ProgressVectorMaxStringSizeInKb / 2),
this.replicatedLogManager.ProgressVector.ToString(Constants.ProgressVectorMaxStringSizeInKb / 2),
this.latestRecoveredAtomicRedoOperationLsn);
FabricEvents.Events.CopyStreamMetadata(tracer.Type, message);
await this.waitForLogFlushUptoLsn(this.uptoLsn).ConfigureAwait(false);
CopyMode copyMode;
await this.checkpointManager.AcquireBackupAndCopyConsistencyLockAsync("Copy").ConfigureAwait(false);
try
{
copyMode = this.checkpointManager.GetLogRecordsToCopy(
this.targetProgressVector,
this.targetLogHeadEpoch,
this.targetLogHeadLsn,
this.currentTargetLsn,
this.latestRecoveredAtomicRedoOperationLsn,
this.targetReplicaId,
out this.sourceStartingLsn,
out this.targetStartingLsn,
out this.logRecordsToCopy,
out this.beginCheckpointRecord);
}
finally
{
this.checkpointManager.ReleaseBackupAndCopyConsistencyLock("Copy");
}
CopyStage copyStageToWrite;
if (copyMode == CopyMode.None)
{
this.copyStage = CopyStage.CopyNone;
copyStageToWrite = CopyStage.CopyNone;
}
else if (copyMode == CopyMode.Full)
{
this.copyStage = CopyStage.CopyState;
copyStageToWrite = CopyStage.CopyState;
this.copyStateStream = this.stateManager.GetCurrentState();
}
else if ((copyMode & CopyMode.FalseProgress) != 0)
{
Utility.Assert(
this.sourceStartingLsn <= this.targetStartingLsn,
"this.sourceStartingLsn ({0}) <= this.targetStartingLsn ({1})",
this.sourceStartingLsn, this.targetStartingLsn);
this.copyStage = CopyStage.CopyFalseProgress;
copyStageToWrite = CopyStage.CopyFalseProgress;
}
else
{
this.copyStage = CopyStage.CopyScanToStartingLsn;
copyStageToWrite = CopyStage.CopyLog;
}
return(this.GetCopyMetadata(copyStageToWrite));
}
if (this.copyStage == CopyStage.CopyState)
{
data = await this.copyStateStream.GetNextAsync(cancellationToken).ConfigureAwait(false);
if (data != null)
{
data.Add(CopyStateOperation);
FabricEvents.Events.CopyStreamGetNextNoise(
tracer.Type,
"Copying State Operation: " + this.copiedRecordNumber);
++this.copiedRecordNumber;
return(data);
}
var disposableCopyStateStream = this.copyStateStream as IDisposable;
if (disposableCopyStateStream != null)
{
disposableCopyStateStream.Dispose();
}
if (this.uptoLsn < beginCheckpointRecord.LastStableLsn)
{
// Ensure we copy records up to the stable LSN of the begin checkpoint record as part of the copy stream
// so that the idle can initiate a checkpoint and apply it as part of the copy pump itself
// Not doing the above will mean that the idle can get promoted to active even before the checkpoint is completed
// uptolsn is inclusive LSN
// BUG: RDBug #9269022:Replicator must rename copy log before changing role to active secondary during full builds
this.uptoLsn = beginCheckpointRecord.LastStableLsn;
var trace = "FullCopy: Copying Logs from " + this.sourceStartingLsn.LSN
+ " to BC.LastStableLsn: " + beginCheckpointRecord.LastStableLsn.LSN + " as the upto lsn: " + this.uptoLsn + " was smaller than the checkpoint stable lsn";
FabricEvents.Events.CopyStreamGetNext(tracer.Type, trace);
}
data = this.GetCopyStateMetadata();
this.copyStage = CopyStage.CopyLog;
this.copiedRecordNumber = 0;
return(data);
}
if (this.copyStage == CopyStage.CopyFalseProgress)
{
this.copyStage = CopyStage.CopyScanToStartingLsn;
this.bw.BaseStream.Position = 0;
this.bw.Write(this.sourceStartingLsn.LSN);
var stream = this.bw.BaseStream as MemoryStream;
data = new OperationData(new ArraySegment <byte>(stream.GetBuffer(), 0, (int)stream.Position));
data.Add(CopyFalseProgressOperation);
var trace = "Copying False Progress. SourceStartingLSN: " + this.sourceStartingLsn.LSN
+ " TargetStartingLSN: " + this.targetStartingLsn.LSN;
FabricEvents.Events.CopyStreamGetNext(tracer.Type, trace);
return(data);
}
LogRecord record;
if (this.copyStage == CopyStage.CopyScanToStartingLsn)
{
var startingLsn = (this.sourceStartingLsn < this.targetStartingLsn)
? this.sourceStartingLsn
: this.targetStartingLsn;
do
{
var hasMoved = await this.logRecordsToCopy.MoveNextAsync(cancellationToken).ConfigureAwait(false);
if (hasMoved == false)
{
goto Finish;
}
record = this.logRecordsToCopy.Current;
Utility.Assert(record.Lsn <= startingLsn, "record.Lsn <= startingLsn");
} while (record.Lsn < startingLsn);
// The log stream is positioned at the end of startingLsn at this point. The enumerator Current is pointing to the "startingLsn"
this.copyStage = CopyStage.CopyLog;
}
if (this.copyStage == CopyStage.CopyLog)
{
do
{
// This will start copying after the startingLsn record as expected
var hasMoved = await this.logRecordsToCopy.MoveNextAsync(cancellationToken).ConfigureAwait(false);
if (hasMoved == false)
{
goto Finish;
}
record = this.logRecordsToCopy.Current;
} while (record is PhysicalLogRecord);
if (record.Lsn > this.uptoLsn)
{
goto Finish;
}
var logicalLogRecord = record as LogicalLogRecord;
Utility.Assert(logicalLogRecord != null, "Must be a logical log record");
data = logicalLogRecord.ToOperationData(this.bw);
data.Add(CopyLogOperation);
var trace = "Copying log record. LogRecordNumber: " + this.copiedRecordNumber + "Record Type: "
+ record.RecordType + " LSN: " + record.Lsn.LSN;
FabricEvents.Events.CopyStreamGetNextNoise(tracer.Type, trace);
++this.copiedRecordNumber;
return(data);
}
Finish:
if (this.logRecordsToCopy != null)
{
this.logRecordsToCopy.Dispose();
}
FabricEvents.Events.CopyStreamFinished(tracer.Type, this.copiedRecordNumber);
return(null);
}
public IOperationDataStream GetCopyState(long sequenceNumber, IOperationDataStream copyContext)
{
return(new AsyncEnumOpData());
}
/// <summary>
/// Gets the copy stream from the primary replica.
/// </summary>
/// <param name="upToSequenceNumber">The upper-bound of the data to return in the copy stream.</param>
/// <param name="copyContext">The copy context from the secondary which will consume the returned copy stream.</param>
/// <returns></returns>
public IOperationDataStream GetCopyState(long upToSequenceNumber, IOperationDataStream copyContext)
{
this.logger.Log($"GetCopyState({upToSequenceNumber})");
return new CopyStateStream(
upToSequenceNumber,
copyContext,
this.progressVector,
this.operationLog,
this,
this.logger,
this.serializer);
}
public IOperationDataStream GetCopyState(long upToSequenceNumber, IOperationDataStream copyContext)
{
this.TraceInfo("Executing GetCopyState.");
return(NullOperationDataStream.Instance);
}
IOperationDataStream IStateProvider.GetCopyState(long upToSequenceNumber, IOperationDataStream copyContext)
{
return(new NullOperationDataStream());
}
public MockCopyOperationStream(IOperationDataStream copyStateEnumerator)
{
this._copyStateEnumerator = copyStateEnumerator;
this._sequenceNumber = 0;
}
/// <inheritdoc />
IOperationDataStream IStateProvider.GetCopyState(long upToSequenceNumber, IOperationDataStream copyContext)
{
// TODO: send the data from primary to secondary
return(new VegaOperationDataStream(upToSequenceNumber));
}
public virtual IOperationDataStream GetCopyState(long sequenceNumber, IOperationDataStream copyContext)
{
throw new NotImplementedException();
}
