private NativeRuntime.IFabricOperationDataStream GetCopyContextHelper() { IOperationDataStream copyContext = this.stateProvider.GetCopyContext(); AsyncEnumerateOperationDataBroker comCopyContext = AsyncEnumerateOperationDataBroker.ToNative(copyContext, this.OperationDataFactory); return(comCopyContext); }
internal LoggingReplicatorCopyStream( ReplicatedLogManager replicatedLogManager, IStateManager stateManager, CheckpointManager checkpointManager, Func <LogicalSequenceNumber, Task> waitForLogFlushUptoLsn, long replicaId, LogicalSequenceNumber uptoLsn, IOperationDataStream copyContext, ITracer tracer) { this.stateManager = stateManager; this.checkpointManager = checkpointManager; this.replicatedLogManager = replicatedLogManager; this.replicaId = replicaId; this.waitForLogFlushUptoLsn = waitForLogFlushUptoLsn; this.uptoLsn = uptoLsn; this.copyContext = copyContext; this.targetReplicaId = 0; this.targetProgressVector = null; this.targetLogHeadEpoch = LogicalSequenceNumber.InvalidEpoch; this.targetLogHeadLsn = LogicalSequenceNumber.InvalidLsn; this.currentTargetLsn = LogicalSequenceNumber.InvalidLsn; this.copyStage = CopyStage.CopyMetadata; this.copyStateStream = null; this.copiedRecordNumber = 0; this.sourceStartingLsn = LogicalSequenceNumber.InvalidLsn; this.targetStartingLsn = LogicalSequenceNumber.InvalidLsn; this.logRecordsToCopy = null; this.beginCheckpointRecord = null; this.bw = new BinaryWriter(new MemoryStream()); this.isDisposed = false; this.tracer = tracer; }
private AsyncEnumerateOperationDataBroker(IOperationDataStream enumerator, OperationDataFactoryWrapper operationDataFactory) { Requires.Argument("enumerator", enumerator).NotNull(); Requires.Argument("operationDataFactory", operationDataFactory).NotNull(); this.enumerator = enumerator; this.operationDataFactory = operationDataFactory; }
private NativeRuntime.IFabricOperationDataStream GetCopyStateHelper(long uptoSequenceNumber, NativeRuntime.IFabricOperationDataStream comCopyContext) { IOperationDataStream copyState = null; AsyncEnumerateOperationData copyContext = comCopyContext == null ? null : new AsyncEnumerateOperationData(comCopyContext); copyState = this.stateProvider.GetCopyState(uptoSequenceNumber, copyContext); return(AsyncEnumerateOperationDataBroker.ToNative(copyState, this.OperationDataFactory)); }
public IOperationDataStream GetCopyState(long upToSequenceNumber, IOperationDataStream copyContext) { LogMessage(nameof(GetCopyState)); var startData = copyContext.GetNextAsync(CancellationToken.None).GetAwaiter().GetResult(); var startLsn = BitConverter.ToInt64(startData[0].Array, 0); return(new CopyStateOperationStream(log, startLsn, upToSequenceNumber)); }
/// <summary> /// Gets operation data from state providers. /// </summary> public async Task <OperationData> GetNextAsync(CancellationToken cancellationToken) { try { while (this.currentIndex < this.operationDataStreamCollection.Count) { StateProviderIdentifier stateProviderIdentifier = this.operationDataStreamCollection[this.currentIndex].Item1; IOperationDataStream operationDataStream = this.operationDataStreamCollection[this.currentIndex].Item2; OperationData operationData = await operationDataStream.GetNextAsync(cancellationToken).ConfigureAwait(false); if (operationData == null) { ++this.currentIndex; continue; } // Reserve the List space with expected size: Version, ID, OperationData List <ArraySegment <byte> > namedBytes = new List <ArraySegment <byte> >(2 + operationData.Count); byte[] versionAsBytes = BitConverter.GetBytes(this.version); namedBytes.Add(new ArraySegment <byte>(versionAsBytes)); long stateProviderId = stateProviderIdentifier.StateProviderId; byte[] idAsBytes = BitConverter.GetBytes(stateProviderId); namedBytes.Add(new ArraySegment <byte>(idAsBytes)); foreach (ArraySegment <byte> roll in operationData) { namedBytes.Add(roll); } return(new OperationData(namedBytes)); } FabricEvents.Events.NamedOperationDataCollectionGetNext(this.traceType); return(null); } catch (Exception e) { string msg = string.Format( CultureInfo.InvariantCulture, "OperationDataStreamCollection Count: {0}, CurrentIndex: {1}.", this.operationDataStreamCollection.Count, this.currentIndex); StateManagerStructuredEvents.TraceException( this.traceType, "NamedOperationDataCollection.GetNextAsync.", msg, e); throw; } }
public static AsyncEnumerateOperationDataBroker ToNative(IOperationDataStream enumerator, OperationDataFactoryWrapper operationDataFactory) { if (enumerator == null) { return(null); } else { var native = new AsyncEnumerateOperationDataBroker(enumerator, operationDataFactory); return(native); } }
/// <summary> /// Gets the copy stream from the primary replica. /// </summary> /// <param name="upToSequenceNumber">The upper-bound of the data to return in the copy stream.</param> /// <param name="copyContext">The copy context from the secondary which will consume the returned copy stream.</param> /// <returns></returns> public IOperationDataStream GetCopyState(long upToSequenceNumber, IOperationDataStream copyContext) { this.logger.Log($"GetCopyState({upToSequenceNumber})"); return(new CopyStateStream( upToSequenceNumber, copyContext, this.progressVector, this.operationLog, this, this.logger, this.serializer)); }
public MockStateReplicator( DataContractSerializer replicationSerializer, IOperationDataStream copyStateEnumerator, List <List <ActorStateDataWrapper> > replicationList, Task replicationStreamReadySignal) { this._copyStream = new MockCopyOperationStream(copyStateEnumerator); this._replicationStream = new MockReplicationOperationStream( replicationSerializer, replicationList, replicationStreamReadySignal); }
/// <summary> /// Adds operation data stream to the collection. /// </summary> public void Add(StateProviderIdentifier stateProviderIdentifier, IOperationDataStream operationDataStream) { Utility.Assert( stateProviderIdentifier != null, "{0}: Named operation data cannot be empty in named operation data collection.", this.traceType); Utility.Assert( operationDataStream != null, "{0}: Operation data stream cannot be null.", this.traceType); this.operationDataStreamCollection.Add( new Tuple <StateProviderIdentifier, IOperationDataStream>(stateProviderIdentifier, operationDataStream)); }
public CopyStateStream( long upToSequenceNumber, IOperationDataStream copyContext, ProgressVector localProgress, LogFile operationLog, IReplicationNotifier replicationNotifier, Logger logger, Serializer serializer) { this.upToSequenceNumber = upToSequenceNumber; this.copyContext = copyContext; this.localProgress = localProgress; this.logger = logger; this.serializer = serializer; this.log = operationLog; this.replicationNotifier = replicationNotifier; this.records = operationLog.GetAllRecords().GetEnumerator(); }
public CopyStateStream( long upToSequenceNumber, IOperationDataStream copyContext, ProgressVector localProgress, LogFile operationLog, IReplicationNotifier replicationNotifier, Logger logger, Serializer serializer) { this.upToSequenceNumber = upToSequenceNumber; this.copyContext = copyContext; this.localProgress = localProgress; this.logger = logger; this.serializer = serializer; this.log = operationLog; this.replicationNotifier = replicationNotifier; this.records = operationLog.GetAllRecords().GetEnumerator(); }
/// <summary> /// Gets copy state. /// </summary> IOperationDataStream IStateProvider.GetCopyState(long upToSequenceNumber, IOperationDataStream copyContext) { return(this.stateManager.GetCopyState(upToSequenceNumber, copyContext)); }
public IOperationDataStream GetCopyState(long sequenceNumber, IOperationDataStream copyContext) { return(null); }
private async Task <OperationData> GetNextAsyncSafe(CancellationToken cancellationToken) { OperationData data; if (this.copyStage == CopyStage.CopyMetadata) { data = await this.copyContext.GetNextAsync(CancellationToken.None).ConfigureAwait(false); Utility.Assert(data.Count == 1, "data.Count == 1"); var bytes = data[0].Array; using (var br = new BinaryReader(new MemoryStream(bytes, data[0].Offset, data[0].Count))) { this.targetReplicaId = br.ReadInt64(); this.targetProgressVector = new ProgressVector(); this.targetProgressVector.Read(br, false); var targetLogHeadDatalossNumber = br.ReadInt64(); var targetLogHeadConfigurationNumber = br.ReadInt64(); this.targetLogHeadEpoch = new Epoch(targetLogHeadDatalossNumber, targetLogHeadConfigurationNumber); this.targetLogHeadLsn = new LogicalSequenceNumber(br.ReadInt64()); this.currentTargetLsn = new LogicalSequenceNumber(br.ReadInt64()); this.latestRecoveredAtomicRedoOperationLsn = br.ReadInt64(); } var message = string.Format( CultureInfo.InvariantCulture, "Target Replica Copy Context: {0}" + Environment.NewLine + " TargetLogHeadEpoch: {1},{2} TargetLogHeadLsn: {3} TargetLogTailEpoch: {4},{5} TargetLogTailLSN: {6}" + Environment.NewLine + " SourceLogHeadEpoch: {7},{8} SourceLogHeadLsn: {9} SourceLogTailEpoch: {10},{11} SourceLogTailLSN: {12}" + Environment.NewLine + " Target ProgressVector: {13}" + Environment.NewLine + " Source ProgressVector: {14}" + Environment.NewLine + " LastRecoveredAtomicRedoOperationLsn: {15}", this.targetReplicaId, this.targetLogHeadEpoch.DataLossNumber, this.targetLogHeadEpoch.ConfigurationNumber, this.targetLogHeadLsn.LSN, this.targetProgressVector.LastProgressVectorEntry.Epoch.DataLossNumber, this.targetProgressVector.LastProgressVectorEntry.Epoch.ConfigurationNumber, this.currentTargetLsn.LSN, this.replicatedLogManager.CurrentLogHeadRecord.CurrentEpoch.DataLossNumber, this.replicatedLogManager.CurrentLogHeadRecord.CurrentEpoch.ConfigurationNumber, this.replicatedLogManager.CurrentLogHeadRecord.Lsn.LSN, this.replicatedLogManager.CurrentLogTailEpoch.DataLossNumber, this.replicatedLogManager.CurrentLogTailEpoch.ConfigurationNumber, this.replicatedLogManager.CurrentLogTailLsn.LSN, this.targetProgressVector.ToString(Constants.ProgressVectorMaxStringSizeInKb / 2), this.replicatedLogManager.ProgressVector.ToString(Constants.ProgressVectorMaxStringSizeInKb / 2), this.latestRecoveredAtomicRedoOperationLsn); FabricEvents.Events.CopyStreamMetadata(tracer.Type, message); await this.waitForLogFlushUptoLsn(this.uptoLsn).ConfigureAwait(false); CopyMode copyMode; await this.checkpointManager.AcquireBackupAndCopyConsistencyLockAsync("Copy").ConfigureAwait(false); try { copyMode = this.checkpointManager.GetLogRecordsToCopy( this.targetProgressVector, this.targetLogHeadEpoch, this.targetLogHeadLsn, this.currentTargetLsn, this.latestRecoveredAtomicRedoOperationLsn, this.targetReplicaId, out this.sourceStartingLsn, out this.targetStartingLsn, out this.logRecordsToCopy, out this.beginCheckpointRecord); } finally { this.checkpointManager.ReleaseBackupAndCopyConsistencyLock("Copy"); } CopyStage copyStageToWrite; if (copyMode == CopyMode.None) { this.copyStage = CopyStage.CopyNone; copyStageToWrite = CopyStage.CopyNone; } else if (copyMode == CopyMode.Full) { this.copyStage = CopyStage.CopyState; copyStageToWrite = CopyStage.CopyState; this.copyStateStream = this.stateManager.GetCurrentState(); } else if ((copyMode & CopyMode.FalseProgress) != 0) { Utility.Assert( this.sourceStartingLsn <= this.targetStartingLsn, "this.sourceStartingLsn ({0}) <= this.targetStartingLsn ({1})", this.sourceStartingLsn, this.targetStartingLsn); this.copyStage = CopyStage.CopyFalseProgress; copyStageToWrite = CopyStage.CopyFalseProgress; } else { this.copyStage = CopyStage.CopyScanToStartingLsn; copyStageToWrite = CopyStage.CopyLog; } return(this.GetCopyMetadata(copyStageToWrite)); } if (this.copyStage == CopyStage.CopyState) { data = await this.copyStateStream.GetNextAsync(cancellationToken).ConfigureAwait(false); if (data != null) { data.Add(CopyStateOperation); FabricEvents.Events.CopyStreamGetNextNoise( tracer.Type, "Copying State Operation: " + this.copiedRecordNumber); ++this.copiedRecordNumber; return(data); } var disposableCopyStateStream = this.copyStateStream as IDisposable; if (disposableCopyStateStream != null) { disposableCopyStateStream.Dispose(); } if (this.uptoLsn < beginCheckpointRecord.LastStableLsn) { // Ensure we copy records up to the stable LSN of the begin checkpoint record as part of the copy stream // so that the idle can initiate a checkpoint and apply it as part of the copy pump itself // Not doing the above will mean that the idle can get promoted to active even before the checkpoint is completed // uptolsn is inclusive LSN // BUG: RDBug #9269022:Replicator must rename copy log before changing role to active secondary during full builds this.uptoLsn = beginCheckpointRecord.LastStableLsn; var trace = "FullCopy: Copying Logs from " + this.sourceStartingLsn.LSN + " to BC.LastStableLsn: " + beginCheckpointRecord.LastStableLsn.LSN + " as the upto lsn: " + this.uptoLsn + " was smaller than the checkpoint stable lsn"; FabricEvents.Events.CopyStreamGetNext(tracer.Type, trace); } data = this.GetCopyStateMetadata(); this.copyStage = CopyStage.CopyLog; this.copiedRecordNumber = 0; return(data); } if (this.copyStage == CopyStage.CopyFalseProgress) { this.copyStage = CopyStage.CopyScanToStartingLsn; this.bw.BaseStream.Position = 0; this.bw.Write(this.sourceStartingLsn.LSN); var stream = this.bw.BaseStream as MemoryStream; data = new OperationData(new ArraySegment <byte>(stream.GetBuffer(), 0, (int)stream.Position)); data.Add(CopyFalseProgressOperation); var trace = "Copying False Progress. SourceStartingLSN: " + this.sourceStartingLsn.LSN + " TargetStartingLSN: " + this.targetStartingLsn.LSN; FabricEvents.Events.CopyStreamGetNext(tracer.Type, trace); return(data); } LogRecord record; if (this.copyStage == CopyStage.CopyScanToStartingLsn) { var startingLsn = (this.sourceStartingLsn < this.targetStartingLsn) ? this.sourceStartingLsn : this.targetStartingLsn; do { var hasMoved = await this.logRecordsToCopy.MoveNextAsync(cancellationToken).ConfigureAwait(false); if (hasMoved == false) { goto Finish; } record = this.logRecordsToCopy.Current; Utility.Assert(record.Lsn <= startingLsn, "record.Lsn <= startingLsn"); } while (record.Lsn < startingLsn); // The log stream is positioned at the end of startingLsn at this point. The enumerator Current is pointing to the "startingLsn" this.copyStage = CopyStage.CopyLog; } if (this.copyStage == CopyStage.CopyLog) { do { // This will start copying after the startingLsn record as expected var hasMoved = await this.logRecordsToCopy.MoveNextAsync(cancellationToken).ConfigureAwait(false); if (hasMoved == false) { goto Finish; } record = this.logRecordsToCopy.Current; } while (record is PhysicalLogRecord); if (record.Lsn > this.uptoLsn) { goto Finish; } var logicalLogRecord = record as LogicalLogRecord; Utility.Assert(logicalLogRecord != null, "Must be a logical log record"); data = logicalLogRecord.ToOperationData(this.bw); data.Add(CopyLogOperation); var trace = "Copying log record. LogRecordNumber: " + this.copiedRecordNumber + "Record Type: " + record.RecordType + " LSN: " + record.Lsn.LSN; FabricEvents.Events.CopyStreamGetNextNoise(tracer.Type, trace); ++this.copiedRecordNumber; return(data); } Finish: if (this.logRecordsToCopy != null) { this.logRecordsToCopy.Dispose(); } FabricEvents.Events.CopyStreamFinished(tracer.Type, this.copiedRecordNumber); return(null); }
public IOperationDataStream GetCopyState(long sequenceNumber, IOperationDataStream copyContext) { return(new AsyncEnumOpData()); }
/// <summary> /// Gets the copy stream from the primary replica. /// </summary> /// <param name="upToSequenceNumber">The upper-bound of the data to return in the copy stream.</param> /// <param name="copyContext">The copy context from the secondary which will consume the returned copy stream.</param> /// <returns></returns> public IOperationDataStream GetCopyState(long upToSequenceNumber, IOperationDataStream copyContext) { this.logger.Log($"GetCopyState({upToSequenceNumber})"); return new CopyStateStream( upToSequenceNumber, copyContext, this.progressVector, this.operationLog, this, this.logger, this.serializer); }
public IOperationDataStream GetCopyState(long upToSequenceNumber, IOperationDataStream copyContext) { this.TraceInfo("Executing GetCopyState."); return(NullOperationDataStream.Instance); }
IOperationDataStream IStateProvider.GetCopyState(long upToSequenceNumber, IOperationDataStream copyContext) { return(new NullOperationDataStream()); }
public MockCopyOperationStream(IOperationDataStream copyStateEnumerator) { this._copyStateEnumerator = copyStateEnumerator; this._sequenceNumber = 0; }
/// <inheritdoc /> IOperationDataStream IStateProvider.GetCopyState(long upToSequenceNumber, IOperationDataStream copyContext) { // TODO: send the data from primary to secondary return(new VegaOperationDataStream(upToSequenceNumber)); }
public virtual IOperationDataStream GetCopyState(long sequenceNumber, IOperationDataStream copyContext) { throw new NotImplementedException(); }