Beispiel #1
0
 internal LoggingReplicatorCopyStream(
     ReplicatedLogManager replicatedLogManager,
     IStateManager stateManager,
     CheckpointManager checkpointManager,
     Func <LogicalSequenceNumber, Task> waitForLogFlushUptoLsn,
     long replicaId,
     LogicalSequenceNumber uptoLsn,
     IOperationDataStream copyContext,
     ITracer tracer)
 {
     this.stateManager           = stateManager;
     this.checkpointManager      = checkpointManager;
     this.replicatedLogManager   = replicatedLogManager;
     this.replicaId              = replicaId;
     this.waitForLogFlushUptoLsn = waitForLogFlushUptoLsn;
     this.uptoLsn               = uptoLsn;
     this.copyContext           = copyContext;
     this.targetReplicaId       = 0;
     this.targetProgressVector  = null;
     this.targetLogHeadEpoch    = LogicalSequenceNumber.InvalidEpoch;
     this.targetLogHeadLsn      = LogicalSequenceNumber.InvalidLsn;
     this.currentTargetLsn      = LogicalSequenceNumber.InvalidLsn;
     this.copyStage             = CopyStage.CopyMetadata;
     this.copyStateStream       = null;
     this.copiedRecordNumber    = 0;
     this.sourceStartingLsn     = LogicalSequenceNumber.InvalidLsn;
     this.targetStartingLsn     = LogicalSequenceNumber.InvalidLsn;
     this.logRecordsToCopy      = null;
     this.beginCheckpointRecord = null;
     this.bw         = new BinaryWriter(new MemoryStream());
     this.isDisposed = false;
     this.tracer     = tracer;
 }
 public void Open(
     RecoveredOrCopiedCheckpointLsn recoveredOrCopiedCheckpointLsn,
     RoleContextDrainState roleContextDrainState,
     OperationProcessor recordsProcessor,
     IStateReplicator fabricReplicator,
     IBackupManager backupManager,
     CheckpointManager checkpointManager,
     TransactionManager transactionManager,
     ReplicatedLogManager replicatedLogManager,
     IStateManager stateManager,
     TransactionalReplicatorSettings replicatorSettings,
     RecoveryManager recoveryManager,
     ITracer tracer)
 {
     this.recoveredOrCopiedCheckpointLsn = recoveredOrCopiedCheckpointLsn;
     this.recoveryManager       = recoveryManager;
     this.replicatorSettings    = replicatorSettings;
     this.stateManager          = stateManager;
     this.tracer                = tracer;
     this.roleContextDrainState = roleContextDrainState;
     this.recordsProcessor      = recordsProcessor;
     this.fabricReplicator      = fabricReplicator;
     this.backupManager         = backupManager;
     this.transactionManager    = transactionManager;
     this.checkpointManager     = checkpointManager;
     this.replicatedLogManager  = replicatedLogManager;
 }
        public void Open(ReplicatedLogManager logManager, ITracer tracer, ulong checkpointSizeMB, ulong minLogSizeInMB, uint truncationThresholdFactor, uint throttlingThresholdFactor)
        {
            this.tracer     = tracer;
            this.logManager = logManager;

            this.checkpointIntervalBytes    = checkpointSizeMB * MBtoBytesMultiplier;
            this.minLogSizeInBytes          = minLogSizeInMB * MBtoBytesMultiplier;
            this.truncationThresholdInBytes = truncationThresholdFactor * this.minLogSizeInBytes;
            this.throttleAtLogUsageBytes    = this.GetThrottleThresholdInBytes(throttlingThresholdFactor, this.checkpointIntervalBytes, this.minLogSizeInBytes);
            this.minTruncationAmountInBytes = this.GetMinTruncationAmountInBytes(this.checkpointIntervalBytes);
            this.indexIntervalBytes         = this.checkpointIntervalBytes / NumberOfIndexRecordsPerCheckpoint;
            this.txnAbortThresholdInBytes   = this.checkpointIntervalBytes / AbortOldTxFactor;

            Utility.Assert(this.minTruncationAmountInBytes > 0, "Min truncation amount in bytes cannot be less than 1");
            Utility.Assert(this.indexIntervalBytes != 0, "Index interval in bytes cannot be 0");
            Utility.Assert(this.checkpointIntervalBytes != 0, "Checkpoint Interval cannot be 0");
            Utility.Assert(this.minLogSizeInBytes != 0, "Min Log Size in bytes cannot be 0");

            Utility.Assert(this.truncationThresholdInBytes >= this.minLogSizeInBytes, "truncationThresholdInBytes {0} must be larger than minLogSizeInBytes {1}", this.truncationThresholdInBytes, this.minLogSizeInBytes);
            Utility.Assert(this.throttleAtLogUsageBytes > this.truncationThresholdInBytes, "throttleAtLogUsageBytes {0} must be larger than truncationThresholdInBytes {1}", this.throttleAtLogUsageBytes, this.truncationThresholdInBytes);
            Utility.Assert(this.throttleAtLogUsageBytes > this.checkpointIntervalBytes, "throttleAtLogUsageBytes {0} must be larger than checkpointIntervalBytes {1}", this.throttleAtLogUsageBytes, this.checkpointIntervalBytes);

            var trace = string.Format(
                CultureInfo.InvariantCulture,
                "IndexIntervalBytes = {0}, CheckpointIntervalBytes = {1}, MinLogSizeInMB = {2}, TruncateIntervalBytes = {3}, ThrottleAtLogUsageBytes = {4}, MinTruncationThresholdInBytes = {5}",
                this.indexIntervalBytes,
                this.checkpointIntervalBytes,
                this.minLogSizeInBytes,
                this.truncationThresholdInBytes,
                this.throttleAtLogUsageBytes,
                this.minTruncationAmountInBytes);

            LoggingReplicator.ReplicatorTraceOnApi(tracer, trace);
        }
Beispiel #4
0
        public TransactionManager(
            RecoveredOrCopiedCheckpointLsn recoveredOrCopiedCheckpointLsn,
            TransactionMap transactionMap,
            FabricPerformanceCounterSetInstance instance,
            CheckpointManager checkpointManager,
            ReplicatedLogManager replicatedLogManager,
            OperationProcessor unlockCallbackManager,
            ITracer tracer,
            long flushAtBufferedBytes)
        {
            this.recoveredOrCopiedCheckpointLsn = recoveredOrCopiedCheckpointLsn;
            this.transactionMap        = transactionMap;
            this.checkpointManager     = checkpointManager;
            this.replicatedLogManager  = replicatedLogManager;
            this.unlockCallbackManager = unlockCallbackManager;
            this.tracer = tracer;
            this.flushAtBufferedBytes = flushAtBufferedBytes;

            this.beginTransactionOperationRatePerformanceCounterWriter =
                new BeginTransactionOperationRatePerformanceCounterWriter(instance);
            this.addOperationRatePerformanceCounterWriter =
                new AddOperationRatePerformanceCounterWriter(instance);
            this.transactionCommitRatePerformanceCounterWriter =
                new TransactionCommitRatePerformanceCounterWriter(instance);
            this.transactionAbortRatePerformanceCounterWriter =
                new TransactionAbortRatePerformanceCounterWriter(instance);
            this.addAtomicOperationRatePerformanceCounterWriter =
                new AddAtomicOperationRatePerformanceCounterWriter(instance);
        }
Beispiel #5
0
        public TruncateTailManager(
            ReplicatedLogManager replicatedLogManager,
            TransactionMap transactionsMap,
            IStateManager stateManager,
            IBackupManager backupManager,
            LogicalSequenceNumber tailLsn,
            ApplyContext falseProgressApplyContext,
            PhysicalLogReader recoveryLogsReader,
            ITracer tracer)
        {
            Utility.Assert(tracer != null, "{0} TruncateTailManager: Input tracer cannot be null");
            Utility.Assert(backupManager != null, "{0} TruncateTailManager: Input backupManager cannot be null", tracer.Type);

            this.replicatedLogManager = replicatedLogManager;
            this.recoveryLogsReader   = recoveryLogsReader;
            this.transactionsMap      = transactionsMap;
            this.stateManager         = stateManager;
            this.backupManager        = backupManager;
            this.tracer  = tracer;
            this.tailLsn = tailLsn;
            this.falseProgressApplyContext = falseProgressApplyContext;
        }
        public async Task PerformRecoveryAsync(
            RecoveredOrCopiedCheckpointLsn recoveredOrCopiedCheckpointLsn,
            OperationProcessor recordsProcessor,
            CheckpointManager checkpointManager,
            TransactionManager transactionManager,
            LogRecordsDispatcher logRecordsDispatcher,
            ReplicatedLogManager replicatedLogManager)
        {
            var lastPhysicalRecord = PhysicalLogRecord.InvalidPhysicalLogRecord;
            LogicalSequenceNumber lsn;
            var recoveredRecords = new List <LogRecord>();

            var recoveredLastCompletedBeginCheckpointRecord = await this.RecoveryLogsReader.GetLastCompletedBeginCheckpointRecord(this.RecoveredLastEndCheckpointRecord).ConfigureAwait(false);

            var recoveredLastCompletedBeginCheckpointRecordPosition = recoveredLastCompletedBeginCheckpointRecord.RecordPosition;
            var recoveryStartingPosition = recoveredLastCompletedBeginCheckpointRecordPosition - recoveredLastCompletedBeginCheckpointRecord.EarliestPendingTransactionOffset;

            // Set the recovered lsn before performing recovery as it impacts unlock callback logic during recovery
            recoveredOrCopiedCheckpointLsn.Update(recoveredLastCompletedBeginCheckpointRecord.Lsn);

            var trace =
                string.Format(
                    CultureInfo.InvariantCulture,
                    "PerformRecoveryAsync: " + Environment.NewLine
                    + "      Recovered last completed begin checkpoint record." + Environment.NewLine
                    + "      Type: {0} LSN: {1} PSN:{2} Position: {3}" + Environment.NewLine
                    + "      Recovery Starting Position: {4} Recovery Starting Epoch: {5},{6}" + Environment.NewLine
                    + "      LogCompleteCheckpointAfterRecovery: {7}" + "      Recovered ProgressVector: {8}"
                    + Environment.NewLine,
                    recoveredLastCompletedBeginCheckpointRecord.RecordType,
                    recoveredLastCompletedBeginCheckpointRecord.Lsn.LSN,
                    recoveredLastCompletedBeginCheckpointRecord.Psn.PSN,
                    recoveredLastCompletedBeginCheckpointRecord.RecordPosition,
                    recoveryStartingPosition,
                    recoveredLastCompletedBeginCheckpointRecord.Epoch.DataLossNumber,
                    recoveredLastCompletedBeginCheckpointRecord.Epoch.ConfigurationNumber,
                    this.logCompleteCheckpointAfterRecovery,
                    recoveredLastCompletedBeginCheckpointRecord.ProgressVector.ToString(Constants.ProgressVectorMaxStringSizeInKb));

            FabricEvents.Events.Api(tracer.Type, trace);

            LogRecord lastRecoverableRecord = null;

            replicatedLogManager.LogManager.SetSequentialAccessReadSize(
                this.RecoveryLogsReader.ReadStream,
                LogManager.ReadAheadBytesForSequentialStream);

            // Updated to the recovered value in LogRecordsMap during successful recovery
            long lastPeriodicTruncationTimeTicks = 0;

            using (var records = new LogRecords(this.RecoveryLogsReader.ReadStream, recoveryStartingPosition, this.tailRecordAtStart.RecordPosition))
            {
                var hasMoved = await records.MoveNextAsync(CancellationToken.None).ConfigureAwait(false);

                Utility.Assert(hasMoved == true, "hasMoved == true");

                lsn = records.Current.Lsn;
                if (recoveredLastCompletedBeginCheckpointRecord.EarliestPendingTransactionOffset != 0)
                {
                    lsn -= 1;
                }

                var logRecordsMap = new LogRecordsMap(
                    lsn,
                    transactionManager.TransactionsMap,
                    recoveredLastCompletedBeginCheckpointRecord.Epoch,
                    checkpointManager.LastStableLsn,
                    recoveredLastCompletedBeginCheckpointRecord.ProgressVector,
                    recoveredLastCompletedBeginCheckpointRecord,
                    this.tracer,
                    this.RecoveredLastEndCheckpointRecord);

                do
                {
                    var isRecoverableRecord = true;
                    var record = records.Current;
                    record.CompletedFlush(null);

                    Utility.Assert(
                        record.PreviousPhysicalRecord == logRecordsMap.LastPhysicalRecord,
                        "record.PreviousPhysicalRecord == lastPhysicalRecord");

                    logRecordsMap.ProcessLogRecord(record, out isRecoverableRecord);

                    if (isRecoverableRecord == true)
                    {
                        recoveredRecords.Add(record);
                        lastRecoverableRecord = record;

                        if (LoggingReplicator.IsBarrierRecord(record) &&
                            recoveredRecords.Count >= ReplicatorDynamicConstants.ParallelRecoveryBatchSize)
                        {
                            logRecordsDispatcher.DispatchLoggedRecords(new LoggedRecords(recoveredRecords, null));
                            recoveredRecords       = new List <LogRecord>();
                            this.recoveryException = recordsProcessor.ServiceException;

                            if (this.recoveryException == null)
                            {
                                this.recoveryException = recordsProcessor.LogException;
                            }

                            if (this.recoveryException != null)
                            {
                                // If there was an apply or unlock failure, report fault does not help during recovery because the replica is not opened yet.
                                // The only solution here is to throw during OpenAsync

                                FabricEvents.Events.Api(
                                    tracer.Type,
                                    "PerformRecoveryAsync: RecoveryFailed");

                                await lastRecoverableRecord.AwaitProcessing().ConfigureAwait(false);

                                await recordsProcessor.WaitForAllRecordsProcessingAsync().ConfigureAwait(false);

                                throw this.recoveryException;
                            }
                        }
                    }
                    else
                    {
                        Utility.Assert(LoggingReplicator.IsBarrierRecord(record) == false, "IsBarrierRecord(record) == false");
                        record.CompletedApply(null);
                        record.CompletedProcessing(null);
                    }

                    hasMoved = await records.MoveNextAsync(CancellationToken.None).ConfigureAwait(false);
                } while (hasMoved == true);

                this.LastRecoveredAtomicRedoOperationLsn = logRecordsMap.LastRecoveredAtomicRedoOperationLsn;
                checkpointManager.ResetStableLsn(logRecordsMap.LastStableLsn);
                lastPhysicalRecord = logRecordsMap.LastPhysicalRecord;
                lsn = logRecordsMap.LastLogicalSequenceNumber;

                if (recoveredRecords.Count > 0)
                {
                    logRecordsDispatcher.DispatchLoggedRecords(new LoggedRecords(recoveredRecords, null));
                }

                var tailRecord = records.Current;

                FabricEvents.Events.Api(
                    tracer.Type,
                    "PerformRecoveryAsync: Current tail record Type: " + tailRecord.RecordType + " LSN: "
                    + tailRecord.Lsn.LSN + " PSN: " + tailRecord.Psn.PSN + " Position: " + tailRecord.RecordPosition);

                Utility.Assert(
                    lastPhysicalRecord == records.LastPhysicalRecord,
                    "lastPhysicalRecord == records.LastPhysicalRecord");
                Utility.Assert(
                    (tailRecord == lastPhysicalRecord) || (tailRecord.PreviousPhysicalRecord == lastPhysicalRecord),
                    "(tailRecord == lastPhysicalRecord) || "
                    + "(tailRecord.PreviousPhysicalRecord == lastPhysicalRecord), " + "LastPhysicalRecord PSN: {0}",
                    lastPhysicalRecord.Psn.PSN);
                Utility.Assert(
                    logRecordsMap.LastCompletedEndCheckpointRecord != null,
                    "this.lastCompletedEndCheckpointRecord != null");
                Utility.Assert(
                    logRecordsMap.LastInProgressCheckpointRecord == null,
                    "this.lastInProgressCheckpointRecord == null");
                Utility.Assert(logRecordsMap.LastLinkedPhysicalRecord != null, "this.lastLinkedPhysicalRecord != nul");
                Utility.Assert(lsn == tailRecord.Lsn, "lsn == tailRecord.LastLogicalSequenceNumber");

                // Disable read ahead as indexing physical records will read the log backwards
                replicatedLogManager.LogManager.SetSequentialAccessReadSize(this.RecoveryLogsReader.ReadStream, 0);

                await this.RecoveryLogsReader.IndexPhysicalRecords(lastPhysicalRecord).ConfigureAwait(false);

                var callbackManager = new PhysicalLogWriterCallbackManager(this.flushedRecordsCallback);
                callbackManager.FlushedPsn = tailRecord.Psn + 1;
                replicatedLogManager.LogManager.PrepareToLog(tailRecord, callbackManager);

                replicatedLogManager.Reuse(
                    recoveredLastCompletedBeginCheckpointRecord.ProgressVector,
                    logRecordsMap.LastCompletedEndCheckpointRecord,
                    logRecordsMap.LastInProgressCheckpointRecord,
                    logRecordsMap.LastLinkedPhysicalRecord,
                    replicatedLogManager.LastInformationRecord,
                    (IndexingLogRecord)this.RecoveryLogsReader.StartingRecord,
                    logRecordsMap.CurrentLogTailEpoch,
                    lsn);

                this.RecoveredLsn = lsn;

                // GopalK: The order of the following statements is significant
                if (this.logCompleteCheckpointAfterRecovery)
                {
                    replicatedLogManager.CompleteCheckpoint();
                }

                replicatedLogManager.Information(InformationEvent.Recovered);

                await replicatedLogManager.LogManager.FlushAsync("PerformRecovery").ConfigureAwait(false);

                await lastRecoverableRecord.AwaitProcessing().ConfigureAwait(false);

                await recordsProcessor.WaitForAllRecordsProcessingAsync().ConfigureAwait(false);

                lastPeriodicTruncationTimeTicks = logRecordsMap.LastPeriodicTruncationTimeTicks;
            }

            this.recoveryException = recordsProcessor.ServiceException;
            if (this.recoveryException == null)
            {
                this.recoveryException = recordsProcessor.LogException;
            }

            if (this.recoveryException != null)
            {
                FabricEvents.Events.Api(
                    tracer.Type,
                    "PerformRecoveryAsync: RecoveryFailed");

                // If there was an apply or unlock failure, report fault does not help during recovery because the replica is not opened yet.
                // The only solution here is to throw during OpenAsync
                throw this.recoveryException;
            }

            checkpointManager.Recover(
                recoveredLastCompletedBeginCheckpointRecord,
                lastPeriodicTruncationTimeTicks);
        }
        private async Task FlushTask(object initiatingTcs)
        {
            var flushingTasks = new List <TaskCompletionSource <object> >();

            flushingTasks.Add((TaskCompletionSource <object>)initiatingTcs);

            var isFlushTask        = true;
            var flushWatch         = Stopwatch.StartNew();
            var serializationWatch = Stopwatch.StartNew();

            do
            {
                TaskCompletionSource <object> notificationTcs;

                try
                {
                    Utility.Assert(this.loggingException == null, "this.loggingException == null");

                    var   numberOfLatencySensitiveRecords = 0;
                    ulong numberOfBytes = 0;
                    var   startingPsn   = this.flushingRecords[0].Psn;

                    FabricEvents.Events.PhysicalLogWriterFlushStart(
                        this.tracer.Type,
                        this.flushingRecords.Count,
                        startingPsn.PSN);

                    // Enable group commit as we are about to issue IO
                    // Operations above this line must not throw.
                    notificationTcs = Interlocked.Exchange <TaskCompletionSource <object> >(
                        ref this.flushNotificationTcs,
                        FlushPendingTcs);

                    flushWatch.Reset();
                    serializationWatch.Reset();

                    foreach (var record in this.flushingRecords)
                    {
                        serializationWatch.Start();
                        var operationData = this.WriteRecord(record, numberOfBytes);
#if DEBUG
                        ReplicatedLogManager.ValidateOperationData(operationData, "Log Write for " + record.RecordType);
#endif
                        var logicalRecord = record as LogicalLogRecord;
                        if (logicalRecord != null && logicalRecord.IsLatencySensitiveRecord == true)
                        {
                            ++numberOfLatencySensitiveRecords;
                        }

                        serializationWatch.Stop();

                        this.avgSerializationLatencyCounterWriter.IncrementBy(serializationWatch);

                        flushWatch.Start();
                        foreach (var arraySegment in operationData)
                        {
                            numberOfBytes += (ulong)arraySegment.Count;
                            await
                            this.logicalLogStream.AppendAsync(
                                arraySegment.Array,
                                arraySegment.Offset,
                                arraySegment.Count,
                                CancellationToken.None).ConfigureAwait(false);
                        }
                        flushWatch.Stop();
                    }

                    Utility.Assert(notificationTcs == FlushPermittedTcs, "NotificationTcs == FlushPermittedTcs");

                    flushWatch.Start();
                    await this.logicalLogStream.FlushWithMarkerAsync(CancellationToken.None).ConfigureAwait(false);

                    flushWatch.Stop();

                    this.UpdateWriteStats(flushWatch, numberOfBytes);

                    var lastWrittenTailRecord = this.flushingRecords[this.flushingRecords.Count - 1];

                    // Save the new log tail position and no need to interlock as single
                    // instruction 64-bit writes are atomic on 64-bit machines and there
                    // can only be one thread here
                    this.currentLogTailPosition += numberOfBytes;
                    this.currentLogTailRecord    = lastWrittenTailRecord;
                    this.bytesPerFlushCounterWriter.IncrementBy((long)numberOfBytes);

                    if (flushWatch.ElapsedMilliseconds > 3000)
                    {
                        FabricEvents.Events.PhysicalLogWriterFlushEndWarning(
                            this.tracer.Type,
                            numberOfBytes,
                            this.flushingRecords.Count,
                            numberOfLatencySensitiveRecords,
                            flushWatch.ElapsedMilliseconds,
                            serializationWatch.ElapsedMilliseconds,
                            (double)this.writeSpeedBytesPerSecondSum / MovingAverageHistory,
                            (double)this.runningLatencySumMs / MovingAverageHistory,
                            this.logicalLogStream.WritePosition - (long)numberOfBytes);
                    }
                    else
                    {
                        FabricEvents.Events.PhysicalLogWriterFlushEnd(
                            this.tracer.Type,
                            numberOfBytes,
                            this.flushingRecords.Count,
                            numberOfLatencySensitiveRecords,
                            flushWatch.ElapsedMilliseconds,
                            serializationWatch.ElapsedMilliseconds,
                            (double)this.writeSpeedBytesPerSecondSum / MovingAverageHistory,
                            (double)this.runningLatencySumMs / MovingAverageHistory,
                            this.logicalLogStream.WritePosition - (long)numberOfBytes);
                    }
                }
                catch (Exception e)
                {
                    this.loggingException = e;
                }
                finally
                {
                    try
                    {
                        // Disable group commit as there is no pending IO
                        notificationTcs = this.FlushCompleted();

                        // GopalK: The order of the following instructions is very important.
                        // It is important to process flushed records first before
                        // waking up flush waiters
                        var flushedTasks   = flushingTasks;
                        var flushedRecords = new LoggedRecords(this.flushingRecords, this.loggingException);
                        this.ProcessFlushedRecords(flushedRecords);

                        isFlushTask = this.ProcessFlushCompletion(flushedRecords, ref flushedTasks, out flushingTasks);

                        this.WakeupFlushWaiters(flushedTasks);
                        if ((isFlushTask == true) && (this.loggingException != null))
                        {
                            this.FailedFlushTask(flushingTasks);
                            isFlushTask = false;
                        }
                    }
                    catch (Exception e)
                    {
                        int innerHResult     = 0;
                        var exception        = Utility.FlattenException(e, out innerHResult);
                        var exceptionMessage =
                            string.Format(
                                CultureInfo.InvariantCulture,
                                "PhysicalLogWriter::FlushTask() - Hit an exception, Type: {0}, Message: {1}, HResult: 0x{2:X8}, Stack Trace: {3}",
                                exception.GetType().ToString(),
                                exception.Message,
                                exception.HResult != 0 ? exception.HResult : innerHResult,
                                exception.StackTrace);
                        FabricEvents.Events.Warning(this.tracer.Type, exceptionMessage);

                        Utility.Assert(false, exceptionMessage);
                        throw;
                    }
                }
            } while (isFlushTask == true);

            return;
        }
        private async Task DrainReplicationStreamAsync(IOperationStream replicationStream)
        {
            FabricEvents.Events.DrainStart(this.tracer.Type, "Replication stream");

            TaskCompletionSource <object> allOperationsAckedTcs = new TaskCompletionSource <object>();
            var  lastReplicatedRecord = LogicalLogRecord.InvalidLogicalLogRecord;
            long replicatedRecordNumber = 0, acksOutstanding = 1, bytesOutstanding = 0;

            this.roleContextDrainState.OnDrainReplication();

            do
            {
                var drainTask = replicationStream.GetOperationAsync(CancellationToken.None);
                if (drainTask.IsCompleted == false)
                {
                    this.replicatedLogManager.LogManager.FlushAsync("DrainReplicationStream.IsEmpty").IgnoreExceptionVoid();
                    await drainTask.ConfigureAwait(false);
                }

                var operation = drainTask.GetAwaiter().GetResult();
                if (operation != null)
                {
                    var data = operation.Data;
#if DEBUG
                    ReplicatedLogManager.ValidateOperationData(data, "DrainReplicationStream LSN: " + operation.SequenceNumber);
#endif
                    lastReplicatedRecord     = (LogicalLogRecord)LogRecord.FromOperationData(data);
                    lastReplicatedRecord.Lsn = new LogicalSequenceNumber(operation.SequenceNumber);

                    await this.LogLogicalRecordOnSecondaryAsync(lastReplicatedRecord).ConfigureAwait(false);

                    var acksRemaining = Interlocked.Increment(ref acksOutstanding);

                    FabricEvents.Events.DrainReplicationReceive(
                        this.tracer.Type,
                        replicatedRecordNumber,
                        (uint)lastReplicatedRecord.RecordType,
                        lastReplicatedRecord.Lsn.LSN,
                        acksRemaining);

                    ++replicatedRecordNumber;

                    long operationSize = Utility.GetOperationSize(data);

                    var bytesRemaining = Interlocked.Add(ref bytesOutstanding, operationSize);
                    if (((this.replicatorSettings.PublicSettings.MaxSecondaryReplicationQueueSize / 2 <= acksRemaining) ||
                         ((this.replicatorSettings.PublicSettings.MaxSecondaryReplicationQueueMemorySize > 0) &&
                          (this.replicatorSettings.PublicSettings.MaxSecondaryReplicationQueueMemorySize / 2 <= bytesRemaining))) ||
                        ((this.replicatorSettings.PublicSettings.MaxPrimaryReplicationQueueSize / 2 <= acksRemaining) ||
                         ((this.replicatorSettings.PublicSettings.MaxPrimaryReplicationQueueMemorySize > 0) &&
                          (this.replicatorSettings.PublicSettings.MaxPrimaryReplicationQueueMemorySize / 2 <= bytesRemaining))))
                    {
                        FabricEvents.Events.DrainReplicationFlush(
                            this.tracer.Type,
                            replicatedRecordNumber,
                            lastReplicatedRecord.Lsn.LSN,
                            acksRemaining,
                            bytesRemaining);

                        this.replicatedLogManager.LogManager.FlushAsync("DrainReplicationStream.IsFull").IgnoreExceptionVoid();
                    }

                    var capturedOperation = operation;
                    var capturedRecord    = lastReplicatedRecord;
                    lastReplicatedRecord.AwaitFlush().IgnoreException().ContinueWith(
                        async task =>
                    {
                        var acksPending = Interlocked.Decrement(ref acksOutstanding);

                        if (task.Exception != null)
                        {
                            // Signal the drain completion task if needed
                            if (acksPending == 0)
                            {
                                allOperationsAckedTcs.TrySetResult(null);
                            }

                            return;
                        }

                        var bytesPending = Interlocked.Add(ref bytesOutstanding, -operationSize);
                        Utility.Assert(
                            (acksPending >= 0) && (bytesPending >= 0),
                            "(acksPending >= 0) && (bytesPending >= 0)");

                        if (acksPending == 0)
                        {
                            allOperationsAckedTcs.TrySetResult(null);
                        }

                        capturedOperation.Acknowledge();

                        FabricEvents.Events.DrainReplicationNoise(
                            this.tracer.Type,
                            capturedRecord.Lsn.LSN,
                            acksPending,
                            bytesPending);

                        await capturedRecord.AwaitApply().ConfigureAwait(false);
                    }).IgnoreExceptionVoid();
                }
                else
                {
                    await this.replicatedLogManager.FlushInformationRecordAsync(
                        InformationEvent.ReplicationFinished,
                        closeLog : false,
                        flushInitiator : "DrainReplicationstream.IsFinished").ConfigureAwait(false);

                    await this.replicatedLogManager.LastInformationRecord.AwaitProcessing().ConfigureAwait(false);

                    await this.recordsProcessor.WaitForLogicalRecordsProcessingAsync().ConfigureAwait(false);

                    var acksPending = Interlocked.Decrement(ref acksOutstanding);
                    Utility.Assert(acksPending >= 0, "acksPending >= 0");
                    if (acksPending != 0)
                    {
                        await allOperationsAckedTcs.Task.ConfigureAwait(false);
                    }

                    Utility.Assert(acksOutstanding == 0, "acksOutstanding == 0");
                    break;
                }
            } while (true);

#if !DotNetCoreClr
            // These are new events defined in System.Fabric, existing CoreCLR apps would break
            // if these events are refernced as it wont be found. As CoreCLR apps carry System.Fabric
            // along with application
            // This is just a mitigation for now. Actual fix being tracked via bug# 11614507

            FabricEvents.Events.DrainCompleted(
                this.tracer.Type,
                "Replication",
                "Completed",
                replicatedRecordNumber,
                (uint)lastReplicatedRecord.RecordType,
                lastReplicatedRecord.Lsn.LSN,
                lastReplicatedRecord.Psn.PSN,
                lastReplicatedRecord.RecordPosition);
#endif
        }
        private async Task DrainCopyStreamAsync(
            IOperationStream copyStream,
            IOperation operation,
            BeginCheckpointLogRecord copiedCheckpointRecord,
            bool renamedCopyLogSuccessfully)
        {
            FabricEvents.Events.DrainStart(this.tracer.Type, "Copy stream: RenamedCopyLogSuccessfully: " + renamedCopyLogSuccessfully);

            var  lastCopiedRecord = LogicalLogRecord.InvalidLogicalLogRecord;
            long copiedRecordNumber = 0, acksOutstanding = 1;

            TaskCompletionSource <object> allOperationsAckedTcs = new TaskCompletionSource <object>();

            try
            {
                if (operation != null)
                {
                    this.roleContextDrainState.OnDrainCopy();

                    do
                    {
                        var data = operation.Data;
#if DEBUG
                        ReplicatedLogManager.ValidateOperationData(data, "DrainCopyStreamAsync LSN: " + operation.SequenceNumber);
#endif
                        lastCopiedRecord = (LogicalLogRecord)LogRecord.FromOperationData(data);

                        await this.LogLogicalRecordOnSecondaryAsync(lastCopiedRecord).ConfigureAwait(false);

                        // After successfully appending the record into the buffer, increment the outstanding ack count
                        var acksRemaining = Interlocked.Increment(ref acksOutstanding);

                        FabricEvents.Events.DrainCopyReceive(
                            this.tracer.Type,
                            copiedRecordNumber,
                            lastCopiedRecord.RecordType.ToString(),
                            lastCopiedRecord.Lsn.LSN,
                            acksRemaining);

                        ++copiedRecordNumber;
                        if (this.replicatorSettings.PublicSettings.MaxCopyQueueSize / 2 <= acksRemaining)
                        {
                            FabricEvents.Events.DrainCopyFlush(
                                this.tracer.Type,
                                copiedRecordNumber,
                                lastCopiedRecord.Lsn.LSN,
                                acksRemaining);

                            this.replicatedLogManager.LogManager.FlushAsync("DrainCopyStream.IsFull").IgnoreExceptionVoid();
                        }

                        var capturedOperation = operation;
                        var capturedRecord    = lastCopiedRecord;
                        if (copiedCheckpointRecord == null)
                        {
                            copiedCheckpointRecord = this.replicatedLogManager.LastInProgressCheckpointRecord;
                            if (copiedCheckpointRecord != null)
                            {
                                Utility.Assert(
                                    copiedCheckpointRecord.Lsn == this.recoveredOrCopiedCheckpointLsn.Value,
                                    "copiedCheckpointRecordLsn {0} == recoveredOrCopiedCheckpointLsn {1}",
                                    copiedCheckpointRecord.Lsn,
                                    this.recoveredOrCopiedCheckpointLsn.Value);
                            }
                        }

                        // If pumped the last operation in the copy stream (indicated by copiedUptoLsn), rename the copy log if this was a full copy
                        // as we are guranteed that the replica has all the data needed to be promoted to an active secondary and we could not have lost any state
                        if (copiedCheckpointRecord != null &&
                            copiedCheckpointRecord != BeginCheckpointLogRecord.InvalidBeginCheckpointLogRecord &&
                            lastCopiedRecord.Lsn == this.copiedUptoLsn &&
                            renamedCopyLogSuccessfully == false) // Copied UE record could have same LSN, so this condition is needed
                        {
                            await this.checkpointManager.CompleteFirstCheckpointOnIdleAndRenameLog(copiedCheckpointRecord, this.copiedUptoLsn.LSN).ConfigureAwait(false);

                            renamedCopyLogSuccessfully = true;
                        }

                        lastCopiedRecord.AwaitFlush().ContinueWith(
                            async task =>
                        {
                            var acksPending = Interlocked.Decrement(ref acksOutstanding);

                            if (task.Exception != null)
                            {
                                // Signal the drain completion task if needed
                                if (acksPending == 0)
                                {
                                    allOperationsAckedTcs.TrySetResult(null);
                                }

                                return;
                            }

                            capturedOperation.Acknowledge();

                            Utility.Assert(acksPending >= 0, "acksPending {0} >= 0", acksPending);

                            if (acksPending == 0)
                            {
                                allOperationsAckedTcs.TrySetResult(null);
                            }

                            FabricEvents.Events.DrainCopyNoise(
                                this.tracer.Type,
                                capturedRecord.Lsn.LSN,
                                acksPending);

                            await capturedRecord.AwaitApply().ConfigureAwait(false);
                        }).IgnoreExceptionVoid();

                        var drainTask = copyStream.GetOperationAsync(CancellationToken.None);
                        if (drainTask.IsCompleted == false)
                        {
                            // GopalK: Currently, we cannot wait for copy to finish because copy might get
                            // abandoned if the primary fails and the product waits for pending
                            // copy operations to get acknowledged before electing a new primary
                            this.replicatedLogManager.LogManager.FlushAsync("DrainCopyStream.IsEmpty").IgnoreExceptionVoid();
                            await drainTask.ConfigureAwait(false);
                        }

                        operation = drainTask.GetAwaiter().GetResult();
                    } while (operation != null);
                }
            }

            // This finally block ensures that before we continue, we cancel the first full copy checkpoint during full build
            // Without having this, it is possible that the above code throws out of this method and any lifecycle API like close might get stuck because
            // there is a pending checkpoint that is not yet fully processed
            finally
            {
                // If the pump prematurely finishes for any reason, it means the copy log cannot be renamed
                if (copiedCheckpointRecord != null &&
                    copiedCheckpointRecord != BeginCheckpointLogRecord.InvalidBeginCheckpointLogRecord &&
                    renamedCopyLogSuccessfully == false)
                {
                    await this.checkpointManager.CancelFirstCheckpointOnIdleDueToIncompleteCopy(copiedCheckpointRecord, this.copiedUptoLsn.LSN);
                }
            }

            await this.replicatedLogManager.FlushInformationRecordAsync(
                InformationEvent.CopyFinished,
                closeLog : false,
                flushInitiator : "DrainCopyStream.IsFinished").ConfigureAwait(false);

            // Awaiting processing of this record,
            // ensures that all operations in the copystream must have been applied Before we complete the drainComplationTcs.
            await this.replicatedLogManager.LastInformationRecord.AwaitProcessing().ConfigureAwait(false);

            await this.recordsProcessor.WaitForLogicalRecordsProcessingAsync().ConfigureAwait(false);

            var acksOpen = Interlocked.Decrement(ref acksOutstanding);
            Utility.Assert(acksOpen >= 0, "acksOpen {0} >= 0", acksOpen);
            if (acksOpen != 0)
            {
                // wait for all the callbacks above to finish running and acknowleding
                await allOperationsAckedTcs.Task.ConfigureAwait(false);
            }

            Utility.Assert(acksOutstanding == 0, "acksOutstanding == 0");

#if !DotNetCoreClr
            // These are new events defined in System.Fabric, existing CoreCLR apps would break
            // if these events are refernced as it wont be found. As CoreCLR apps carry System.Fabric
            // along with application
            // This is just a mitigation for now. Actual fix being tracked via bug# 11614507

            FabricEvents.Events.DrainCompleted(
                this.tracer.Type,
                "Copy",
                "Completed",
                copiedRecordNumber,
                (uint)lastCopiedRecord.RecordType,
                lastCopiedRecord.Lsn.LSN,
                lastCopiedRecord.Psn.PSN,
                lastCopiedRecord.RecordPosition);
#endif
        }