private async Task DrainReplicationStreamAsync(IOperationStream replicationStream) { FabricEvents.Events.DrainStart(this.tracer.Type, "Replication stream"); TaskCompletionSource <object> allOperationsAckedTcs = new TaskCompletionSource <object>(); var lastReplicatedRecord = LogicalLogRecord.InvalidLogicalLogRecord; long replicatedRecordNumber = 0, acksOutstanding = 1, bytesOutstanding = 0; this.roleContextDrainState.OnDrainReplication(); do { var drainTask = replicationStream.GetOperationAsync(CancellationToken.None); if (drainTask.IsCompleted == false) { this.replicatedLogManager.LogManager.FlushAsync("DrainReplicationStream.IsEmpty").IgnoreExceptionVoid(); await drainTask.ConfigureAwait(false); } var operation = drainTask.GetAwaiter().GetResult(); if (operation != null) { var data = operation.Data; #if DEBUG ReplicatedLogManager.ValidateOperationData(data, "DrainReplicationStream LSN: " + operation.SequenceNumber); #endif lastReplicatedRecord = (LogicalLogRecord)LogRecord.FromOperationData(data); lastReplicatedRecord.Lsn = new LogicalSequenceNumber(operation.SequenceNumber); await this.LogLogicalRecordOnSecondaryAsync(lastReplicatedRecord).ConfigureAwait(false); var acksRemaining = Interlocked.Increment(ref acksOutstanding); FabricEvents.Events.DrainReplicationReceive( this.tracer.Type, replicatedRecordNumber, (uint)lastReplicatedRecord.RecordType, lastReplicatedRecord.Lsn.LSN, acksRemaining); ++replicatedRecordNumber; long operationSize = Utility.GetOperationSize(data); var bytesRemaining = Interlocked.Add(ref bytesOutstanding, operationSize); if (((this.replicatorSettings.PublicSettings.MaxSecondaryReplicationQueueSize / 2 <= acksRemaining) || ((this.replicatorSettings.PublicSettings.MaxSecondaryReplicationQueueMemorySize > 0) && (this.replicatorSettings.PublicSettings.MaxSecondaryReplicationQueueMemorySize / 2 <= bytesRemaining))) || ((this.replicatorSettings.PublicSettings.MaxPrimaryReplicationQueueSize / 2 <= acksRemaining) || ((this.replicatorSettings.PublicSettings.MaxPrimaryReplicationQueueMemorySize > 0) && (this.replicatorSettings.PublicSettings.MaxPrimaryReplicationQueueMemorySize / 2 <= bytesRemaining)))) { FabricEvents.Events.DrainReplicationFlush( this.tracer.Type, replicatedRecordNumber, lastReplicatedRecord.Lsn.LSN, acksRemaining, bytesRemaining); this.replicatedLogManager.LogManager.FlushAsync("DrainReplicationStream.IsFull").IgnoreExceptionVoid(); } var capturedOperation = operation; var capturedRecord = lastReplicatedRecord; lastReplicatedRecord.AwaitFlush().IgnoreException().ContinueWith( async task => { var acksPending = Interlocked.Decrement(ref acksOutstanding); if (task.Exception != null) { // Signal the drain completion task if needed if (acksPending == 0) { allOperationsAckedTcs.TrySetResult(null); } return; } var bytesPending = Interlocked.Add(ref bytesOutstanding, -operationSize); Utility.Assert( (acksPending >= 0) && (bytesPending >= 0), "(acksPending >= 0) && (bytesPending >= 0)"); if (acksPending == 0) { allOperationsAckedTcs.TrySetResult(null); } capturedOperation.Acknowledge(); FabricEvents.Events.DrainReplicationNoise( this.tracer.Type, capturedRecord.Lsn.LSN, acksPending, bytesPending); await capturedRecord.AwaitApply().ConfigureAwait(false); }).IgnoreExceptionVoid(); } else { await this.replicatedLogManager.FlushInformationRecordAsync( InformationEvent.ReplicationFinished, closeLog : false, flushInitiator : "DrainReplicationstream.IsFinished").ConfigureAwait(false); await this.replicatedLogManager.LastInformationRecord.AwaitProcessing().ConfigureAwait(false); await this.recordsProcessor.WaitForLogicalRecordsProcessingAsync().ConfigureAwait(false); var acksPending = Interlocked.Decrement(ref acksOutstanding); Utility.Assert(acksPending >= 0, "acksPending >= 0"); if (acksPending != 0) { await allOperationsAckedTcs.Task.ConfigureAwait(false); } Utility.Assert(acksOutstanding == 0, "acksOutstanding == 0"); break; } } while (true); #if !DotNetCoreClr // These are new events defined in System.Fabric, existing CoreCLR apps would break // if these events are refernced as it wont be found. As CoreCLR apps carry System.Fabric // along with application // This is just a mitigation for now. Actual fix being tracked via bug# 11614507 FabricEvents.Events.DrainCompleted( this.tracer.Type, "Replication", "Completed", replicatedRecordNumber, (uint)lastReplicatedRecord.RecordType, lastReplicatedRecord.Lsn.LSN, lastReplicatedRecord.Psn.PSN, lastReplicatedRecord.RecordPosition); #endif }