private async Task DrainReplicationStreamAsync(IOperationStream replicationStream) { FabricEvents.Events.DrainStart(this.tracer.Type, "Replication stream"); TaskCompletionSource <object> allOperationsAckedTcs = new TaskCompletionSource <object>(); var lastReplicatedRecord = LogicalLogRecord.InvalidLogicalLogRecord; long replicatedRecordNumber = 0, acksOutstanding = 1, bytesOutstanding = 0; this.roleContextDrainState.OnDrainReplication(); do { var drainTask = replicationStream.GetOperationAsync(CancellationToken.None); if (drainTask.IsCompleted == false) { this.replicatedLogManager.LogManager.FlushAsync("DrainReplicationStream.IsEmpty").IgnoreExceptionVoid(); await drainTask.ConfigureAwait(false); } var operation = drainTask.GetAwaiter().GetResult(); if (operation != null) { var data = operation.Data; #if DEBUG ReplicatedLogManager.ValidateOperationData(data, "DrainReplicationStream LSN: " + operation.SequenceNumber); #endif lastReplicatedRecord = (LogicalLogRecord)LogRecord.FromOperationData(data); lastReplicatedRecord.Lsn = new LogicalSequenceNumber(operation.SequenceNumber); await this.LogLogicalRecordOnSecondaryAsync(lastReplicatedRecord).ConfigureAwait(false); var acksRemaining = Interlocked.Increment(ref acksOutstanding); FabricEvents.Events.DrainReplicationReceive( this.tracer.Type, replicatedRecordNumber, (uint)lastReplicatedRecord.RecordType, lastReplicatedRecord.Lsn.LSN, acksRemaining); ++replicatedRecordNumber; long operationSize = Utility.GetOperationSize(data); var bytesRemaining = Interlocked.Add(ref bytesOutstanding, operationSize); if (((this.replicatorSettings.PublicSettings.MaxSecondaryReplicationQueueSize / 2 <= acksRemaining) || ((this.replicatorSettings.PublicSettings.MaxSecondaryReplicationQueueMemorySize > 0) && (this.replicatorSettings.PublicSettings.MaxSecondaryReplicationQueueMemorySize / 2 <= bytesRemaining))) || ((this.replicatorSettings.PublicSettings.MaxPrimaryReplicationQueueSize / 2 <= acksRemaining) || ((this.replicatorSettings.PublicSettings.MaxPrimaryReplicationQueueMemorySize > 0) && (this.replicatorSettings.PublicSettings.MaxPrimaryReplicationQueueMemorySize / 2 <= bytesRemaining)))) { FabricEvents.Events.DrainReplicationFlush( this.tracer.Type, replicatedRecordNumber, lastReplicatedRecord.Lsn.LSN, acksRemaining, bytesRemaining); this.replicatedLogManager.LogManager.FlushAsync("DrainReplicationStream.IsFull").IgnoreExceptionVoid(); } var capturedOperation = operation; var capturedRecord = lastReplicatedRecord; lastReplicatedRecord.AwaitFlush().IgnoreException().ContinueWith( async task => { var acksPending = Interlocked.Decrement(ref acksOutstanding); if (task.Exception != null) { // Signal the drain completion task if needed if (acksPending == 0) { allOperationsAckedTcs.TrySetResult(null); } return; } var bytesPending = Interlocked.Add(ref bytesOutstanding, -operationSize); Utility.Assert( (acksPending >= 0) && (bytesPending >= 0), "(acksPending >= 0) && (bytesPending >= 0)"); if (acksPending == 0) { allOperationsAckedTcs.TrySetResult(null); } capturedOperation.Acknowledge(); FabricEvents.Events.DrainReplicationNoise( this.tracer.Type, capturedRecord.Lsn.LSN, acksPending, bytesPending); await capturedRecord.AwaitApply().ConfigureAwait(false); }).IgnoreExceptionVoid(); } else { await this.replicatedLogManager.FlushInformationRecordAsync( InformationEvent.ReplicationFinished, closeLog : false, flushInitiator : "DrainReplicationstream.IsFinished").ConfigureAwait(false); await this.replicatedLogManager.LastInformationRecord.AwaitProcessing().ConfigureAwait(false); await this.recordsProcessor.WaitForLogicalRecordsProcessingAsync().ConfigureAwait(false); var acksPending = Interlocked.Decrement(ref acksOutstanding); Utility.Assert(acksPending >= 0, "acksPending >= 0"); if (acksPending != 0) { await allOperationsAckedTcs.Task.ConfigureAwait(false); } Utility.Assert(acksOutstanding == 0, "acksOutstanding == 0"); break; } } while (true); #if !DotNetCoreClr // These are new events defined in System.Fabric, existing CoreCLR apps would break // if these events are refernced as it wont be found. As CoreCLR apps carry System.Fabric // along with application // This is just a mitigation for now. Actual fix being tracked via bug# 11614507 FabricEvents.Events.DrainCompleted( this.tracer.Type, "Replication", "Completed", replicatedRecordNumber, (uint)lastReplicatedRecord.RecordType, lastReplicatedRecord.Lsn.LSN, lastReplicatedRecord.Psn.PSN, lastReplicatedRecord.RecordPosition); #endif }
private async Task FlushTask(object initiatingTcs) { var flushingTasks = new List <TaskCompletionSource <object> >(); flushingTasks.Add((TaskCompletionSource <object>)initiatingTcs); var isFlushTask = true; var flushWatch = Stopwatch.StartNew(); var serializationWatch = Stopwatch.StartNew(); do { TaskCompletionSource <object> notificationTcs; try { Utility.Assert(this.loggingException == null, "this.loggingException == null"); var numberOfLatencySensitiveRecords = 0; ulong numberOfBytes = 0; var startingPsn = this.flushingRecords[0].Psn; FabricEvents.Events.PhysicalLogWriterFlushStart( this.tracer.Type, this.flushingRecords.Count, startingPsn.PSN); // Enable group commit as we are about to issue IO // Operations above this line must not throw. notificationTcs = Interlocked.Exchange <TaskCompletionSource <object> >( ref this.flushNotificationTcs, FlushPendingTcs); flushWatch.Reset(); serializationWatch.Reset(); foreach (var record in this.flushingRecords) { serializationWatch.Start(); var operationData = this.WriteRecord(record, numberOfBytes); #if DEBUG ReplicatedLogManager.ValidateOperationData(operationData, "Log Write for " + record.RecordType); #endif var logicalRecord = record as LogicalLogRecord; if (logicalRecord != null && logicalRecord.IsLatencySensitiveRecord == true) { ++numberOfLatencySensitiveRecords; } serializationWatch.Stop(); this.avgSerializationLatencyCounterWriter.IncrementBy(serializationWatch); flushWatch.Start(); foreach (var arraySegment in operationData) { numberOfBytes += (ulong)arraySegment.Count; await this.logicalLogStream.AppendAsync( arraySegment.Array, arraySegment.Offset, arraySegment.Count, CancellationToken.None).ConfigureAwait(false); } flushWatch.Stop(); } Utility.Assert(notificationTcs == FlushPermittedTcs, "NotificationTcs == FlushPermittedTcs"); flushWatch.Start(); await this.logicalLogStream.FlushWithMarkerAsync(CancellationToken.None).ConfigureAwait(false); flushWatch.Stop(); this.UpdateWriteStats(flushWatch, numberOfBytes); var lastWrittenTailRecord = this.flushingRecords[this.flushingRecords.Count - 1]; // Save the new log tail position and no need to interlock as single // instruction 64-bit writes are atomic on 64-bit machines and there // can only be one thread here this.currentLogTailPosition += numberOfBytes; this.currentLogTailRecord = lastWrittenTailRecord; this.bytesPerFlushCounterWriter.IncrementBy((long)numberOfBytes); if (flushWatch.ElapsedMilliseconds > 3000) { FabricEvents.Events.PhysicalLogWriterFlushEndWarning( this.tracer.Type, numberOfBytes, this.flushingRecords.Count, numberOfLatencySensitiveRecords, flushWatch.ElapsedMilliseconds, serializationWatch.ElapsedMilliseconds, (double)this.writeSpeedBytesPerSecondSum / MovingAverageHistory, (double)this.runningLatencySumMs / MovingAverageHistory, this.logicalLogStream.WritePosition - (long)numberOfBytes); } else { FabricEvents.Events.PhysicalLogWriterFlushEnd( this.tracer.Type, numberOfBytes, this.flushingRecords.Count, numberOfLatencySensitiveRecords, flushWatch.ElapsedMilliseconds, serializationWatch.ElapsedMilliseconds, (double)this.writeSpeedBytesPerSecondSum / MovingAverageHistory, (double)this.runningLatencySumMs / MovingAverageHistory, this.logicalLogStream.WritePosition - (long)numberOfBytes); } } catch (Exception e) { this.loggingException = e; } finally { try { // Disable group commit as there is no pending IO notificationTcs = this.FlushCompleted(); // GopalK: The order of the following instructions is very important. // It is important to process flushed records first before // waking up flush waiters var flushedTasks = flushingTasks; var flushedRecords = new LoggedRecords(this.flushingRecords, this.loggingException); this.ProcessFlushedRecords(flushedRecords); isFlushTask = this.ProcessFlushCompletion(flushedRecords, ref flushedTasks, out flushingTasks); this.WakeupFlushWaiters(flushedTasks); if ((isFlushTask == true) && (this.loggingException != null)) { this.FailedFlushTask(flushingTasks); isFlushTask = false; } } catch (Exception e) { int innerHResult = 0; var exception = Utility.FlattenException(e, out innerHResult); var exceptionMessage = string.Format( CultureInfo.InvariantCulture, "PhysicalLogWriter::FlushTask() - Hit an exception, Type: {0}, Message: {1}, HResult: 0x{2:X8}, Stack Trace: {3}", exception.GetType().ToString(), exception.Message, exception.HResult != 0 ? exception.HResult : innerHResult, exception.StackTrace); FabricEvents.Events.Warning(this.tracer.Type, exceptionMessage); Utility.Assert(false, exceptionMessage); throw; } } } while (isFlushTask == true); return; }
private async Task DrainCopyStreamAsync( IOperationStream copyStream, IOperation operation, BeginCheckpointLogRecord copiedCheckpointRecord, bool renamedCopyLogSuccessfully) { FabricEvents.Events.DrainStart(this.tracer.Type, "Copy stream: RenamedCopyLogSuccessfully: " + renamedCopyLogSuccessfully); var lastCopiedRecord = LogicalLogRecord.InvalidLogicalLogRecord; long copiedRecordNumber = 0, acksOutstanding = 1; TaskCompletionSource <object> allOperationsAckedTcs = new TaskCompletionSource <object>(); try { if (operation != null) { this.roleContextDrainState.OnDrainCopy(); do { var data = operation.Data; #if DEBUG ReplicatedLogManager.ValidateOperationData(data, "DrainCopyStreamAsync LSN: " + operation.SequenceNumber); #endif lastCopiedRecord = (LogicalLogRecord)LogRecord.FromOperationData(data); await this.LogLogicalRecordOnSecondaryAsync(lastCopiedRecord).ConfigureAwait(false); // After successfully appending the record into the buffer, increment the outstanding ack count var acksRemaining = Interlocked.Increment(ref acksOutstanding); FabricEvents.Events.DrainCopyReceive( this.tracer.Type, copiedRecordNumber, lastCopiedRecord.RecordType.ToString(), lastCopiedRecord.Lsn.LSN, acksRemaining); ++copiedRecordNumber; if (this.replicatorSettings.PublicSettings.MaxCopyQueueSize / 2 <= acksRemaining) { FabricEvents.Events.DrainCopyFlush( this.tracer.Type, copiedRecordNumber, lastCopiedRecord.Lsn.LSN, acksRemaining); this.replicatedLogManager.LogManager.FlushAsync("DrainCopyStream.IsFull").IgnoreExceptionVoid(); } var capturedOperation = operation; var capturedRecord = lastCopiedRecord; if (copiedCheckpointRecord == null) { copiedCheckpointRecord = this.replicatedLogManager.LastInProgressCheckpointRecord; if (copiedCheckpointRecord != null) { Utility.Assert( copiedCheckpointRecord.Lsn == this.recoveredOrCopiedCheckpointLsn.Value, "copiedCheckpointRecordLsn {0} == recoveredOrCopiedCheckpointLsn {1}", copiedCheckpointRecord.Lsn, this.recoveredOrCopiedCheckpointLsn.Value); } } // If pumped the last operation in the copy stream (indicated by copiedUptoLsn), rename the copy log if this was a full copy // as we are guranteed that the replica has all the data needed to be promoted to an active secondary and we could not have lost any state if (copiedCheckpointRecord != null && copiedCheckpointRecord != BeginCheckpointLogRecord.InvalidBeginCheckpointLogRecord && lastCopiedRecord.Lsn == this.copiedUptoLsn && renamedCopyLogSuccessfully == false) // Copied UE record could have same LSN, so this condition is needed { await this.checkpointManager.CompleteFirstCheckpointOnIdleAndRenameLog(copiedCheckpointRecord, this.copiedUptoLsn.LSN).ConfigureAwait(false); renamedCopyLogSuccessfully = true; } lastCopiedRecord.AwaitFlush().ContinueWith( async task => { var acksPending = Interlocked.Decrement(ref acksOutstanding); if (task.Exception != null) { // Signal the drain completion task if needed if (acksPending == 0) { allOperationsAckedTcs.TrySetResult(null); } return; } capturedOperation.Acknowledge(); Utility.Assert(acksPending >= 0, "acksPending {0} >= 0", acksPending); if (acksPending == 0) { allOperationsAckedTcs.TrySetResult(null); } FabricEvents.Events.DrainCopyNoise( this.tracer.Type, capturedRecord.Lsn.LSN, acksPending); await capturedRecord.AwaitApply().ConfigureAwait(false); }).IgnoreExceptionVoid(); var drainTask = copyStream.GetOperationAsync(CancellationToken.None); if (drainTask.IsCompleted == false) { // GopalK: Currently, we cannot wait for copy to finish because copy might get // abandoned if the primary fails and the product waits for pending // copy operations to get acknowledged before electing a new primary this.replicatedLogManager.LogManager.FlushAsync("DrainCopyStream.IsEmpty").IgnoreExceptionVoid(); await drainTask.ConfigureAwait(false); } operation = drainTask.GetAwaiter().GetResult(); } while (operation != null); } } // This finally block ensures that before we continue, we cancel the first full copy checkpoint during full build // Without having this, it is possible that the above code throws out of this method and any lifecycle API like close might get stuck because // there is a pending checkpoint that is not yet fully processed finally { // If the pump prematurely finishes for any reason, it means the copy log cannot be renamed if (copiedCheckpointRecord != null && copiedCheckpointRecord != BeginCheckpointLogRecord.InvalidBeginCheckpointLogRecord && renamedCopyLogSuccessfully == false) { await this.checkpointManager.CancelFirstCheckpointOnIdleDueToIncompleteCopy(copiedCheckpointRecord, this.copiedUptoLsn.LSN); } } await this.replicatedLogManager.FlushInformationRecordAsync( InformationEvent.CopyFinished, closeLog : false, flushInitiator : "DrainCopyStream.IsFinished").ConfigureAwait(false); // Awaiting processing of this record, // ensures that all operations in the copystream must have been applied Before we complete the drainComplationTcs. await this.replicatedLogManager.LastInformationRecord.AwaitProcessing().ConfigureAwait(false); await this.recordsProcessor.WaitForLogicalRecordsProcessingAsync().ConfigureAwait(false); var acksOpen = Interlocked.Decrement(ref acksOutstanding); Utility.Assert(acksOpen >= 0, "acksOpen {0} >= 0", acksOpen); if (acksOpen != 0) { // wait for all the callbacks above to finish running and acknowleding await allOperationsAckedTcs.Task.ConfigureAwait(false); } Utility.Assert(acksOutstanding == 0, "acksOutstanding == 0"); #if !DotNetCoreClr // These are new events defined in System.Fabric, existing CoreCLR apps would break // if these events are refernced as it wont be found. As CoreCLR apps carry System.Fabric // along with application // This is just a mitigation for now. Actual fix being tracked via bug# 11614507 FabricEvents.Events.DrainCompleted( this.tracer.Type, "Copy", "Completed", copiedRecordNumber, (uint)lastCopiedRecord.RecordType, lastCopiedRecord.Lsn.LSN, lastCopiedRecord.Psn.PSN, lastCopiedRecord.RecordPosition); #endif }