internal void ProcessFlushedRecords(LoggedRecords flushedRecords) { lock (this.callbackLock) { if (this.isCallingback == false) { Utility.Assert(this.loggedRecords == null, "this.loggedRecords == null"); this.isCallingback = true; } else { if (this.loggedRecords == null) { this.loggedRecords = new List <LoggedRecords>(); } this.loggedRecords.Add(flushedRecords); return; } } var records = new List <LoggedRecords>(); records.Add(flushedRecords); if (this.flushedPsn == PhysicalSequenceNumber.InvalidPsn) { this.flushedPsn = flushedRecords.Records[0].Psn; } Task.Factory.StartNew(this.InvokeFlushedRecordsCallback, records).IgnoreExceptionVoid(); return; }
private void EmptyFlushedRecordsCallback(LoggedRecords loggedRecords) { var records = loggedRecords.Records; var exception = loggedRecords.Exception; foreach (var record in records) { if (exception == null) { FabricEvents.Events.LogManagerNoOpRecordsCallback( this.Tracer.Type, record.RecordType.ToString(), record.Lsn.LSN, record.Psn.PSN, record.RecordPosition); } else { FabricEvents.Events.LogManagerNoOpRecordsCallbackFailed( this.Tracer.Type, record.RecordType.ToString(), record.Lsn.LSN, record.Psn.PSN, record.RecordPosition); } record.CompletedFlush(exception); } }
/// <summary> /// Inserts a record to the buffer where it waits to be flushed in batches. /// A successful run will set following properties in the record. /// record.Psn and record.PreviousPhysicalLogRecord /// </summary> /// <param name="record"></param> /// <returns>The pending size of the buffered records</returns> internal long InsertBufferedRecord(LogRecord record) { Utility.Assert(LogRecord.IsInvalidRecord(record) == false, "LogRecord.IsInvalidRecord(record) == false"); var physicalRecord = record as PhysicalLogRecord; lock (this.flushLock) { if (this.closedException == null) { // No record can be inserted after 'RemovingState' Information log record if (this.lastPhysicalRecord != null && this.lastPhysicalRecord.RecordType == LogRecordType.Information) { var lastInformationRecord = this.lastPhysicalRecord as InformationLogRecord; Utility.Assert( lastInformationRecord.InformationEvent != InformationEvent.RemovingState, "No record should be inserted after 'RemovingState'. Current Record = {0}", record); } // Update record and tail ++this.currentLogTailPsn; record.Psn = this.currentLogTailPsn; this.incomingBytesRateCounterWriter.IncrementBy(record.ApproximateSizeOnDisk); record.PreviousPhysicalRecord = this.lastPhysicalRecord; if (physicalRecord != null) { if (this.lastPhysicalRecord != null) { this.lastPhysicalRecord.NextPhysicalRecord = physicalRecord; } this.lastPhysicalRecord = physicalRecord; } // Buffer record if (this.bufferedRecords == null) { this.bufferedRecords = new List <LogRecord>(); } this.bufferedRecords.Add(record); var newBufferedRecordsBytes = Interlocked.Add( ref this.bufferedRecordsBytes, record.ApproximateSizeOnDisk); return(newBufferedRecordsBytes); } } var failedRecord = new LoggedRecords(record, this.closedException); this.ProcessFlushedRecords(failedRecord); return(Interlocked.Read(ref this.bufferedRecordsBytes)); }
private void ProcessFlushedRecords(LoggedRecords flushedRecords) { if (this.callbackManager != null) { this.callbackManager.ProcessFlushedRecords(flushedRecords); } return; }
private bool ProcessFlushCompletion( LoggedRecords flushedRecords, ref List <TaskCompletionSource <object> > flushedTasks, out List <TaskCompletionSource <object> > flushingTasks) { Utility.Assert(flushedTasks != null, "flushedTasks != null"); bool isFlushTask; lock (this.flushLock) { foreach (var record in flushedRecords.Records) { // Subtract the approximate size on disk for the records that got flushed var result = Interlocked.Add(ref this.pendingFlushRecordsBytes, -record.ApproximateSizeOnDisk); #if DEBUG Utility.Assert( result >= 0, "Subtraction of ApproximateSizeOnDisk {0} for record Lsn: {1} yielded negative value for pending flush records {2}", record.ApproximateSizeOnDisk, record.Lsn.LSN, result); #endif } if (this.exitTcs != null) { this.exitTcs.SetResult(null); Thread.Sleep(Timeout.InfiniteTimeSpan); } this.flushingRecords = this.pendingFlushRecords; if (this.flushingRecords == null) { if (this.pendingFlushTasks != null) { flushedTasks.AddRange(this.pendingFlushTasks); this.pendingFlushTasks = null; } flushingTasks = null; isFlushTask = false; } else { flushingTasks = this.pendingFlushTasks; this.pendingFlushRecords = null; this.pendingFlushTasks = null; isFlushTask = true; } } return(isFlushTask); }
private void FailedFlushTask(List <TaskCompletionSource <object> > flushingTasks) { Utility.Assert(this.loggingException != null, "this.loggingException != null"); var isFlushTask = true; do { // GopalK: The order of the following instructions is very important. // It is important to process flushed records first before // waking up flush waiters var flushedTasks = flushingTasks; var flushedRecords = new LoggedRecords(this.flushingRecords, this.loggingException); this.ProcessFlushedRecords(flushedRecords); isFlushTask = this.ProcessFlushCompletion(flushedRecords, ref flushedTasks, out flushingTasks); this.WakeupFlushWaiters(flushedTasks); } while (isFlushTask == true); return; }
public void DispatchLoggedRecords(LoggedRecords loggedRecords) { lock (this.dispatchLock) { if (this.processingRecords != null) { if (this.flushedRecords == null) { this.flushedRecords = new List <LoggedRecords>(); } this.flushedRecords.Add(loggedRecords); return; } Utility.Assert(this.flushedRecords == null, "this.flushedRecords == null"); Utility.Assert(this.processingIndex == 0, "this.processingIndex == 0"); this.processingRecords = new List <LoggedRecords>(); this.processingRecords.Add(loggedRecords); } this.StartProcessingLoggedRecords().IgnoreExceptionVoid(); return; }
private async Task FlushTask(object initiatingTcs) { var flushingTasks = new List <TaskCompletionSource <object> >(); flushingTasks.Add((TaskCompletionSource <object>)initiatingTcs); var isFlushTask = true; var flushWatch = Stopwatch.StartNew(); var serializationWatch = Stopwatch.StartNew(); do { TaskCompletionSource <object> notificationTcs; try { Utility.Assert(this.loggingException == null, "this.loggingException == null"); var numberOfLatencySensitiveRecords = 0; ulong numberOfBytes = 0; var startingPsn = this.flushingRecords[0].Psn; FabricEvents.Events.PhysicalLogWriterFlushStart( this.tracer.Type, this.flushingRecords.Count, startingPsn.PSN); // Enable group commit as we are about to issue IO // Operations above this line must not throw. notificationTcs = Interlocked.Exchange <TaskCompletionSource <object> >( ref this.flushNotificationTcs, FlushPendingTcs); flushWatch.Reset(); serializationWatch.Reset(); foreach (var record in this.flushingRecords) { serializationWatch.Start(); var operationData = this.WriteRecord(record, numberOfBytes); #if DEBUG ReplicatedLogManager.ValidateOperationData(operationData, "Log Write for " + record.RecordType); #endif var logicalRecord = record as LogicalLogRecord; if (logicalRecord != null && logicalRecord.IsLatencySensitiveRecord == true) { ++numberOfLatencySensitiveRecords; } serializationWatch.Stop(); this.avgSerializationLatencyCounterWriter.IncrementBy(serializationWatch); flushWatch.Start(); foreach (var arraySegment in operationData) { numberOfBytes += (ulong)arraySegment.Count; await this.logicalLogStream.AppendAsync( arraySegment.Array, arraySegment.Offset, arraySegment.Count, CancellationToken.None).ConfigureAwait(false); } flushWatch.Stop(); } Utility.Assert(notificationTcs == FlushPermittedTcs, "NotificationTcs == FlushPermittedTcs"); flushWatch.Start(); await this.logicalLogStream.FlushWithMarkerAsync(CancellationToken.None).ConfigureAwait(false); flushWatch.Stop(); this.UpdateWriteStats(flushWatch, numberOfBytes); var lastWrittenTailRecord = this.flushingRecords[this.flushingRecords.Count - 1]; // Save the new log tail position and no need to interlock as single // instruction 64-bit writes are atomic on 64-bit machines and there // can only be one thread here this.currentLogTailPosition += numberOfBytes; this.currentLogTailRecord = lastWrittenTailRecord; this.bytesPerFlushCounterWriter.IncrementBy((long)numberOfBytes); if (flushWatch.ElapsedMilliseconds > 3000) { FabricEvents.Events.PhysicalLogWriterFlushEndWarning( this.tracer.Type, numberOfBytes, this.flushingRecords.Count, numberOfLatencySensitiveRecords, flushWatch.ElapsedMilliseconds, serializationWatch.ElapsedMilliseconds, (double)this.writeSpeedBytesPerSecondSum / MovingAverageHistory, (double)this.runningLatencySumMs / MovingAverageHistory, this.logicalLogStream.WritePosition - (long)numberOfBytes); } else { FabricEvents.Events.PhysicalLogWriterFlushEnd( this.tracer.Type, numberOfBytes, this.flushingRecords.Count, numberOfLatencySensitiveRecords, flushWatch.ElapsedMilliseconds, serializationWatch.ElapsedMilliseconds, (double)this.writeSpeedBytesPerSecondSum / MovingAverageHistory, (double)this.runningLatencySumMs / MovingAverageHistory, this.logicalLogStream.WritePosition - (long)numberOfBytes); } } catch (Exception e) { this.loggingException = e; } finally { try { // Disable group commit as there is no pending IO notificationTcs = this.FlushCompleted(); // GopalK: The order of the following instructions is very important. // It is important to process flushed records first before // waking up flush waiters var flushedTasks = flushingTasks; var flushedRecords = new LoggedRecords(this.flushingRecords, this.loggingException); this.ProcessFlushedRecords(flushedRecords); isFlushTask = this.ProcessFlushCompletion(flushedRecords, ref flushedTasks, out flushingTasks); this.WakeupFlushWaiters(flushedTasks); if ((isFlushTask == true) && (this.loggingException != null)) { this.FailedFlushTask(flushingTasks); isFlushTask = false; } } catch (Exception e) { int innerHResult = 0; var exception = Utility.FlattenException(e, out innerHResult); var exceptionMessage = string.Format( CultureInfo.InvariantCulture, "PhysicalLogWriter::FlushTask() - Hit an exception, Type: {0}, Message: {1}, HResult: 0x{2:X8}, Stack Trace: {3}", exception.GetType().ToString(), exception.Message, exception.HResult != 0 ? exception.HResult : innerHResult, exception.StackTrace); FabricEvents.Events.Warning(this.tracer.Type, exceptionMessage); Utility.Assert(false, exceptionMessage); throw; } } } while (isFlushTask == true); return; }