Ejemplo n.º 1
0
        internal void ProcessFlushedRecords(LoggedRecords flushedRecords)
        {
            lock (this.callbackLock)
            {
                if (this.isCallingback == false)
                {
                    Utility.Assert(this.loggedRecords == null, "this.loggedRecords == null");
                    this.isCallingback = true;
                }
                else
                {
                    if (this.loggedRecords == null)
                    {
                        this.loggedRecords = new List <LoggedRecords>();
                    }

                    this.loggedRecords.Add(flushedRecords);
                    return;
                }
            }

            var records = new List <LoggedRecords>();

            records.Add(flushedRecords);
            if (this.flushedPsn == PhysicalSequenceNumber.InvalidPsn)
            {
                this.flushedPsn = flushedRecords.Records[0].Psn;
            }

            Task.Factory.StartNew(this.InvokeFlushedRecordsCallback, records).IgnoreExceptionVoid();
            return;
        }
Ejemplo n.º 2
0
        private void EmptyFlushedRecordsCallback(LoggedRecords loggedRecords)
        {
            var records   = loggedRecords.Records;
            var exception = loggedRecords.Exception;

            foreach (var record in records)
            {
                if (exception == null)
                {
                    FabricEvents.Events.LogManagerNoOpRecordsCallback(
                        this.Tracer.Type,
                        record.RecordType.ToString(),
                        record.Lsn.LSN,
                        record.Psn.PSN,
                        record.RecordPosition);
                }
                else
                {
                    FabricEvents.Events.LogManagerNoOpRecordsCallbackFailed(
                        this.Tracer.Type,
                        record.RecordType.ToString(),
                        record.Lsn.LSN,
                        record.Psn.PSN,
                        record.RecordPosition);
                }

                record.CompletedFlush(exception);
            }
        }
Ejemplo n.º 3
0
        /// <summary>
        /// Inserts a record to the buffer where it waits to be flushed in batches.
        /// A successful run will set following properties in the record.
        ///     record.Psn and record.PreviousPhysicalLogRecord
        /// </summary>
        /// <param name="record"></param>
        /// <returns>The pending size of the buffered records</returns>
        internal long InsertBufferedRecord(LogRecord record)
        {
            Utility.Assert(LogRecord.IsInvalidRecord(record) == false, "LogRecord.IsInvalidRecord(record) == false");

            var physicalRecord = record as PhysicalLogRecord;

            lock (this.flushLock)
            {
                if (this.closedException == null)
                {
                    // No record can be inserted after 'RemovingState' Information log record
                    if (this.lastPhysicalRecord != null &&
                        this.lastPhysicalRecord.RecordType == LogRecordType.Information)
                    {
                        var lastInformationRecord = this.lastPhysicalRecord as InformationLogRecord;
                        Utility.Assert(
                            lastInformationRecord.InformationEvent != InformationEvent.RemovingState,
                            "No record should be inserted after 'RemovingState'. Current Record = {0}",
                            record);
                    }

                    // Update record and tail
                    ++this.currentLogTailPsn;
                    record.Psn = this.currentLogTailPsn;
                    this.incomingBytesRateCounterWriter.IncrementBy(record.ApproximateSizeOnDisk);

                    record.PreviousPhysicalRecord = this.lastPhysicalRecord;
                    if (physicalRecord != null)
                    {
                        if (this.lastPhysicalRecord != null)
                        {
                            this.lastPhysicalRecord.NextPhysicalRecord = physicalRecord;
                        }

                        this.lastPhysicalRecord = physicalRecord;
                    }

                    // Buffer record
                    if (this.bufferedRecords == null)
                    {
                        this.bufferedRecords = new List <LogRecord>();
                    }

                    this.bufferedRecords.Add(record);
                    var newBufferedRecordsBytes = Interlocked.Add(
                        ref this.bufferedRecordsBytes,
                        record.ApproximateSizeOnDisk);

                    return(newBufferedRecordsBytes);
                }
            }

            var failedRecord = new LoggedRecords(record, this.closedException);

            this.ProcessFlushedRecords(failedRecord);

            return(Interlocked.Read(ref this.bufferedRecordsBytes));
        }
Ejemplo n.º 4
0
        private void ProcessFlushedRecords(LoggedRecords flushedRecords)
        {
            if (this.callbackManager != null)
            {
                this.callbackManager.ProcessFlushedRecords(flushedRecords);
            }

            return;
        }
Ejemplo n.º 5
0
        private bool ProcessFlushCompletion(
            LoggedRecords flushedRecords,
            ref List <TaskCompletionSource <object> > flushedTasks,
            out List <TaskCompletionSource <object> > flushingTasks)
        {
            Utility.Assert(flushedTasks != null, "flushedTasks != null");

            bool isFlushTask;

            lock (this.flushLock)
            {
                foreach (var record in flushedRecords.Records)
                {
                    // Subtract the approximate size on disk for the records that got flushed
                    var result = Interlocked.Add(ref this.pendingFlushRecordsBytes, -record.ApproximateSizeOnDisk);
#if DEBUG
                    Utility.Assert(
                        result >= 0,
                        "Subtraction of ApproximateSizeOnDisk {0} for record Lsn: {1} yielded negative value for pending flush records {2}",
                        record.ApproximateSizeOnDisk, record.Lsn.LSN, result);
#endif
                }

                if (this.exitTcs != null)
                {
                    this.exitTcs.SetResult(null);

                    Thread.Sleep(Timeout.InfiniteTimeSpan);
                }

                this.flushingRecords = this.pendingFlushRecords;
                if (this.flushingRecords == null)
                {
                    if (this.pendingFlushTasks != null)
                    {
                        flushedTasks.AddRange(this.pendingFlushTasks);
                        this.pendingFlushTasks = null;
                    }

                    flushingTasks = null;
                    isFlushTask   = false;
                }
                else
                {
                    flushingTasks            = this.pendingFlushTasks;
                    this.pendingFlushRecords = null;
                    this.pendingFlushTasks   = null;

                    isFlushTask = true;
                }
            }

            return(isFlushTask);
        }
Ejemplo n.º 6
0
        private void FailedFlushTask(List <TaskCompletionSource <object> > flushingTasks)
        {
            Utility.Assert(this.loggingException != null, "this.loggingException != null");

            var isFlushTask = true;

            do
            {
                // GopalK: The order of the following instructions is very important.
                // It is important to process flushed records first before
                // waking up flush waiters
                var flushedTasks   = flushingTasks;
                var flushedRecords = new LoggedRecords(this.flushingRecords, this.loggingException);
                this.ProcessFlushedRecords(flushedRecords);

                isFlushTask = this.ProcessFlushCompletion(flushedRecords, ref flushedTasks, out flushingTasks);

                this.WakeupFlushWaiters(flushedTasks);
            } while (isFlushTask == true);

            return;
        }
Ejemplo n.º 7
0
        public void DispatchLoggedRecords(LoggedRecords loggedRecords)
        {
            lock (this.dispatchLock)
            {
                if (this.processingRecords != null)
                {
                    if (this.flushedRecords == null)
                    {
                        this.flushedRecords = new List <LoggedRecords>();
                    }

                    this.flushedRecords.Add(loggedRecords);
                    return;
                }

                Utility.Assert(this.flushedRecords == null, "this.flushedRecords == null");
                Utility.Assert(this.processingIndex == 0, "this.processingIndex == 0");
                this.processingRecords = new List <LoggedRecords>();
                this.processingRecords.Add(loggedRecords);
            }

            this.StartProcessingLoggedRecords().IgnoreExceptionVoid();
            return;
        }
Ejemplo n.º 8
0
        private async Task FlushTask(object initiatingTcs)
        {
            var flushingTasks = new List <TaskCompletionSource <object> >();

            flushingTasks.Add((TaskCompletionSource <object>)initiatingTcs);

            var isFlushTask        = true;
            var flushWatch         = Stopwatch.StartNew();
            var serializationWatch = Stopwatch.StartNew();

            do
            {
                TaskCompletionSource <object> notificationTcs;

                try
                {
                    Utility.Assert(this.loggingException == null, "this.loggingException == null");

                    var   numberOfLatencySensitiveRecords = 0;
                    ulong numberOfBytes = 0;
                    var   startingPsn   = this.flushingRecords[0].Psn;

                    FabricEvents.Events.PhysicalLogWriterFlushStart(
                        this.tracer.Type,
                        this.flushingRecords.Count,
                        startingPsn.PSN);

                    // Enable group commit as we are about to issue IO
                    // Operations above this line must not throw.
                    notificationTcs = Interlocked.Exchange <TaskCompletionSource <object> >(
                        ref this.flushNotificationTcs,
                        FlushPendingTcs);

                    flushWatch.Reset();
                    serializationWatch.Reset();

                    foreach (var record in this.flushingRecords)
                    {
                        serializationWatch.Start();
                        var operationData = this.WriteRecord(record, numberOfBytes);
#if DEBUG
                        ReplicatedLogManager.ValidateOperationData(operationData, "Log Write for " + record.RecordType);
#endif
                        var logicalRecord = record as LogicalLogRecord;
                        if (logicalRecord != null && logicalRecord.IsLatencySensitiveRecord == true)
                        {
                            ++numberOfLatencySensitiveRecords;
                        }

                        serializationWatch.Stop();

                        this.avgSerializationLatencyCounterWriter.IncrementBy(serializationWatch);

                        flushWatch.Start();
                        foreach (var arraySegment in operationData)
                        {
                            numberOfBytes += (ulong)arraySegment.Count;
                            await
                            this.logicalLogStream.AppendAsync(
                                arraySegment.Array,
                                arraySegment.Offset,
                                arraySegment.Count,
                                CancellationToken.None).ConfigureAwait(false);
                        }
                        flushWatch.Stop();
                    }

                    Utility.Assert(notificationTcs == FlushPermittedTcs, "NotificationTcs == FlushPermittedTcs");

                    flushWatch.Start();
                    await this.logicalLogStream.FlushWithMarkerAsync(CancellationToken.None).ConfigureAwait(false);

                    flushWatch.Stop();

                    this.UpdateWriteStats(flushWatch, numberOfBytes);

                    var lastWrittenTailRecord = this.flushingRecords[this.flushingRecords.Count - 1];

                    // Save the new log tail position and no need to interlock as single
                    // instruction 64-bit writes are atomic on 64-bit machines and there
                    // can only be one thread here
                    this.currentLogTailPosition += numberOfBytes;
                    this.currentLogTailRecord    = lastWrittenTailRecord;
                    this.bytesPerFlushCounterWriter.IncrementBy((long)numberOfBytes);

                    if (flushWatch.ElapsedMilliseconds > 3000)
                    {
                        FabricEvents.Events.PhysicalLogWriterFlushEndWarning(
                            this.tracer.Type,
                            numberOfBytes,
                            this.flushingRecords.Count,
                            numberOfLatencySensitiveRecords,
                            flushWatch.ElapsedMilliseconds,
                            serializationWatch.ElapsedMilliseconds,
                            (double)this.writeSpeedBytesPerSecondSum / MovingAverageHistory,
                            (double)this.runningLatencySumMs / MovingAverageHistory,
                            this.logicalLogStream.WritePosition - (long)numberOfBytes);
                    }
                    else
                    {
                        FabricEvents.Events.PhysicalLogWriterFlushEnd(
                            this.tracer.Type,
                            numberOfBytes,
                            this.flushingRecords.Count,
                            numberOfLatencySensitiveRecords,
                            flushWatch.ElapsedMilliseconds,
                            serializationWatch.ElapsedMilliseconds,
                            (double)this.writeSpeedBytesPerSecondSum / MovingAverageHistory,
                            (double)this.runningLatencySumMs / MovingAverageHistory,
                            this.logicalLogStream.WritePosition - (long)numberOfBytes);
                    }
                }
                catch (Exception e)
                {
                    this.loggingException = e;
                }
                finally
                {
                    try
                    {
                        // Disable group commit as there is no pending IO
                        notificationTcs = this.FlushCompleted();

                        // GopalK: The order of the following instructions is very important.
                        // It is important to process flushed records first before
                        // waking up flush waiters
                        var flushedTasks   = flushingTasks;
                        var flushedRecords = new LoggedRecords(this.flushingRecords, this.loggingException);
                        this.ProcessFlushedRecords(flushedRecords);

                        isFlushTask = this.ProcessFlushCompletion(flushedRecords, ref flushedTasks, out flushingTasks);

                        this.WakeupFlushWaiters(flushedTasks);
                        if ((isFlushTask == true) && (this.loggingException != null))
                        {
                            this.FailedFlushTask(flushingTasks);
                            isFlushTask = false;
                        }
                    }
                    catch (Exception e)
                    {
                        int innerHResult     = 0;
                        var exception        = Utility.FlattenException(e, out innerHResult);
                        var exceptionMessage =
                            string.Format(
                                CultureInfo.InvariantCulture,
                                "PhysicalLogWriter::FlushTask() - Hit an exception, Type: {0}, Message: {1}, HResult: 0x{2:X8}, Stack Trace: {3}",
                                exception.GetType().ToString(),
                                exception.Message,
                                exception.HResult != 0 ? exception.HResult : innerHResult,
                                exception.StackTrace);
                        FabricEvents.Events.Warning(this.tracer.Type, exceptionMessage);

                        Utility.Assert(false, exceptionMessage);
                        throw;
                    }
                }
            } while (isFlushTask == true);

            return;
        }