internal CompleteCheckpointLogRecord( LogicalSequenceNumber lsn, IndexingLogRecord logHeadRecord, PhysicalLogRecord lastLinkedPhysicalRecord) : base(LogRecordType.CompleteCheckpoint, logHeadRecord, lsn, lastLinkedPhysicalRecord) { }
protected PhysicalLogRecord() : base() { this.linkedPhysicalRecordOffset = InvalidPhysicalRecordOffset; this.linkedPhysicalRecord = null; this.nextPhysicalRecord = null; }
// // Invoked when this indexing record is chosen as the new head // Returns the number of physical links we jumped ahead to free previous backward references and how many actually free'd links older than the head // // This also implies we cannot have backward references from logical records to other records // If this requirement comes up, we must be able to nagivate through the logical records 1 by 1 to free the links // public void OnTruncateHead(out int freeLinkCallCount, out int freeLinkCallTrueCount) { freeLinkCallCount = 0; freeLinkCallTrueCount = 0; // Start from the current record (new head) all the way to the latest physical record free-ing previous links if they point to earlier than the new head // Without doing so, we slowly start leaking physical links // Refer to V2ReplicatorTests/unittest/Integration/PhysicalRecordLeakTest.cs PhysicalLogRecord currentPhysicalRecord = this; do { freeLinkCallCount += 1; if (currentPhysicalRecord.FreePreviousLinksLowerThanPsn(Psn)) { freeLinkCallTrueCount += 1; } currentPhysicalRecord = currentPhysicalRecord.NextPhysicalRecord; // We reach the end of the chain if (currentPhysicalRecord.Psn == PhysicalSequenceNumber.InvalidPsn) { break; } } while (true); }
protected PhysicalLogRecord(LogRecordType recordType, ulong recordPosition, long lsn) : base(recordType, recordPosition, lsn) { this.linkedPhysicalRecordOffset = InvalidPhysicalRecordOffset; this.linkedPhysicalRecord = InvalidPhysicalLogRecord; this.nextPhysicalRecord = new WeakReference <PhysicalLogRecord>(InvalidPhysicalLogRecord); }
internal PhysicalLogWriter( ILogicalLog logicalLogStream, PhysicalLogWriterCallbackManager callbackManager, ITracer tracer, int maxWriteCacheSizeInMB, IncomingBytesRateCounterWriter incomingBytesRateCounterWriter, LogFlushBytesRateCounterWriter logFlushBytesRateCounterWriter, AvgBytesPerFlushCounterWriter bytesPerFlushCounterWriter, AvgFlushLatencyCounterWriter avgFlushLatencyCounterWriter, AvgSerializationLatencyCounterWriter avgSerializationLatencyCounterWriter, bool recomputeRecordOffsets) { Utility.Assert(callbackManager != null, "PhysicalLogWriter cannot accept null callback managers"); this.maxWriteCacheSizeInBytes = maxWriteCacheSizeInMB * 1024 * 1024; this.Init(tracer, null); this.logicalLogStream = logicalLogStream; this.callbackManager = callbackManager; this.currentLogTailPosition = (ulong)logicalLogStream.WritePosition; this.currentLogTailRecord = LogRecord.InvalidLogRecord; this.currentLogTailPsn = new PhysicalSequenceNumber(-1); this.lastPhysicalRecord = null; this.incomingBytesRateCounterWriter = incomingBytesRateCounterWriter; this.logFlushBytesRateCounterWriter = logFlushBytesRateCounterWriter; this.bytesPerFlushCounterWriter = bytesPerFlushCounterWriter; this.avgFlushLatencyCounterWriter = avgFlushLatencyCounterWriter; this.avgSerializationLatencyCounterWriter = avgSerializationLatencyCounterWriter; this.recordWriter = new BinaryWriter(new MemoryStream()); this.recomputeRecordOffsets = recomputeRecordOffsets; }
/// <summary> /// Inserts a record to the buffer where it waits to be flushed in batches. /// A successful run will set following properties in the record. /// record.Psn and record.PreviousPhysicalLogRecord /// </summary> /// <param name="record"></param> /// <returns>The pending size of the buffered records</returns> internal long InsertBufferedRecord(LogRecord record) { Utility.Assert(LogRecord.IsInvalidRecord(record) == false, "LogRecord.IsInvalidRecord(record) == false"); var physicalRecord = record as PhysicalLogRecord; lock (this.flushLock) { if (this.closedException == null) { // No record can be inserted after 'RemovingState' Information log record if (this.lastPhysicalRecord != null && this.lastPhysicalRecord.RecordType == LogRecordType.Information) { var lastInformationRecord = this.lastPhysicalRecord as InformationLogRecord; Utility.Assert( lastInformationRecord.InformationEvent != InformationEvent.RemovingState, "No record should be inserted after 'RemovingState'. Current Record = {0}", record); } // Update record and tail ++this.currentLogTailPsn; record.Psn = this.currentLogTailPsn; this.incomingBytesRateCounterWriter.IncrementBy(record.ApproximateSizeOnDisk); record.PreviousPhysicalRecord = this.lastPhysicalRecord; if (physicalRecord != null) { if (this.lastPhysicalRecord != null) { this.lastPhysicalRecord.NextPhysicalRecord = physicalRecord; } this.lastPhysicalRecord = physicalRecord; } // Buffer record if (this.bufferedRecords == null) { this.bufferedRecords = new List <LogRecord>(); } this.bufferedRecords.Add(record); var newBufferedRecordsBytes = Interlocked.Add( ref this.bufferedRecordsBytes, record.ApproximateSizeOnDisk); return(newBufferedRecordsBytes); } } var failedRecord = new LoggedRecords(record, this.closedException); this.ProcessFlushedRecords(failedRecord); return(Interlocked.Read(ref this.bufferedRecordsBytes)); }
internal IndexingLogRecord( Epoch currentEpoch, LogicalSequenceNumber lsn, PhysicalLogRecord linkedPhysicalRecord) : base(LogRecordType.Indexing, lsn, linkedPhysicalRecord) { this.currentEpoch = currentEpoch; this.UpdateApproximateDiskSize(); }
internal InformationLogRecord( LogicalSequenceNumber lsn, PhysicalLogRecord linkedPhysicalRecord, InformationEvent informationEvent) : base(LogRecordType.Information, lsn, linkedPhysicalRecord) { this.Initialize(informationEvent); this.UpdateApproximateDiskSize(); }
public async Task <bool> MoveNextAsync(CancellationToken cancellationToken) { if (this.readStream.Position <= ((long)this.enumerationEndingPosition) && this.isDisposed == false) { var record = await LogRecord.ReadNextRecordAsync(this.readStream).ConfigureAwait(false); if (record != null) { var physicalRecord = record as PhysicalLogRecord; if (LogRecord.IsInvalidRecord(this.lastPhysicalRecord) == false) { record.PreviousPhysicalRecord = this.lastPhysicalRecord; if (physicalRecord != null) { this.lastPhysicalRecord.NextPhysicalRecord = physicalRecord; this.lastPhysicalRecord = physicalRecord; } } else if (physicalRecord != null) { this.lastPhysicalRecord = physicalRecord; } this.currentRecord = record; // When the starting LSN is not initialized (recovery reader case), initialize it to the first record's LSN if (this.enumerationStartedAtLsn == long.MaxValue) { this.enumerationStartedAtLsn = this.currentRecord.Lsn.LSN; } // Aggresively trim the starting position for enabling truncations if (this.logManager != null && this.currentRecord.RecordPosition - this.enumerationStartingPosition > UpdateStartingPositionAfterBytes) { // First, add the new range and only then remove the older one and update the starting position for removing it later again var isValid = this.logManager.AddLogReader( this.enumerationStartedAtLsn, this.currentRecord.RecordPosition, this.enumerationEndingPosition, this.readerName, this.readerType); Utility.Assert(isValid, "isValid == true"); this.logManager.RemoveLogReader(this.enumerationStartingPosition); this.enumerationStartingPosition = this.currentRecord.RecordPosition; } return(true); } } return(false); }
internal async Task IndexPhysicalRecords(PhysicalLogRecord record) { while (record.RecordPosition > this.startingRecordPosition) { record = await this.GetPreviousPhysicalRecord(record).ConfigureAwait(false); } Utility.Assert( LogRecord.IsInvalidRecord(this.startingRecord) == false, "LogRecord.IsInvalidRecord(this.startingRecord) == false"); }
/// <summary> /// Do a check if our log matches truncate head criteria, and append truncate head record /// </summary> public TruncateHeadLogRecord TruncateHead( bool isStable, long lastPeriodicTruncationTimeTicks, Func <IndexingLogRecord, bool> isGoodLogHeadCandidateCalculator) { lock (lsnOrderingLock) { var earliestRecord = this.GetEarliestRecordCallerHoldsLock(); IndexingLogRecord previousIndexingRecord = null; PhysicalLogRecord previousPhysicalRecord = this.LastCompletedBeginCheckpointRecord; TruncateHeadLogRecord record = null; do { // Search for the last Indexing Log Record do { previousPhysicalRecord = previousPhysicalRecord.PreviousPhysicalRecord; } while (!(previousPhysicalRecord is IndexingLogRecord)); previousIndexingRecord = previousPhysicalRecord as IndexingLogRecord; // This index record is not before the ealiest pending transaction record if (previousIndexingRecord.RecordPosition >= earliestRecord.RecordPosition) { continue; } // We reached log head, so do not continue to look for last index log record if (previousIndexingRecord == this.CurrentLogHeadRecord) { return(null); } if (isGoodLogHeadCandidateCalculator(previousIndexingRecord)) { break; } } while (true); record = new TruncateHeadLogRecord( previousIndexingRecord, this.CurrentLogTailLsn, this.LastLinkedPhysicalRecord, isStable, lastPeriodicTruncationTimeTicks); Test_TruncateHead(record, previousIndexingRecord); return(record); } }
internal TruncateHeadLogRecord( IndexingLogRecord logHeadRecord, LogicalSequenceNumber lsn, PhysicalLogRecord lastLinkedPhysicalRecord, bool isStable, long periodicTruncationTimeTicks) : base(LogRecordType.TruncateHead, logHeadRecord, lsn, lastLinkedPhysicalRecord) { this.truncationState = TruncationState.Invalid; this.isStable = isStable; this.periodicTruncationTimeTicks = periodicTruncationTimeTicks; this.UpdateApproximateDiskSize(); }
internal LogHeadRecord( LogRecordType recordType, IndexingLogRecord logHeadRecord, LogicalSequenceNumber lsn, PhysicalLogRecord lastLinkedPhysicalRecord) : base(recordType, lsn, lastLinkedPhysicalRecord) { this.logHeadEpoch = logHeadRecord.CurrentEpoch; this.logHeadLsn = logHeadRecord.Lsn; this.logHeadPsn = logHeadRecord.Psn; this.logHeadRecordOffset = InvalidPhysicalRecordOffset; this.logHeadRecord = logHeadRecord; this.UpdateApproximateDiskSize(); }
private void SetTailRecord(LogRecord tailRecord) { this.currentLogTailPosition = tailRecord.RecordPosition + tailRecord.RecordSize; this.currentLogTailRecord = tailRecord; this.currentLogTailPsn = tailRecord.Psn; var physicalRecord = tailRecord as PhysicalLogRecord; this.lastPhysicalRecord = (physicalRecord == null) ? tailRecord.PreviousPhysicalRecord : physicalRecord; Utility.Assert( this.lastPhysicalRecord != PhysicalLogRecord.InvalidPhysicalLogRecord, "this.lastPhysicalRecord ({0}) != PhysicalLogRecord.InvalidPhysicalLogRecord", this.lastPhysicalRecord); return; }
protected LogRecord() { this.recordType = LogRecordType.Invalid; this.lsn = LogicalSequenceNumber.InvalidLsn; this.psn = PhysicalSequenceNumber.InvalidPsn; this.previousPhysicalRecordOffset = InvalidPhysicalRecordOffset; this.recordPosition = InvalidRecordPosition; this.recordLength = InvalidRecordLength; this.previousPhysicalRecord = null; this.flushedTask = this.invalidCompletionTask; this.appliedTask = this.invalidCompletionTask; this.processedTask = this.invalidCompletionTask; this.ApproximateSizeOnDisk = 0; }
protected LogRecord(LogRecordType recordType) { this.recordType = recordType; this.lsn = LogicalSequenceNumber.InvalidLsn; this.psn = PhysicalSequenceNumber.InvalidPsn; this.previousPhysicalRecordOffset = InvalidPhysicalRecordOffset; this.recordLength = InvalidRecordLength; this.recordPosition = InvalidRecordPosition; this.previousPhysicalRecord = PhysicalLogRecord.InvalidPhysicalLogRecord; this.flushedTask = new CompletionTask(); this.appliedTask = new CompletionTask(); this.processedTask = new CompletionTask(); this.UpdateApproximateDiskSize(); }
protected LogRecord(LogRecordType recordType, ulong recordPosition, long lsn) { this.recordType = recordType; this.lsn = new LogicalSequenceNumber(lsn); this.psn = PhysicalSequenceNumber.InvalidPsn; this.previousPhysicalRecordOffset = InvalidPhysicalRecordOffset; this.recordPosition = recordPosition; this.recordLength = InvalidRecordLength; this.previousPhysicalRecord = PhysicalLogRecord.InvalidPhysicalLogRecord; this.flushedTask = new CompletionTask(); this.appliedTask = new CompletionTask(); this.processedTask = new CompletionTask(); this.ApproximateSizeOnDisk = 0; }
internal EndCheckpointLogRecord( BeginCheckpointLogRecord lastCompletedBeginCheckpointRecord, IndexingLogRecord logHeadRecord, LogicalSequenceNumber lsn, PhysicalLogRecord linkedPhysicalRecord) : base(LogRecordType.EndCheckpoint, logHeadRecord, lsn, linkedPhysicalRecord) { this.lastCompletedBeginCheckpointRecordOffset = InvalidPhysicalRecordOffset; this.lastStableLsn = lastCompletedBeginCheckpointRecord.LastStableLsn; this.lastCompletedBeginCheckpointRecord = lastCompletedBeginCheckpointRecord; Utility.Assert( this.lastStableLsn >= this.lastCompletedBeginCheckpointRecord.Lsn, "this.lastStableLsn >= this.lastCompletedBeginCheckpointRecord.LastLogicalSequenceNumber. Last stable lsn is : {0} and last completed checkpoint record is {1}", this.lastStableLsn.LSN, this.lastCompletedBeginCheckpointRecord.Lsn.LSN); this.UpdateApproximateDiskSize(); }
internal virtual bool FreePreviousLinksLowerThanPsn(PhysicalSequenceNumber newHeadPsn) { if (this.previousPhysicalRecord != null && this.previousPhysicalRecord.Psn < newHeadPsn) { Utility.Assert( this.previousPhysicalRecordOffset != InvalidPhysicalRecordOffset, "FreePreviousLinksLowerThanPsn: PreviousPhysicalRecordOffset is invalid. Record LSN is {0}", this.lsn.LSN); this.previousPhysicalRecord = (this.previousPhysicalRecordOffset == 0) ? null : PhysicalLogRecord.InvalidPhysicalLogRecord; } return(false); }
internal override bool FreePreviousLinksLowerThanPsn(PhysicalSequenceNumber newHeadPsn) { bool ret = base.FreePreviousLinksLowerThanPsn(newHeadPsn); if (this.linkedPhysicalRecord != null && this.linkedPhysicalRecord.Psn < newHeadPsn) { Utility.Assert( this.linkedPhysicalRecordOffset != InvalidPhysicalRecordOffset, "FreePreviousLinksLowerThanPsn: PreviousPhysicalRecordOffset is invalid. Record LSN is {0}"); this.linkedPhysicalRecord = (this.linkedPhysicalRecordOffset == 0) ? null : PhysicalLogRecord.InvalidPhysicalLogRecord; return(true); } return(ret); }
public void Reuse( ProgressVector progressVector, EndCheckpointLogRecord lastCompletedEndCheckpointRecord, BeginCheckpointLogRecord lastInProgressBeginCheckpointRecord, PhysicalLogRecord lastLinkedPhysicalRecord, InformationLogRecord lastInformationRecord, IndexingLogRecord currentLogHeadRecord, Epoch tailEpoch, LogicalSequenceNumber tailLsn) { this.LastInProgressCheckpointRecord = lastInProgressBeginCheckpointRecord; Utility.Assert(this.LastInProgressCheckpointRecord == null, "ReInitialize of ReplicatedLogManager must have null in progress checkpoint"); this.LastInProgressTruncateHeadRecord = null; this.ProgressVector = progressVector; this.LastCompletedEndCheckpointRecord = lastCompletedEndCheckpointRecord; this.LastLinkedPhysicalRecord = lastLinkedPhysicalRecord; this.CurrentLogHeadRecord = currentLogHeadRecord; this.LastInformationRecord = lastInformationRecord; this.CurrentLogTailEpoch = tailEpoch; this.CurrentLogTailLsn = tailLsn; }
protected PhysicalLogRecord( LogRecordType recordType, LogicalSequenceNumber lsn, PhysicalLogRecord linkedPhysicalRecord) : base(recordType) { this.Lsn = lsn; if (linkedPhysicalRecord != null) { this.linkedPhysicalRecordOffset = InvalidPhysicalRecordOffset; this.linkedPhysicalRecord = linkedPhysicalRecord; } else { this.linkedPhysicalRecordOffset = 0; this.linkedPhysicalRecord = null; } this.nextPhysicalRecord = new WeakReference <PhysicalLogRecord>(InvalidPhysicalLogRecord); this.UpdateApproximateDiskSize(); }
protected virtual void Read(BinaryReader br, bool isPhysical) { if (isPhysical) { // Metadata section. var startingPosition = br.BaseStream.Position; var sizeOfSection = br.ReadInt32(); var endPosition = startingPosition + sizeOfSection; // Read Physical metadata section this.psn = new PhysicalSequenceNumber(br.ReadInt64()); this.previousPhysicalRecordOffset = br.ReadUInt64(); // Jump to the end of the section ignoring fields that are not understood. Utility.Assert(endPosition >= br.BaseStream.Position, "Could not have read more than section size."); br.BaseStream.Position = endPosition; if (this.previousPhysicalRecordOffset == 0) { this.previousPhysicalRecord = null; } } }
public void ProcessedPhysicalRecord(PhysicalLogRecord record) { if (record.FlushException != null) { FabricEvents.Events.ProcessedPhysicalRecordSkip( this.tracer.Type, (int)this.roleContextDrainState.DrainingStream, record.Psn.PSN); return; } var outstandingNumberOfRecordsBeingProcessed = Interlocked.Decrement(ref this.numberOfPhysicalRecordsBeingProcessed); Utility.Assert( outstandingNumberOfRecordsBeingProcessed >= 0, "outstandingNumberOfRecordsBeingProcessed {0} >= 0", outstandingNumberOfRecordsBeingProcessed); if (outstandingNumberOfRecordsBeingProcessed == 0) { this.physicalRecordsProcessingTcs.SetResult(record); } FabricEvents.Events.ProcessedPhysicalRecord( this.tracer.Type, (int)this.roleContextDrainState.DrainingStream, record.RecordType.ToString(), record.Lsn.LSN, record.Psn.PSN, record.RecordPosition); record.CompletedProcessing(null); return; }
/// <summary> /// Initializes a new instance of the BeginCheckpointLogRecord class. /// </summary> /// <remarks>Called when the replicator decides to checkpoint.</remarks> internal BeginCheckpointLogRecord( bool isFirstCheckpointOnFullCopy, ProgressVector progressVector, BeginTransactionOperationLogRecord earliestPendingTransaction, Epoch headEpoch, Epoch epoch, LogicalSequenceNumber lsn, PhysicalLogRecord lastLinkedPhysicalRecord, BackupLogRecord lastCompletedBackupLogRecord, uint progressVectorMaxEntries, long periodicCheckpointTimeTicks, long periodicTruncationTimeTicks) : base(LogRecordType.BeginCheckpoint, lsn, lastLinkedPhysicalRecord) { this.IsFirstCheckpointOnFullCopy = isFirstCheckpointOnFullCopy; this.progressVector = ProgressVector.Clone(progressVector, progressVectorMaxEntries, lastCompletedBackupLogRecord.HighestBackedUpEpoch, headEpoch); this.earliestPendingTransactionOffset = LogicalLogRecord.InvalidLogicalRecordOffset; this.earliestPendingTransaction = earliestPendingTransaction; this.checkpointState = CheckpointState.Invalid; this.lastStableLsn = LogicalSequenceNumber.InvalidLsn; this.epoch = (earliestPendingTransaction != null) ? earliestPendingTransaction.RecordEpoch : epoch; // Initialize backup log record fields. this.highestBackedUpEpoch = lastCompletedBackupLogRecord.HighestBackedUpEpoch; this.highestBackedUpLsn = lastCompletedBackupLogRecord.HighestBackedUpLsn; this.backupLogRecordCount = lastCompletedBackupLogRecord.BackupLogRecordCount; this.backupLogSize = lastCompletedBackupLogRecord.BackupLogSizeInKB; this.earliestPendingTransactionInvalidated = 0; this.lastPeriodicCheckpointTimeTicks = periodicCheckpointTimeTicks; this.lastPeriodicTruncationTimeTicks = periodicTruncationTimeTicks; this.UpdateApproximateDiskSize(); }
internal async Task <PhysicalLogRecord> GetLinkedPhysicalRecord(PhysicalLogRecord record) { var linkedPhysicalRecord = record.LinkedPhysicalRecord; if (LogRecord.IsInvalidRecord(linkedPhysicalRecord)) { var linkedPhysicalRecordOffset = record.LinkedPhysicalRecordOffset; Utility.Assert( linkedPhysicalRecordOffset != LogRecord.InvalidPhysicalRecordOffset, "linkedPhysicalRecordOffset != PhysicalLogRecord.INVALID_PHYSICAL_RECORD_OFFSET"); if (linkedPhysicalRecordOffset == 0) { record.LinkedPhysicalRecord = null; return(null); } // Read desired linked record var recordPosition = record.RecordPosition; Utility.Assert( recordPosition != LogRecord.InvalidRecordPosition, "recordPosition != LogRecord.INVALID_RECORD_POSITION"); var linkedPhysicalRecordPosition = recordPosition - linkedPhysicalRecordOffset; Utility.Assert( linkedPhysicalRecordPosition >= this.startingRecordPosition, "linkedPhysicalRecordPosition >= this.startingRecordPosition"); linkedPhysicalRecord = (PhysicalLogRecord)await this.GetNextLogRecord(linkedPhysicalRecordPosition).ConfigureAwait(false); record.LinkedPhysicalRecord = linkedPhysicalRecord; } Utility.Assert( (linkedPhysicalRecord == null) == (record.LinkedPhysicalRecordOffset == 0), "(linkedPhysicalRecord == null) == (record.LinkedPhysicalRecordOffset == 0)"); return(linkedPhysicalRecord); }
public void Reset() { this.readStream.Position = (long)this.enumerationStartingPosition; this.currentRecord = LogRecord.InvalidLogRecord; this.lastPhysicalRecord = PhysicalLogRecord.InvalidPhysicalLogRecord; }
/// <summary> /// Links to the last "Linked Physical Log Record". /// </summary> /// <param name="physicalLogRecord">The physical log record.</param> private void ProcessLogRecord(PhysicalLogRecord physicalLogRecord) { this.LastPhysicalRecord = physicalLogRecord; this.LastPhysicalRecord.LinkedPhysicalRecord = this.LastLinkedPhysicalRecord; }
internal TruncateTailLogRecord(LogicalSequenceNumber lsn, PhysicalLogRecord lastLinkedPhysicalRecord) : base(LogRecordType.TruncateTail, lsn, lastLinkedPhysicalRecord) { }