internal PhysicalLogWriter( ILogicalLog logicalLogStream, LogRecord tailRecord, PhysicalLogWriterCallbackManager callbackManager, Exception closedException, ITracer tracer, int maxWriteCacheSizeInMB, IncomingBytesRateCounterWriter incomingBytesRateCounterWriter, LogFlushBytesRateCounterWriter logFlushBytesRateCounterWriter, AvgBytesPerFlushCounterWriter bytesPerFlushCounterWriter, AvgFlushLatencyCounterWriter avgFlushLatencyCounterWriter, AvgSerializationLatencyCounterWriter avgSerializationLatencyCounterWriter) { Utility.Assert(callbackManager != null, "PhysicalLogWriter cannot accept null callback managers"); this.incomingBytesRateCounterWriter = incomingBytesRateCounterWriter; this.logFlushBytesRateCounterWriter = logFlushBytesRateCounterWriter; this.bytesPerFlushCounterWriter = bytesPerFlushCounterWriter; this.avgFlushLatencyCounterWriter = avgFlushLatencyCounterWriter; this.avgSerializationLatencyCounterWriter = avgSerializationLatencyCounterWriter; this.maxWriteCacheSizeInBytes = maxWriteCacheSizeInMB * 1024 * 1024; this.Init(tracer, closedException); this.logicalLogStream = logicalLogStream; this.callbackManager = callbackManager; this.SetTailRecord(tailRecord); Utility.Assert( this.currentLogTailPosition == (ulong)logicalLogStream.WritePosition, "this.currentLogTailPosition:{0} should be equal to logicalLogStream.WritePosition:{1}", this.currentLogTailPosition, logicalLogStream.WritePosition); this.recordWriter = new BinaryWriter(new MemoryStream()); this.recomputeRecordOffsets = false; }
internal PhysicalLogWriter( ILogicalLog logicalLogStream, PhysicalLogWriterCallbackManager callbackManager, ITracer tracer, int maxWriteCacheSizeInMB, IncomingBytesRateCounterWriter incomingBytesRateCounterWriter, LogFlushBytesRateCounterWriter logFlushBytesRateCounterWriter, AvgBytesPerFlushCounterWriter bytesPerFlushCounterWriter, AvgFlushLatencyCounterWriter avgFlushLatencyCounterWriter, AvgSerializationLatencyCounterWriter avgSerializationLatencyCounterWriter, bool recomputeRecordOffsets) { Utility.Assert(callbackManager != null, "PhysicalLogWriter cannot accept null callback managers"); this.maxWriteCacheSizeInBytes = maxWriteCacheSizeInMB * 1024 * 1024; this.Init(tracer, null); this.logicalLogStream = logicalLogStream; this.callbackManager = callbackManager; this.currentLogTailPosition = (ulong)logicalLogStream.WritePosition; this.currentLogTailRecord = LogRecord.InvalidLogRecord; this.currentLogTailPsn = new PhysicalSequenceNumber(-1); this.lastPhysicalRecord = null; this.incomingBytesRateCounterWriter = incomingBytesRateCounterWriter; this.logFlushBytesRateCounterWriter = logFlushBytesRateCounterWriter; this.bytesPerFlushCounterWriter = bytesPerFlushCounterWriter; this.avgFlushLatencyCounterWriter = avgFlushLatencyCounterWriter; this.avgSerializationLatencyCounterWriter = avgSerializationLatencyCounterWriter; this.recordWriter = new BinaryWriter(new MemoryStream()); this.recomputeRecordOffsets = recomputeRecordOffsets; }
internal virtual Task InitializeAsync(ITracer tracer, LogManagerInitialization init, TransactionalReplicatorSettings settings) { this.Tracer = tracer; this.LogFileDirectoryPath = init.dedicatedLogPath; this.MaxWriteCacheSizeInMB = DefaultWriteCacheSizeMb; this.emptyCallbackManager = new PhysicalLogWriterCallbackManager(this.EmptyFlushedRecordsCallback, tracer); return(Task.FromResult(0)); }
internal override async Task <IndexingLogRecord> CreateCopyLogAsync( Epoch startingEpoch, LogicalSequenceNumber startingLsn) { var flushCallback = this.PhysicalLogWriter.CallbackManager.Callback; await this.CloseCurrentLogAsync().ConfigureAwait(false); this.CurrentLogFileAlias = this.BaseLogFileAlias + CopySuffix; try { var aliasGuid = await this.physicalLog.ResolveAliasAsync(this.CurrentLogFileAlias, CancellationToken.None).ConfigureAwait(false); FabricEvents.Events.LogManager( this.Tracer.Type, "CreateCopyLog: Attempt to delete logical log " + this.CurrentLogFileAlias + " guid: " + aliasGuid); await this.physicalLog.DeleteLogicalLogAsync(aliasGuid, CancellationToken.None).ConfigureAwait(false); } catch (Exception ex) { FabricEvents.Events.LogManager( this.Tracer.Type, "CreateCopyLog: Delete logical log: " + this.CurrentLogFileAlias + " failed: " + ex); } this.LogicalLog = await this.CreateLogFileAsync(true, CancellationToken.None).ConfigureAwait(false); var callbackManager = new PhysicalLogWriterCallbackManager( flushCallback, this.Tracer); this.PhysicalLogWriter = new PhysicalLogWriter( this.LogicalLog, callbackManager, this.Tracer, this.MaxWriteCacheSizeInMB, this.IncomingBytesRateCounterWriter, this.LogFlushBytesRateCounterWriter, this.BytesPerFlushCounterWriter, this.AvgFlushLatencyCounterWriter, this.AvgSerializationLatencyCounterWriter, false); var firstIndexingRecord = new IndexingLogRecord(startingEpoch, startingLsn, null); this.PhysicalLogWriter.InsertBufferedRecord(firstIndexingRecord); await this.PhysicalLogWriter.FlushAsync("CreateCopyLogAsync").ConfigureAwait(false); this.LogHeadRecordPosition = firstIndexingRecord.RecordPosition; return(firstIndexingRecord); }
internal void PrepareToLog(LogRecord tailRecord, PhysicalLogWriterCallbackManager callbackManager) { this.PhysicalLogWriter = new PhysicalLogWriter( this.LogicalLog, tailRecord, callbackManager, null, this.Tracer, this.MaxWriteCacheSizeInMB, this.IncomingBytesRateCounterWriter, this.LogFlushBytesRateCounterWriter, this.BytesPerFlushCounterWriter, this.AvgFlushLatencyCounterWriter, this.AvgSerializationLatencyCounterWriter); }
internal override async Task <IndexingLogRecord> CreateCopyLogAsync( Epoch startingEpoch, LogicalSequenceNumber startingLsn) { var flushCallback = this.PhysicalLogWriter.CallbackManager.Callback; await this.CloseCurrentLogAsync().ConfigureAwait(false); this.CurrentLogFileAlias = this.BaseLogFileAlias + CopySuffix; this.LogicalLog = await this.CreateLogFileAsync(true, CancellationToken.None).ConfigureAwait(false); var callbackManager = new PhysicalLogWriterCallbackManager( flushCallback, this.Tracer); this.PhysicalLogWriter = new PhysicalLogWriter( this.LogicalLog, callbackManager, this.Tracer, this.MaxWriteCacheSizeInMB, this.IncomingBytesRateCounterWriter, this.LogFlushBytesRateCounterWriter, this.BytesPerFlushCounterWriter, this.AvgFlushLatencyCounterWriter, this.AvgSerializationLatencyCounterWriter, false); var firstIndexingRecord = new IndexingLogRecord(startingEpoch, startingLsn, null); this.PhysicalLogWriter.InsertBufferedRecord(firstIndexingRecord); await this.PhysicalLogWriter.FlushAsync("CreateCopyLogAsync").ConfigureAwait(false); this.LogHeadRecordPosition = firstIndexingRecord.RecordPosition; return(firstIndexingRecord); }
public async Task PerformRecoveryAsync( RecoveredOrCopiedCheckpointLsn recoveredOrCopiedCheckpointLsn, OperationProcessor recordsProcessor, CheckpointManager checkpointManager, TransactionManager transactionManager, LogRecordsDispatcher logRecordsDispatcher, ReplicatedLogManager replicatedLogManager) { var lastPhysicalRecord = PhysicalLogRecord.InvalidPhysicalLogRecord; LogicalSequenceNumber lsn; var recoveredRecords = new List <LogRecord>(); var recoveredLastCompletedBeginCheckpointRecord = await this.RecoveryLogsReader.GetLastCompletedBeginCheckpointRecord(this.RecoveredLastEndCheckpointRecord).ConfigureAwait(false); var recoveredLastCompletedBeginCheckpointRecordPosition = recoveredLastCompletedBeginCheckpointRecord.RecordPosition; var recoveryStartingPosition = recoveredLastCompletedBeginCheckpointRecordPosition - recoveredLastCompletedBeginCheckpointRecord.EarliestPendingTransactionOffset; // Set the recovered lsn before performing recovery as it impacts unlock callback logic during recovery recoveredOrCopiedCheckpointLsn.Update(recoveredLastCompletedBeginCheckpointRecord.Lsn); var trace = string.Format( CultureInfo.InvariantCulture, "PerformRecoveryAsync: " + Environment.NewLine + " Recovered last completed begin checkpoint record." + Environment.NewLine + " Type: {0} LSN: {1} PSN:{2} Position: {3}" + Environment.NewLine + " Recovery Starting Position: {4} Recovery Starting Epoch: {5},{6}" + Environment.NewLine + " LogCompleteCheckpointAfterRecovery: {7}" + " Recovered ProgressVector: {8}" + Environment.NewLine, recoveredLastCompletedBeginCheckpointRecord.RecordType, recoveredLastCompletedBeginCheckpointRecord.Lsn.LSN, recoveredLastCompletedBeginCheckpointRecord.Psn.PSN, recoveredLastCompletedBeginCheckpointRecord.RecordPosition, recoveryStartingPosition, recoveredLastCompletedBeginCheckpointRecord.Epoch.DataLossNumber, recoveredLastCompletedBeginCheckpointRecord.Epoch.ConfigurationNumber, this.logCompleteCheckpointAfterRecovery, recoveredLastCompletedBeginCheckpointRecord.ProgressVector.ToString(Constants.ProgressVectorMaxStringSizeInKb)); FabricEvents.Events.Api(tracer.Type, trace); LogRecord lastRecoverableRecord = null; replicatedLogManager.LogManager.SetSequentialAccessReadSize( this.RecoveryLogsReader.ReadStream, LogManager.ReadAheadBytesForSequentialStream); // Updated to the recovered value in LogRecordsMap during successful recovery long lastPeriodicTruncationTimeTicks = 0; using (var records = new LogRecords(this.RecoveryLogsReader.ReadStream, recoveryStartingPosition, this.tailRecordAtStart.RecordPosition)) { var hasMoved = await records.MoveNextAsync(CancellationToken.None).ConfigureAwait(false); Utility.Assert(hasMoved == true, "hasMoved == true"); lsn = records.Current.Lsn; if (recoveredLastCompletedBeginCheckpointRecord.EarliestPendingTransactionOffset != 0) { lsn -= 1; } var logRecordsMap = new LogRecordsMap( lsn, transactionManager.TransactionsMap, recoveredLastCompletedBeginCheckpointRecord.Epoch, checkpointManager.LastStableLsn, recoveredLastCompletedBeginCheckpointRecord.ProgressVector, recoveredLastCompletedBeginCheckpointRecord, this.tracer, this.RecoveredLastEndCheckpointRecord); do { var isRecoverableRecord = true; var record = records.Current; record.CompletedFlush(null); Utility.Assert( record.PreviousPhysicalRecord == logRecordsMap.LastPhysicalRecord, "record.PreviousPhysicalRecord == lastPhysicalRecord"); logRecordsMap.ProcessLogRecord(record, out isRecoverableRecord); if (isRecoverableRecord == true) { recoveredRecords.Add(record); lastRecoverableRecord = record; if (LoggingReplicator.IsBarrierRecord(record) && recoveredRecords.Count >= ReplicatorDynamicConstants.ParallelRecoveryBatchSize) { logRecordsDispatcher.DispatchLoggedRecords(new LoggedRecords(recoveredRecords, null)); recoveredRecords = new List <LogRecord>(); this.recoveryException = recordsProcessor.ServiceException; if (this.recoveryException == null) { this.recoveryException = recordsProcessor.LogException; } if (this.recoveryException != null) { // If there was an apply or unlock failure, report fault does not help during recovery because the replica is not opened yet. // The only solution here is to throw during OpenAsync FabricEvents.Events.Api( tracer.Type, "PerformRecoveryAsync: RecoveryFailed"); await lastRecoverableRecord.AwaitProcessing().ConfigureAwait(false); await recordsProcessor.WaitForAllRecordsProcessingAsync().ConfigureAwait(false); throw this.recoveryException; } } } else { Utility.Assert(LoggingReplicator.IsBarrierRecord(record) == false, "IsBarrierRecord(record) == false"); record.CompletedApply(null); record.CompletedProcessing(null); } hasMoved = await records.MoveNextAsync(CancellationToken.None).ConfigureAwait(false); } while (hasMoved == true); this.LastRecoveredAtomicRedoOperationLsn = logRecordsMap.LastRecoveredAtomicRedoOperationLsn; checkpointManager.ResetStableLsn(logRecordsMap.LastStableLsn); lastPhysicalRecord = logRecordsMap.LastPhysicalRecord; lsn = logRecordsMap.LastLogicalSequenceNumber; if (recoveredRecords.Count > 0) { logRecordsDispatcher.DispatchLoggedRecords(new LoggedRecords(recoveredRecords, null)); } var tailRecord = records.Current; FabricEvents.Events.Api( tracer.Type, "PerformRecoveryAsync: Current tail record Type: " + tailRecord.RecordType + " LSN: " + tailRecord.Lsn.LSN + " PSN: " + tailRecord.Psn.PSN + " Position: " + tailRecord.RecordPosition); Utility.Assert( lastPhysicalRecord == records.LastPhysicalRecord, "lastPhysicalRecord == records.LastPhysicalRecord"); Utility.Assert( (tailRecord == lastPhysicalRecord) || (tailRecord.PreviousPhysicalRecord == lastPhysicalRecord), "(tailRecord == lastPhysicalRecord) || " + "(tailRecord.PreviousPhysicalRecord == lastPhysicalRecord), " + "LastPhysicalRecord PSN: {0}", lastPhysicalRecord.Psn.PSN); Utility.Assert( logRecordsMap.LastCompletedEndCheckpointRecord != null, "this.lastCompletedEndCheckpointRecord != null"); Utility.Assert( logRecordsMap.LastInProgressCheckpointRecord == null, "this.lastInProgressCheckpointRecord == null"); Utility.Assert(logRecordsMap.LastLinkedPhysicalRecord != null, "this.lastLinkedPhysicalRecord != nul"); Utility.Assert(lsn == tailRecord.Lsn, "lsn == tailRecord.LastLogicalSequenceNumber"); // Disable read ahead as indexing physical records will read the log backwards replicatedLogManager.LogManager.SetSequentialAccessReadSize(this.RecoveryLogsReader.ReadStream, 0); await this.RecoveryLogsReader.IndexPhysicalRecords(lastPhysicalRecord).ConfigureAwait(false); var callbackManager = new PhysicalLogWriterCallbackManager(this.flushedRecordsCallback); callbackManager.FlushedPsn = tailRecord.Psn + 1; replicatedLogManager.LogManager.PrepareToLog(tailRecord, callbackManager); replicatedLogManager.Reuse( recoveredLastCompletedBeginCheckpointRecord.ProgressVector, logRecordsMap.LastCompletedEndCheckpointRecord, logRecordsMap.LastInProgressCheckpointRecord, logRecordsMap.LastLinkedPhysicalRecord, replicatedLogManager.LastInformationRecord, (IndexingLogRecord)this.RecoveryLogsReader.StartingRecord, logRecordsMap.CurrentLogTailEpoch, lsn); this.RecoveredLsn = lsn; // GopalK: The order of the following statements is significant if (this.logCompleteCheckpointAfterRecovery) { replicatedLogManager.CompleteCheckpoint(); } replicatedLogManager.Information(InformationEvent.Recovered); await replicatedLogManager.LogManager.FlushAsync("PerformRecovery").ConfigureAwait(false); await lastRecoverableRecord.AwaitProcessing().ConfigureAwait(false); await recordsProcessor.WaitForAllRecordsProcessingAsync().ConfigureAwait(false); lastPeriodicTruncationTimeTicks = logRecordsMap.LastPeriodicTruncationTimeTicks; } this.recoveryException = recordsProcessor.ServiceException; if (this.recoveryException == null) { this.recoveryException = recordsProcessor.LogException; } if (this.recoveryException != null) { FabricEvents.Events.Api( tracer.Type, "PerformRecoveryAsync: RecoveryFailed"); // If there was an apply or unlock failure, report fault does not help during recovery because the replica is not opened yet. // The only solution here is to throw during OpenAsync throw this.recoveryException; } checkpointManager.Recover( recoveredLastCompletedBeginCheckpointRecord, lastPeriodicTruncationTimeTicks); }