Пример #1
0
 public void Open(
     RecoveredOrCopiedCheckpointLsn recoveredOrCopiedCheckpointLsn,
     RoleContextDrainState roleContextDrainState,
     OperationProcessor recordsProcessor,
     IStateReplicator fabricReplicator,
     IBackupManager backupManager,
     CheckpointManager checkpointManager,
     TransactionManager transactionManager,
     ReplicatedLogManager replicatedLogManager,
     IStateManager stateManager,
     TransactionalReplicatorSettings replicatorSettings,
     RecoveryManager recoveryManager,
     ITracer tracer)
 {
     this.recoveredOrCopiedCheckpointLsn = recoveredOrCopiedCheckpointLsn;
     this.recoveryManager       = recoveryManager;
     this.replicatorSettings    = replicatorSettings;
     this.stateManager          = stateManager;
     this.tracer                = tracer;
     this.roleContextDrainState = roleContextDrainState;
     this.recordsProcessor      = recordsProcessor;
     this.fabricReplicator      = fabricReplicator;
     this.backupManager         = backupManager;
     this.transactionManager    = transactionManager;
     this.checkpointManager     = checkpointManager;
     this.replicatedLogManager  = replicatedLogManager;
 }
Пример #2
0
        public TransactionManager(
            RecoveredOrCopiedCheckpointLsn recoveredOrCopiedCheckpointLsn,
            TransactionMap transactionMap,
            FabricPerformanceCounterSetInstance instance,
            CheckpointManager checkpointManager,
            ReplicatedLogManager replicatedLogManager,
            OperationProcessor unlockCallbackManager,
            ITracer tracer,
            long flushAtBufferedBytes)
        {
            this.recoveredOrCopiedCheckpointLsn = recoveredOrCopiedCheckpointLsn;
            this.transactionMap        = transactionMap;
            this.checkpointManager     = checkpointManager;
            this.replicatedLogManager  = replicatedLogManager;
            this.unlockCallbackManager = unlockCallbackManager;
            this.tracer = tracer;
            this.flushAtBufferedBytes = flushAtBufferedBytes;

            this.beginTransactionOperationRatePerformanceCounterWriter =
                new BeginTransactionOperationRatePerformanceCounterWriter(instance);
            this.addOperationRatePerformanceCounterWriter =
                new AddOperationRatePerformanceCounterWriter(instance);
            this.transactionCommitRatePerformanceCounterWriter =
                new TransactionCommitRatePerformanceCounterWriter(instance);
            this.transactionAbortRatePerformanceCounterWriter =
                new TransactionAbortRatePerformanceCounterWriter(instance);
            this.addAtomicOperationRatePerformanceCounterWriter =
                new AddAtomicOperationRatePerformanceCounterWriter(instance);
        }
Пример #3
0
        public async Task PerformRecoveryAsync(
            RecoveredOrCopiedCheckpointLsn recoveredOrCopiedCheckpointLsn,
            OperationProcessor recordsProcessor,
            CheckpointManager checkpointManager,
            TransactionManager transactionManager,
            LogRecordsDispatcher logRecordsDispatcher,
            ReplicatedLogManager replicatedLogManager)
        {
            var lastPhysicalRecord = PhysicalLogRecord.InvalidPhysicalLogRecord;
            LogicalSequenceNumber lsn;
            var recoveredRecords = new List <LogRecord>();

            var recoveredLastCompletedBeginCheckpointRecord = await this.RecoveryLogsReader.GetLastCompletedBeginCheckpointRecord(this.RecoveredLastEndCheckpointRecord).ConfigureAwait(false);

            var recoveredLastCompletedBeginCheckpointRecordPosition = recoveredLastCompletedBeginCheckpointRecord.RecordPosition;
            var recoveryStartingPosition = recoveredLastCompletedBeginCheckpointRecordPosition - recoveredLastCompletedBeginCheckpointRecord.EarliestPendingTransactionOffset;

            // Set the recovered lsn before performing recovery as it impacts unlock callback logic during recovery
            recoveredOrCopiedCheckpointLsn.Update(recoveredLastCompletedBeginCheckpointRecord.Lsn);

            var trace =
                string.Format(
                    CultureInfo.InvariantCulture,
                    "PerformRecoveryAsync: " + Environment.NewLine
                    + "      Recovered last completed begin checkpoint record." + Environment.NewLine
                    + "      Type: {0} LSN: {1} PSN:{2} Position: {3}" + Environment.NewLine
                    + "      Recovery Starting Position: {4} Recovery Starting Epoch: {5},{6}" + Environment.NewLine
                    + "      LogCompleteCheckpointAfterRecovery: {7}" + "      Recovered ProgressVector: {8}"
                    + Environment.NewLine,
                    recoveredLastCompletedBeginCheckpointRecord.RecordType,
                    recoveredLastCompletedBeginCheckpointRecord.Lsn.LSN,
                    recoveredLastCompletedBeginCheckpointRecord.Psn.PSN,
                    recoveredLastCompletedBeginCheckpointRecord.RecordPosition,
                    recoveryStartingPosition,
                    recoveredLastCompletedBeginCheckpointRecord.Epoch.DataLossNumber,
                    recoveredLastCompletedBeginCheckpointRecord.Epoch.ConfigurationNumber,
                    this.logCompleteCheckpointAfterRecovery,
                    recoveredLastCompletedBeginCheckpointRecord.ProgressVector.ToString(Constants.ProgressVectorMaxStringSizeInKb));

            FabricEvents.Events.Api(tracer.Type, trace);

            LogRecord lastRecoverableRecord = null;

            replicatedLogManager.LogManager.SetSequentialAccessReadSize(
                this.RecoveryLogsReader.ReadStream,
                LogManager.ReadAheadBytesForSequentialStream);

            // Updated to the recovered value in LogRecordsMap during successful recovery
            long lastPeriodicTruncationTimeTicks = 0;

            using (var records = new LogRecords(this.RecoveryLogsReader.ReadStream, recoveryStartingPosition, this.tailRecordAtStart.RecordPosition))
            {
                var hasMoved = await records.MoveNextAsync(CancellationToken.None).ConfigureAwait(false);

                Utility.Assert(hasMoved == true, "hasMoved == true");

                lsn = records.Current.Lsn;
                if (recoveredLastCompletedBeginCheckpointRecord.EarliestPendingTransactionOffset != 0)
                {
                    lsn -= 1;
                }

                var logRecordsMap = new LogRecordsMap(
                    lsn,
                    transactionManager.TransactionsMap,
                    recoveredLastCompletedBeginCheckpointRecord.Epoch,
                    checkpointManager.LastStableLsn,
                    recoveredLastCompletedBeginCheckpointRecord.ProgressVector,
                    recoveredLastCompletedBeginCheckpointRecord,
                    this.tracer,
                    this.RecoveredLastEndCheckpointRecord);

                do
                {
                    var isRecoverableRecord = true;
                    var record = records.Current;
                    record.CompletedFlush(null);

                    Utility.Assert(
                        record.PreviousPhysicalRecord == logRecordsMap.LastPhysicalRecord,
                        "record.PreviousPhysicalRecord == lastPhysicalRecord");

                    logRecordsMap.ProcessLogRecord(record, out isRecoverableRecord);

                    if (isRecoverableRecord == true)
                    {
                        recoveredRecords.Add(record);
                        lastRecoverableRecord = record;

                        if (LoggingReplicator.IsBarrierRecord(record) &&
                            recoveredRecords.Count >= ReplicatorDynamicConstants.ParallelRecoveryBatchSize)
                        {
                            logRecordsDispatcher.DispatchLoggedRecords(new LoggedRecords(recoveredRecords, null));
                            recoveredRecords       = new List <LogRecord>();
                            this.recoveryException = recordsProcessor.ServiceException;

                            if (this.recoveryException == null)
                            {
                                this.recoveryException = recordsProcessor.LogException;
                            }

                            if (this.recoveryException != null)
                            {
                                // If there was an apply or unlock failure, report fault does not help during recovery because the replica is not opened yet.
                                // The only solution here is to throw during OpenAsync

                                FabricEvents.Events.Api(
                                    tracer.Type,
                                    "PerformRecoveryAsync: RecoveryFailed");

                                await lastRecoverableRecord.AwaitProcessing().ConfigureAwait(false);

                                await recordsProcessor.WaitForAllRecordsProcessingAsync().ConfigureAwait(false);

                                throw this.recoveryException;
                            }
                        }
                    }
                    else
                    {
                        Utility.Assert(LoggingReplicator.IsBarrierRecord(record) == false, "IsBarrierRecord(record) == false");
                        record.CompletedApply(null);
                        record.CompletedProcessing(null);
                    }

                    hasMoved = await records.MoveNextAsync(CancellationToken.None).ConfigureAwait(false);
                } while (hasMoved == true);

                this.LastRecoveredAtomicRedoOperationLsn = logRecordsMap.LastRecoveredAtomicRedoOperationLsn;
                checkpointManager.ResetStableLsn(logRecordsMap.LastStableLsn);
                lastPhysicalRecord = logRecordsMap.LastPhysicalRecord;
                lsn = logRecordsMap.LastLogicalSequenceNumber;

                if (recoveredRecords.Count > 0)
                {
                    logRecordsDispatcher.DispatchLoggedRecords(new LoggedRecords(recoveredRecords, null));
                }

                var tailRecord = records.Current;

                FabricEvents.Events.Api(
                    tracer.Type,
                    "PerformRecoveryAsync: Current tail record Type: " + tailRecord.RecordType + " LSN: "
                    + tailRecord.Lsn.LSN + " PSN: " + tailRecord.Psn.PSN + " Position: " + tailRecord.RecordPosition);

                Utility.Assert(
                    lastPhysicalRecord == records.LastPhysicalRecord,
                    "lastPhysicalRecord == records.LastPhysicalRecord");
                Utility.Assert(
                    (tailRecord == lastPhysicalRecord) || (tailRecord.PreviousPhysicalRecord == lastPhysicalRecord),
                    "(tailRecord == lastPhysicalRecord) || "
                    + "(tailRecord.PreviousPhysicalRecord == lastPhysicalRecord), " + "LastPhysicalRecord PSN: {0}",
                    lastPhysicalRecord.Psn.PSN);
                Utility.Assert(
                    logRecordsMap.LastCompletedEndCheckpointRecord != null,
                    "this.lastCompletedEndCheckpointRecord != null");
                Utility.Assert(
                    logRecordsMap.LastInProgressCheckpointRecord == null,
                    "this.lastInProgressCheckpointRecord == null");
                Utility.Assert(logRecordsMap.LastLinkedPhysicalRecord != null, "this.lastLinkedPhysicalRecord != nul");
                Utility.Assert(lsn == tailRecord.Lsn, "lsn == tailRecord.LastLogicalSequenceNumber");

                // Disable read ahead as indexing physical records will read the log backwards
                replicatedLogManager.LogManager.SetSequentialAccessReadSize(this.RecoveryLogsReader.ReadStream, 0);

                await this.RecoveryLogsReader.IndexPhysicalRecords(lastPhysicalRecord).ConfigureAwait(false);

                var callbackManager = new PhysicalLogWriterCallbackManager(this.flushedRecordsCallback);
                callbackManager.FlushedPsn = tailRecord.Psn + 1;
                replicatedLogManager.LogManager.PrepareToLog(tailRecord, callbackManager);

                replicatedLogManager.Reuse(
                    recoveredLastCompletedBeginCheckpointRecord.ProgressVector,
                    logRecordsMap.LastCompletedEndCheckpointRecord,
                    logRecordsMap.LastInProgressCheckpointRecord,
                    logRecordsMap.LastLinkedPhysicalRecord,
                    replicatedLogManager.LastInformationRecord,
                    (IndexingLogRecord)this.RecoveryLogsReader.StartingRecord,
                    logRecordsMap.CurrentLogTailEpoch,
                    lsn);

                this.RecoveredLsn = lsn;

                // GopalK: The order of the following statements is significant
                if (this.logCompleteCheckpointAfterRecovery)
                {
                    replicatedLogManager.CompleteCheckpoint();
                }

                replicatedLogManager.Information(InformationEvent.Recovered);

                await replicatedLogManager.LogManager.FlushAsync("PerformRecovery").ConfigureAwait(false);

                await lastRecoverableRecord.AwaitProcessing().ConfigureAwait(false);

                await recordsProcessor.WaitForAllRecordsProcessingAsync().ConfigureAwait(false);

                lastPeriodicTruncationTimeTicks = logRecordsMap.LastPeriodicTruncationTimeTicks;
            }

            this.recoveryException = recordsProcessor.ServiceException;
            if (this.recoveryException == null)
            {
                this.recoveryException = recordsProcessor.LogException;
            }

            if (this.recoveryException != null)
            {
                FabricEvents.Events.Api(
                    tracer.Type,
                    "PerformRecoveryAsync: RecoveryFailed");

                // If there was an apply or unlock failure, report fault does not help during recovery because the replica is not opened yet.
                // The only solution here is to throw during OpenAsync
                throw this.recoveryException;
            }

            checkpointManager.Recover(
                recoveredLastCompletedBeginCheckpointRecord,
                lastPeriodicTruncationTimeTicks);
        }