public void Open( RecoveredOrCopiedCheckpointLsn recoveredOrCopiedCheckpointLsn, RoleContextDrainState roleContextDrainState, OperationProcessor recordsProcessor, IStateReplicator fabricReplicator, IBackupManager backupManager, CheckpointManager checkpointManager, TransactionManager transactionManager, ReplicatedLogManager replicatedLogManager, IStateManager stateManager, TransactionalReplicatorSettings replicatorSettings, RecoveryManager recoveryManager, ITracer tracer) { this.recoveredOrCopiedCheckpointLsn = recoveredOrCopiedCheckpointLsn; this.recoveryManager = recoveryManager; this.replicatorSettings = replicatorSettings; this.stateManager = stateManager; this.tracer = tracer; this.roleContextDrainState = roleContextDrainState; this.recordsProcessor = recordsProcessor; this.fabricReplicator = fabricReplicator; this.backupManager = backupManager; this.transactionManager = transactionManager; this.checkpointManager = checkpointManager; this.replicatedLogManager = replicatedLogManager; }
/// <summary> /// Initializes resources necessary for the replicator and for all the state providers. /// </summary> internal async Task OpenAsync( ReplicaOpenMode openMode, TransactionalReplicatorSettings transactionalReplicatorSettings) { this.metricManager.StartTask(); await this.stateManager.OpenAsync(openMode, transactionalReplicatorSettings).ConfigureAwait(false); }
/// <summary> /// Initializes an instance of the ReplicatorApiMonitor class /// ReplicatorApiMonitor takes ServiceContext, StatefulServicePartition and Replicator setttings as arguments. /// The default value of SlowApiMonitoringDuration in the replicator settings object is 120 seconds /// </summary> /// <param name="serviceContext">Current service serviceContext</param> /// <param name="statefulPartition">Current statefulPartition</param> /// <param name="settings">Current replicator settings</param> public void Initialize(ServiceContext serviceContext, IStatefulServicePartition statefulPartition, TransactionalReplicatorSettings settings) { this.replicatorSettings = settings; this.context = serviceContext; this.partition = statefulPartition; this.traceType = string.Format("{0}.{1}.{2}", SR.ReplicatorApiMonitor_TraceTypeBase, this.context.PartitionId, this.context.ReplicaOrInstanceId); var monitorParameters = new MonitoringComponentParameters(this.replicatorSettings.PublicSettings.SlowApiMonitoringDuration.Value); this.fabricMonitor = new FabricApiMonitoringComponent(monitorParameters); this.initialized = true; }
public async Task <RecoveryInformation> OpenAsync( ReplicaOpenMode openMode, TransactionalReplicatorSettings replicatorSettings, ITracer tracer, LogManager logManager, bool shouldLocalStateBeRemoved, PhysicalLogReader recoveryReader, bool isRestoring) { this.RecoveryLogsReader = recoveryReader; this.tracer = tracer; PhysicalLogRecord recoveredLinkedPhysicalRecord; TruncateHeadLogRecord recoveredLastTruncateHeadLogRecord = null; CompleteCheckpointLogRecord recoveredCompleteCheckpointRecord = null; this.logCompleteCheckpointAfterRecovery = true; this.tailRecordAtStart = await this.RecoveryLogsReader.SeekToLastRecord().ConfigureAwait(false); var logUsage = logManager.Length; var trace = "OpenAsync: Log Usage: " + logUsage + " Recovered Replica Tail Record Type: " + tailRecordAtStart.RecordType + " LSN: " + tailRecordAtStart.Lsn.LSN + " PSN: " + tailRecordAtStart.Psn.PSN + " Position: " + tailRecordAtStart.RecordPosition; FabricEvents.Events.Api(tracer.Type, trace); var lastPhysicalRecord = tailRecordAtStart as PhysicalLogRecord; if (lastPhysicalRecord == null) { lastPhysicalRecord = await this.RecoveryLogsReader.GetPreviousPhysicalRecord(tailRecordAtStart).ConfigureAwait(false); } trace = "OpenAsync: Recovered last physical record Type: " + lastPhysicalRecord.RecordType + " LSN: " + lastPhysicalRecord.Lsn.LSN + " PSN: " + lastPhysicalRecord.Psn.PSN + " Position: " + lastPhysicalRecord.RecordPosition; FabricEvents.Events.Api(tracer.Type, trace); if (lastPhysicalRecord.RecordType == LogRecordType.Information) { var removingStateRecord = lastPhysicalRecord as InformationLogRecord; if (removingStateRecord.InformationEvent == InformationEvent.RemovingState) { Utility.Assert( lastPhysicalRecord == tailRecordAtStart, "Last Physical Record {0} should be same as tail record at start {1}", lastPhysicalRecord, tailRecordAtStart); FabricEvents.Events.Api(tracer.Type, "OpenAsync: Skipping Recovery due to pending RemovingState"); this.IsRemovingStateAfterOpen = true; return(new RecoveryInformation(shouldLocalStateBeRemoved)); } } if (lastPhysicalRecord.RecordType == LogRecordType.TruncateHead) { recoveredLastTruncateHeadLogRecord = lastPhysicalRecord as TruncateHeadLogRecord; } else if (lastPhysicalRecord.RecordType == LogRecordType.EndCheckpoint) { this.RecoveredLastEndCheckpointRecord = lastPhysicalRecord as EndCheckpointLogRecord; } else if (lastPhysicalRecord.RecordType == LogRecordType.CompleteCheckpoint) { recoveredCompleteCheckpointRecord = lastPhysicalRecord as CompleteCheckpointLogRecord; } else { recoveredLinkedPhysicalRecord = await this.RecoveryLogsReader.GetLinkedPhysicalRecord(lastPhysicalRecord).ConfigureAwait(false); Utility.Assert( (recoveredLinkedPhysicalRecord != null) && ((recoveredLinkedPhysicalRecord.RecordType == LogRecordType.TruncateHead) || (recoveredLinkedPhysicalRecord.RecordType == LogRecordType.EndCheckpoint) || (recoveredLinkedPhysicalRecord.RecordType == LogRecordType.CompleteCheckpoint)), "Record type should be truncate head or end checkpoint or complete checkpoint. Record type is : {0}", recoveredLinkedPhysicalRecord.RecordType); trace = "OpenAsync: RecoveredLinkedPhysicalRecord: " + recoveredLinkedPhysicalRecord.RecordType + " LSN: " + recoveredLinkedPhysicalRecord.Lsn.LSN + " PSN: " + recoveredLinkedPhysicalRecord.Psn.PSN + " Position: " + recoveredLinkedPhysicalRecord.RecordPosition; FabricEvents.Events.Api(tracer.Type, trace); if (recoveredLinkedPhysicalRecord.RecordType == LogRecordType.TruncateHead) { recoveredLastTruncateHeadLogRecord = recoveredLinkedPhysicalRecord as TruncateHeadLogRecord; } else if (recoveredLinkedPhysicalRecord.RecordType == LogRecordType.EndCheckpoint) { this.RecoveredLastEndCheckpointRecord = recoveredLinkedPhysicalRecord as EndCheckpointLogRecord; } else { recoveredCompleteCheckpointRecord = recoveredLinkedPhysicalRecord as CompleteCheckpointLogRecord; } } ulong logHeadPosition = 0; long logHeadLsn = 0; if (recoveredLastTruncateHeadLogRecord != null) { logHeadPosition = recoveredLastTruncateHeadLogRecord.LogHeadRecordPosition; logHeadLsn = recoveredLastTruncateHeadLogRecord.LogHeadLsn.LSN; trace = "OpenAsync: Recovered last truncate head record Type: " + recoveredLastTruncateHeadLogRecord.RecordType + " LSN: " + recoveredLastTruncateHeadLogRecord.Lsn.LSN + " PSN: " + recoveredLastTruncateHeadLogRecord.Psn.PSN + " Position: " + recoveredLastTruncateHeadLogRecord.RecordPosition + " LogHeadLSN: " + recoveredLastTruncateHeadLogRecord.LogHeadLsn.LSN + " LogHeadPosition: " + recoveredLastTruncateHeadLogRecord.LogHeadRecordPosition; FabricEvents.Events.Api(tracer.Type, trace); recoveredLinkedPhysicalRecord = recoveredLastTruncateHeadLogRecord; do { recoveredLinkedPhysicalRecord = await this.RecoveryLogsReader.GetLinkedPhysicalRecord(recoveredLinkedPhysicalRecord).ConfigureAwait(false); Utility.Assert( (recoveredLinkedPhysicalRecord != null) && ((recoveredLinkedPhysicalRecord.RecordType == LogRecordType.TruncateHead) || (recoveredLinkedPhysicalRecord.RecordType == LogRecordType.EndCheckpoint) || (recoveredLinkedPhysicalRecord.RecordType == LogRecordType.CompleteCheckpoint)), "Record type should be truncate head or end checkpoint or complete checkpoint. Record type is : {0}", recoveredLinkedPhysicalRecord.RecordType); this.RecoveredLastEndCheckpointRecord = recoveredLinkedPhysicalRecord as EndCheckpointLogRecord; if (recoveredLinkedPhysicalRecord.RecordType == LogRecordType.CompleteCheckpoint) { recoveredCompleteCheckpointRecord = recoveredLinkedPhysicalRecord as CompleteCheckpointLogRecord; } } while (this.RecoveredLastEndCheckpointRecord == null); } else if (recoveredCompleteCheckpointRecord != null) { logHeadPosition = recoveredCompleteCheckpointRecord.LogHeadRecordPosition; logHeadLsn = recoveredCompleteCheckpointRecord.LogHeadLsn.LSN; trace = "OpenAsync: Recovered last complete checkpoint record Type: " + recoveredCompleteCheckpointRecord.RecordType + " LSN: " + recoveredCompleteCheckpointRecord.Lsn.LSN + " PSN: " + recoveredCompleteCheckpointRecord.Psn.PSN + " Position: " + recoveredCompleteCheckpointRecord.RecordPosition + " LogHeadLsn: " + recoveredCompleteCheckpointRecord.LogHeadLsn.LSN + " LogHeadPosition: " + recoveredCompleteCheckpointRecord.LogHeadRecordPosition; FabricEvents.Events.Api(tracer.Type, trace); recoveredLinkedPhysicalRecord = recoveredCompleteCheckpointRecord; do { recoveredLinkedPhysicalRecord = await this.RecoveryLogsReader.GetLinkedPhysicalRecord(recoveredLinkedPhysicalRecord).ConfigureAwait(false); Utility.Assert( (recoveredLinkedPhysicalRecord != null) && ((recoveredLinkedPhysicalRecord.RecordType == LogRecordType.TruncateHead) || (recoveredLinkedPhysicalRecord.RecordType == LogRecordType.EndCheckpoint) || (recoveredLinkedPhysicalRecord.RecordType == LogRecordType.CompleteCheckpoint)), "Record type should be truncate head or end checkpoint or complete checkpoint. Record type is : {0}", recoveredLinkedPhysicalRecord.RecordType); this.RecoveredLastEndCheckpointRecord = recoveredLinkedPhysicalRecord as EndCheckpointLogRecord; } while (this.RecoveredLastEndCheckpointRecord == null); } else { logHeadPosition = this.RecoveredLastEndCheckpointRecord.LogHeadRecordPosition; logHeadLsn = this.RecoveredLastEndCheckpointRecord.LogHeadLsn.LSN; } trace = "OpenAsync: Recovered last end checkpoint record Type: " + this.RecoveredLastEndCheckpointRecord.RecordType + " LSN: " + this.RecoveredLastEndCheckpointRecord.Lsn.LSN + " PSN: " + this.RecoveredLastEndCheckpointRecord.Psn.PSN + " Position: " + this.RecoveredLastEndCheckpointRecord.RecordPosition + " LogHeadLSN: " + this.RecoveredLastEndCheckpointRecord.LogHeadLsn.LSN + " LogHeadPosition: " + this.RecoveredLastEndCheckpointRecord.LogHeadRecordPosition; FabricEvents.Events.Api(tracer.Type, trace); if (recoveredCompleteCheckpointRecord != null) { this.logCompleteCheckpointAfterRecovery = false; // This is fine since both the begin and end records guarantee all the SP's have been renamed trace = "OpenAsync: " + " LogHeadPosition: " + this.RecoveredLastEndCheckpointRecord.LogHeadRecordPosition + Environment.NewLine + " LogHeadLSN: " + this.RecoveredLastEndCheckpointRecord.LogHeadLsn.LSN + Environment.NewLine + " CompleteCheckpoint record found with LSN: " + recoveredCompleteCheckpointRecord.Lsn.LSN + " PSN: " + recoveredCompleteCheckpointRecord.Psn.PSN + " Position: " + recoveredCompleteCheckpointRecord.RecordPosition; FabricEvents.Events.Api(tracer.Type, trace); } else { this.logCompleteCheckpointAfterRecovery = true; trace = "OpenAsync: " + " LogHeadPosition: " + this.RecoveredLastEndCheckpointRecord.LogHeadRecordPosition + Environment.NewLine + " LogHeadLSN: " + this.RecoveredLastEndCheckpointRecord.LogHeadLsn.LSN + Environment.NewLine + " CompleteCheckpoint record missing"; FabricEvents.Events.Api(tracer.Type, trace); } // For restore cases, logHeadPosition must be 0. if (isRestoring == true) { Utility.Assert( logHeadPosition == 0, "Full backup: LogHead Position ({0}) must be 0", logHeadPosition); } this.RecoveryLogsReader.MoveStartingRecordPosition( logHeadLsn, logHeadPosition, "recovery", LogManager.LogReaderType.Recovery); logManager.SetLogHeadRecordPosition(logHeadPosition); return(new RecoveryInformation( this.logCompleteCheckpointAfterRecovery, shouldLocalStateBeRemoved)); }
internal override async Task InitializeAsync(ITracer tracer, LogManagerInitialization init, TransactionalReplicatorSettings settings) { this.maxStreamSize = settings.PublicSettings.CheckpointThresholdInMB.Value * LogTruncationManager.ThrottleAfterPendingCheckpointCount; await base.InitializeAsync(tracer, init, settings).ConfigureAwait(false); }
internal override async Task InitializeAsync(ITracer inputTracer, LogManagerInitialization init, TransactionalReplicatorSettings settings) { // // Determine if we are in a container and if so then get the host namespace for the application // // When running in a container, the container will have a different file system namespace as the // host. The filenames are generated from within the container, however the KTLLogger driver will // create them while running in the host namespace. In order for the files to be created in the // correct location, the filename needs to be mapped from the container namespace to the host // namespace. This mapping is done here. // // Fabric provides the mapping into the host namespace in the environment variable named // "Fabric_HostApplicationDirectory" // string FabricHostApplicationDirectory; try { FabricHostApplicationDirectory = Environment.GetEnvironmentVariable(Constants.Fabric_HostApplicationDirectory); } catch (Exception) { FabricHostApplicationDirectory = null; } string containerDedicatedLogPath = null; if (FabricHostApplicationDirectory != null) { var message = string.Format(CultureInfo.InvariantCulture, "Mapped dedicated log path for container: From: {0} To: {1}", init.dedicatedLogPath, FabricHostApplicationDirectory); FabricEvents.Events.LogManager(inputTracer.Type, message); containerDedicatedLogPath = FabricHostApplicationDirectory; } // // Make a whole bunch of decisions about which type of log manager and logs to use. At the end // this.useStagingLog and the correct path for dedicatedLogPath will be setup for this replica. // string sharedLogPath = settings.PublicSettings.SharedLogPath; Guid sharedLogId = Guid.Parse(settings.PublicSettings.SharedLogId); bool isDefaultSettings = (settings.PublicSettings.SharedLogId == Constants.DefaultSharedLogContainerGuid); bool isInContainer = (containerDedicatedLogPath != null); if (this.physicalLogManager == null) { this.physicalLogManager = await System.Fabric.Data.Log.LogManager.OpenAsync(globalLogManagerType, CancellationToken.None).ConfigureAwait(false); } System.Fabric.Data.Log.LogManager.LoggerType logManagerType = this.physicalLogManager.GetLoggerType(); if (isDefaultSettings) { if (logManagerType == System.Fabric.Data.Log.LogManager.LoggerType.Inproc) { // // When running inproc, we must use the staging log // Format: <workDir>/<partitionId>_<replicaId>.stlog // this.useStagingLog = true; sharedLogId = Guid.NewGuid(); sharedLogPath = Path.Combine(init.workDirectory, init.partitionId.ToString() + "_" + init.replicaId.ToString() + ".stlog"); } else if (logManagerType == System.Fabric.Data.Log.LogManager.LoggerType.Driver) { // // When running with the driver then we do not use staging log // this.useStagingLog = false; if (isInContainer) { // // If running with the driver then the dedicated log path needs to be mapped // into the host namespace. // init.dedicatedLogPath = containerDedicatedLogPath; } } else { // // Only Inproc and Driver are supported, bail out // FabricEvents.Events.LogManager( inputTracer.Type, "KtlLogManager.InitializeAsync: Unknown LogManagerType " + (int)logManagerType + " " + logManagerType.ToString()); throw new InvalidOperationException(); } } else { // // If we are not using default shared log settings then the application must know what they are doing // and so just go with what they want and hope for the best. // this.useStagingLog = false; if (isInContainer) { FabricEvents.Events.LogManager( inputTracer.Type, "KtlLogManager.InitializeAsync: Default shared log is not be used within a container." + " LogManagerType: " + logManagerType.ToString() + " Shared Log: " + settings.PublicSettings.SharedLogPath + " " + settings.PublicSettings.SharedLogId.ToString()); } } FabricEvents.Events.LogManager( inputTracer.Type, "KtlLogManager.InitializeAsync: " + " globalLogManagerType: " + globalLogManagerType.ToString() + " LogManagerType: " + logManagerType.ToString() + " isDefaultSettings: " + isDefaultSettings.ToString() + " isInContainer: " + isInContainer.ToString() + " partitionId: " + init.partitionId.ToString() + " replicaId: " + init.replicaId.ToString() + " Shared Log: " + sharedLogPath + " " + sharedLogId.ToString()); await base.InitializeAsync(inputTracer, init, settings).ConfigureAwait(false); this.physicalLogName = sharedLogPath; this.physicalLogId = sharedLogId; this.maximumMetadataSizeInKB = (int)settings.PublicSettings.MaxMetadataSizeInKB; this.maximumRecordSizeInKb = settings.PublicSettings.MaxRecordSizeInKB.Value; this.maximumStreamSizeInMb = (int)settings.PublicSettings.MaxStreamSizeInMB; this.optimizeForLocalSsd = settings.PublicSettings.OptimizeForLocalSSD.Value; this.optimizeForLowerDiskUsage = settings.PublicSettings.OptimizeLogForLowerDiskUsage.Value; this.maxWriteQueueDepthInKb = (int)settings.PublicSettings.MaxWriteQueueDepthInKB; await this.EnsureSharedLogContainerAsync(this.physicalLogName, this.physicalLogId).ConfigureAwait(false); }
/// <summary> /// OpenAsync is called when the replica is going to be actually used /// </summary> /// <param name="openMode">Open mode.</param> /// <param name="partitionObject">Service partition</param> /// <param name="cancellationToken">Cancellation token.</param> /// <returns>Task that represents the asynchronous operation.</returns> async Task <IReplicator> IStatefulServiceReplica.OpenAsync( ReplicaOpenMode openMode, IStatefulServicePartition partitionObject, CancellationToken cancellationToken) { FabricEvents.Events.Lifecycle( this.tracer.Type, "OpenAsync" + " openMode: " + openMode + " replica: " + this.initializationParameters.ReplicaId); TaskScheduler.UnobservedTaskException += this.ProcessUnobservedTaskException; // Store the partitionObject this.partition = partitionObject; var statefulServiceContext = new StatefulServiceContext( FabricRuntime.GetNodeContext(), this.initializationParameters.CodePackageActivationContext, this.initializationParameters.ServiceTypeName, this.initializationParameters.ServiceName, this.initializationParameters.InitializationData, this.initializationParameters.PartitionId, this.initializationParameters.ReplicaId); this.transactionalReplicator.Initialize(statefulServiceContext, partitionObject); FabricEvents.Events.Lifecycle(this.tracer.Type, "OpenAsync: StateManager initialized"); // create the replicator // The Windows Fabric Replicator is used to actually replicate the data // The ReplicatorSettings are used for configuring the replicator - here we ask for them from the derived class // When using service groups Replicator Settings are described in the Settings.xml inside the configuration package. // So when the service is part of a service group it should use null as its Replicator Setting. FabricReplicator replicator; ReliableStateManagerReplicatorSettings replicatorSettings = null; var onOpenInvoked = false; try { if (this.Partition is IServiceGroupPartition) { replicatorSettings = new ReliableStateManagerReplicatorSettings(); ReliableStateManagerReplicatorSettingsUtil.LoadDefaultsIfNotSet(ref replicatorSettings); replicator = this.partition.CreateReplicator( this.transactionalReplicator, ReliableStateManagerReplicatorSettingsUtil.ToReplicatorSettings(replicatorSettings)); } else { replicatorSettings = await this.OnOpenAsync(cancellationToken).ConfigureAwait(false); onOpenInvoked = true; if (replicatorSettings == null) { replicatorSettings = new ReliableStateManagerReplicatorSettings(); } ReliableStateManagerReplicatorSettingsUtil.LoadDefaultsIfNotSet(ref replicatorSettings); replicator = this.partition.CreateReplicator( this.transactionalReplicator, ReliableStateManagerReplicatorSettingsUtil.ToReplicatorSettings(replicatorSettings)); } ServiceReplicaUtils.SetupLoggerPath(ref replicatorSettings, ref this.initializationParameters); var transactionalReplicatorSettings = new TransactionalReplicatorSettings { PublicSettings = replicatorSettings }; ReliableStateManagerReplicatorSettingsUtil.LoadInternalSettingsDefault(ref transactionalReplicatorSettings); this.stateReplicator = replicator.StateReplicator; this.transactionalReplicator.LoggingReplicator.FabricReplicator = this.stateReplicator; this.transactionalReplicator.LoggingReplicator.Tracer = this.tracer; // Starting local recovery await this.transactionalReplicator.OpenAsync(openMode, transactionalReplicatorSettings).ConfigureAwait(false); // Change state this.replicaState = ReplicaState.Opened; FabricEvents.Events.Lifecycle( this.tracer.Type, "OpenAsync: Finished opening replica " + this.initializationParameters.ReplicaId + " ReplicatorAddress: " + replicatorSettings.ReplicatorAddress + " ReplicatorListenAddress: " + replicatorSettings.ReplicatorListenAddress + " ReplicatorPublishAddress: " + replicatorSettings.ReplicatorPublishAddress); return(replicator); } catch (Exception e) { int innerHResult = 0; var flattenedException = Utility.FlattenException(e, out innerHResult); FabricEvents.Events.Exception_TStatefulServiceReplica( this.tracer.Type, "OpenAsync", flattenedException.GetType().ToString(), flattenedException.Message, flattenedException.HResult != 0 ? flattenedException.HResult : innerHResult, flattenedException.StackTrace); if (onOpenInvoked) { this.OnAbort(); } TaskScheduler.UnobservedTaskException -= this.ProcessUnobservedTaskException; throw; } }
internal override async Task InitializeAsync(ITracer paramTracer, LogManagerInitialization init, TransactionalReplicatorSettings settings) { await base.InitializeAsync(paramTracer, init, settings).ConfigureAwait(false); }
internal virtual Task InitializeAsync(ITracer tracer, LogManagerInitialization init, TransactionalReplicatorSettings settings) { this.Tracer = tracer; this.LogFileDirectoryPath = init.dedicatedLogPath; this.MaxWriteCacheSizeInMB = DefaultWriteCacheSizeMb; this.emptyCallbackManager = new PhysicalLogWriterCallbackManager(this.EmptyFlushedRecordsCallback, tracer); return(Task.FromResult(0)); }