コード例 #1
0
        internal async Task InitializeAsync(FabricClient fabricClient, ChaosMessageProcessor chaosMessageProcessor, CancellationToken cancellationToken)
        {
            TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "InitializeAsync - started");

            this.fabricClient          = fabricClient;
            this.chaosMessageProcessor = chaosMessageProcessor;

            await this.RestartRecoveryAsync(cancellationToken).ConfigureAwait(false);

#pragma warning disable 4014
            Task.Run(() => this.StartAndRunAsync(cancellationToken)).ContinueWith(task =>
            {
                if (task.IsCanceled)
                {
                    // This will print after the thread running the scheduler ends.
                    TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "Scheduler task cancelled.");
                }
            });
#pragma warning restore 4014

            TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "InitializeAsync - ended");
        }
コード例 #2
0
        private async Task InitializeAsync(CancellationToken cancellationToken)
        {
            System.Fabric.Common.NativeConfigStore configStore = System.Fabric.Common.NativeConfigStore.FabricGetConfigStore();
            string isTestModeEnabled = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, "TestModeEnabled");

            if (isTestModeEnabled.ToLowerInvariant() == "true")
            {
                TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Enabling internal test mode");
                this.IsTestMode = true;
            }

            string apiTestModeString = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, "ApiTestMode");
            int    apiTestMode;

            if (!int.TryParse(apiTestModeString, out apiTestMode))
            {
                apiTestMode = 0;
            }

            string requestTimeout = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.RequestTimeoutInSecondsName);
            int    requestTimeoutInSecondsAsInt = string.IsNullOrEmpty(requestTimeout) ? FASConstants.DefaultRequestTimeoutInSeconds : int.Parse(requestTimeout);

            string operationTimeout = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.OperationTimeoutInSecondsName);
            int    operationTimeoutInSecondsAsInt = string.IsNullOrEmpty(operationTimeout) ? FASConstants.DefaultOperationTimeoutInSeconds : int.Parse(operationTimeout);

            TestabilityTrace.TraceSource.WriteInfo(
                FaultAnalysisServiceTraceType,
                "Setting requestTimeout='{0}' and operationTimeout='{1}'",
                TimeSpan.FromSeconds(requestTimeoutInSecondsAsInt),
                TimeSpan.FromSeconds(operationTimeoutInSecondsAsInt));

            string maxStoredActionCount       = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.MaxStoredActionCountName);
            long   maxStoredActionCountAsLong = string.IsNullOrEmpty(maxStoredActionCount) ? FASConstants.DefaultMaxStoredActionCount : long.Parse(maxStoredActionCount);

            string storedActionCleanupIntervalInSeconds      = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.StoredActionCleanupIntervalInSecondsName);
            int    storedActionCleanupIntervalInSecondsAsInt = string.IsNullOrEmpty(storedActionCleanupIntervalInSeconds) ? FASConstants.DefaultStoredActionCleanupIntervalInSeconds : int.Parse(storedActionCleanupIntervalInSeconds);

            string completedActionKeepDurationInSeconds      = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.CompletedActionKeepDurationInSecondsName);
            int    completedActionKeepDurationInSecondsAsInt = string.IsNullOrEmpty(completedActionKeepDurationInSeconds) ? FASConstants.DefaultCompletedActionKeepDurationInSeconds : int.Parse(completedActionKeepDurationInSeconds);

            TestabilityTrace.TraceSource.WriteInfo(
                FaultAnalysisServiceTraceType,
                "MaxStoredActionCount={0}, StoredActionCleanupIntervalInSeconds={1}, CompletedActionKeepDurationInSecondsName={2}",
                maxStoredActionCountAsLong,
                storedActionCleanupIntervalInSecondsAsInt,
                completedActionKeepDurationInSecondsAsInt);

            string commandStepRetryBackoffInSeconds      = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.CommandStepRetryBackoffInSecondsName);
            int    commandStepRetryBackoffInSecondsAsInt = string.IsNullOrEmpty(commandStepRetryBackoffInSeconds) ? FASConstants.DefaultCommandStepRetryBackoffInSeconds : int.Parse(commandStepRetryBackoffInSeconds);

            TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Read from config commandStepRetryBackoffInSeconds='{0}'", commandStepRetryBackoffInSecondsAsInt);

            string concurrentRequestsAsString = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.ConcurrentRequestsName);
            int    concurrentRequests         = string.IsNullOrEmpty(concurrentRequestsAsString) ? FASConstants.DefaultConcurrentRequests : int.Parse(concurrentRequestsAsString);

            TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Read from config concurrentRequests='{0}'", concurrentRequests);

            string dataLossCheckWaitDurationInSecondsAsString = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.DataLossCheckWaitDurationInSecondsName);
            int    dataLossCheckWaitDurationInSeconds         = string.IsNullOrEmpty(dataLossCheckWaitDurationInSecondsAsString) ? FASConstants.DefaultDataLossCheckWaitDurationInSeconds : int.Parse(dataLossCheckWaitDurationInSecondsAsString);

            TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Read from config dataLossCheckWaitDurationInSeconds='{0}'", dataLossCheckWaitDurationInSeconds);

            string dataLossCheckPollIntervalInSecondsAsString = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.DataLossCheckPollIntervalInSecondsName);
            int    dataLossCheckPollIntervalInSeconds         = string.IsNullOrEmpty(dataLossCheckPollIntervalInSecondsAsString) ? FASConstants.DefaultDataLossCheckPollIntervalInSeconds : int.Parse(dataLossCheckPollIntervalInSecondsAsString);

            TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Read from config dataLossCheckPollIntervalInSeconds='{0}'", dataLossCheckPollIntervalInSeconds);

            string replicaDropWaitDurationInSecondsAsString = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.ReplicaDropWaitDurationInSecondsName);
            int    replicaDropWaitDurationInSeconds         = string.IsNullOrEmpty(replicaDropWaitDurationInSecondsAsString) ? FASConstants.DefaultReplicaDropWaitDurationInSeconds : int.Parse(replicaDropWaitDurationInSecondsAsString);

            TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Read from config replicaDropWaitDurationInSeconds='{0}'", replicaDropWaitDurationInSeconds);

            string chaosTelemetrySamplingProbabilityAsString = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.ChaosTelemetrySamplingProbabilityConfigKeyName);
            double chaosTelemetrySamplingProbability         = string.IsNullOrEmpty(chaosTelemetrySamplingProbabilityAsString) ? FASConstants.DefaultChaosTelemetrySamplingProbability : double.Parse(chaosTelemetrySamplingProbabilityAsString);

            TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Read from config chaosTelemetrySamplingProbability='{0}'", chaosTelemetrySamplingProbability);

            string chaosTelemetryReportPeriodInSecondsAsString = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.ChaostelemetryReportPeriodInSecondsConfigKeyName);
            int    chaostelemetryReportPeriodInSeconds         = string.IsNullOrEmpty(chaosTelemetryReportPeriodInSecondsAsString) ? FASConstants.DefaultTelemetryReportPeriodInSeconds : int.Parse(chaosTelemetryReportPeriodInSecondsAsString);

            TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Read from config chaostelemetryReportPeriodInSeconds='{0}'", chaostelemetryReportPeriodInSeconds);

            TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Creating action tables");
            var actionTable = await this.StateManager.GetOrAddAsync <IReliableDictionary <Guid, byte[]> >(FASConstants.ActionReliableDictionaryName).ConfigureAwait(false);

            var historyTable = await this.StateManager.GetOrAddAsync <IReliableDictionary <Guid, byte[]> >(FASConstants.HistoryReliableDictionaryName).ConfigureAwait(false);

            var stoppedNodeTable = await this.StateManager.GetOrAddAsync <IReliableDictionary <string, bool> >(FASConstants.StoppedNodeTable);

            TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Creating ActionStore");
            this.actionStore = new ActionStore(
                this.StateManager,
                this.Partition,
                actionTable,
                historyTable,
                stoppedNodeTable,
                maxStoredActionCountAsLong,
                storedActionCleanupIntervalInSecondsAsInt,
                completedActionKeepDurationInSecondsAsInt,
                this.IsTestMode,
                cancellationToken);

            FabricClient fabricClient = new FabricClient();

            this.engine           = new ReliableFaultsEngine(this.actionStore, this.IsTestMode, this.Partition, commandStepRetryBackoffInSecondsAsInt);
            this.messageProcessor = new FaultAnalysisServiceMessageProcessor(
                this.Partition,
                fabricClient,
                this.engine,
                this.actionStore,
                this.StateManager,
                stoppedNodeTable,
                TimeSpan.FromSeconds(requestTimeoutInSecondsAsInt),
                TimeSpan.FromSeconds(operationTimeoutInSecondsAsInt),
                concurrentRequests,
                dataLossCheckWaitDurationInSeconds,
                dataLossCheckPollIntervalInSeconds,
                replicaDropWaitDurationInSeconds,
                cancellationToken);

            string maxStoredChaosEventCleanupIntervalInSeconds      = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.StoredChaosEventCleanupIntervalInSecondsConfigName);
            int    maxStoredChaosEventCleanupIntervalInSecondsAsInt = string.IsNullOrEmpty(maxStoredChaosEventCleanupIntervalInSeconds) ? FASConstants.StoredChaosEventCleanupIntervalInSecondsDefault : int.Parse(maxStoredChaosEventCleanupIntervalInSeconds);

            string maxStoredChaosEventCount      = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.MaxStoredChaosEventCountConfigName);
            int    maxStoredChaosEventCountAsInt = string.IsNullOrEmpty(maxStoredChaosEventCount) ? FASConstants.MaxStoredChaosEventCountDefault : int.Parse(maxStoredChaosEventCount);

            this.chaosMessageProcessor = new ChaosMessageProcessor(
                this.StateManager,
                apiTestMode,
                maxStoredChaosEventCleanupIntervalInSecondsAsInt,
                maxStoredChaosEventCountAsInt,
                chaosTelemetrySamplingProbability,
                chaostelemetryReportPeriodInSeconds,
                this.Partition,
                fabricClient,
                this.scheduler,
                cancellationToken);

            await this.messageProcessor.ResumePendingActionsAsync(fabricClient, cancellationToken).ConfigureAwait(false);

            this.faultAnalysisServiceImpl.MessageProcessor = this.messageProcessor;

            this.faultAnalysisServiceImpl.ChaosMessageProcessor = this.chaosMessageProcessor;
            this.faultAnalysisServiceImpl.LoadActionMapping();

            await this.scheduler.InitializeAsync(fabricClient, this.chaosMessageProcessor, cancellationToken).ConfigureAwait(false);

            TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Done initializing engine");

            await this.InitClusterAnalysisEngineAsync(fabricClient, configStore, cancellationToken).ConfigureAwait(false);
        }