public static async Task <ActionTest> CreateAsync(FaultAnalysisServiceMessageProcessor messageProcessor, ActionStore actionStore, IReliableStateManager stateManager) { ThrowIf.Null(messageProcessor, "messageProcessor"); ThrowIf.Null(actionStore, "actionStore"); ThrowIf.Null(stateManager, "stateManager"); IReliableDictionary <string, string> internalTestsStarted = await stateManager.GetOrAddAsync <IReliableDictionary <string, string> >("internalTests"); return(new ActionTest(messageProcessor, actionStore, stateManager, internalTestsStarted)); }
private ActionTest(FaultAnalysisServiceMessageProcessor messageProcessor, ActionStore actionStore, IReliableStateManager stateManager, IReliableDictionary <string, string> internalTestsStarted) { ThrowIf.Null(internalTestsStarted, "internalTestsStarted"); this.actionStore = actionStore; this.stateManager = stateManager; this.internalTestsStarted = internalTestsStarted; this.mockClient = new MockClient(messageProcessor); }
public MockClient(FaultAnalysisServiceMessageProcessor messageProcessor) { ThrowIf.Null(messageProcessor, "messageProcessor"); this.messageProcessor = messageProcessor; }
private async Task InitializeAsync(CancellationToken cancellationToken) { System.Fabric.Common.NativeConfigStore configStore = System.Fabric.Common.NativeConfigStore.FabricGetConfigStore(); string isTestModeEnabled = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, "TestModeEnabled"); if (isTestModeEnabled.ToLowerInvariant() == "true") { TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Enabling internal test mode"); this.IsTestMode = true; } string apiTestModeString = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, "ApiTestMode"); int apiTestMode; if (!int.TryParse(apiTestModeString, out apiTestMode)) { apiTestMode = 0; } string requestTimeout = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.RequestTimeoutInSecondsName); int requestTimeoutInSecondsAsInt = string.IsNullOrEmpty(requestTimeout) ? FASConstants.DefaultRequestTimeoutInSeconds : int.Parse(requestTimeout); string operationTimeout = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.OperationTimeoutInSecondsName); int operationTimeoutInSecondsAsInt = string.IsNullOrEmpty(operationTimeout) ? FASConstants.DefaultOperationTimeoutInSeconds : int.Parse(operationTimeout); TestabilityTrace.TraceSource.WriteInfo( FaultAnalysisServiceTraceType, "Setting requestTimeout='{0}' and operationTimeout='{1}'", TimeSpan.FromSeconds(requestTimeoutInSecondsAsInt), TimeSpan.FromSeconds(operationTimeoutInSecondsAsInt)); string maxStoredActionCount = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.MaxStoredActionCountName); long maxStoredActionCountAsLong = string.IsNullOrEmpty(maxStoredActionCount) ? FASConstants.DefaultMaxStoredActionCount : long.Parse(maxStoredActionCount); string storedActionCleanupIntervalInSeconds = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.StoredActionCleanupIntervalInSecondsName); int storedActionCleanupIntervalInSecondsAsInt = string.IsNullOrEmpty(storedActionCleanupIntervalInSeconds) ? FASConstants.DefaultStoredActionCleanupIntervalInSeconds : int.Parse(storedActionCleanupIntervalInSeconds); string completedActionKeepDurationInSeconds = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.CompletedActionKeepDurationInSecondsName); int completedActionKeepDurationInSecondsAsInt = string.IsNullOrEmpty(completedActionKeepDurationInSeconds) ? FASConstants.DefaultCompletedActionKeepDurationInSeconds : int.Parse(completedActionKeepDurationInSeconds); TestabilityTrace.TraceSource.WriteInfo( FaultAnalysisServiceTraceType, "MaxStoredActionCount={0}, StoredActionCleanupIntervalInSeconds={1}, CompletedActionKeepDurationInSecondsName={2}", maxStoredActionCountAsLong, storedActionCleanupIntervalInSecondsAsInt, completedActionKeepDurationInSecondsAsInt); string commandStepRetryBackoffInSeconds = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.CommandStepRetryBackoffInSecondsName); int commandStepRetryBackoffInSecondsAsInt = string.IsNullOrEmpty(commandStepRetryBackoffInSeconds) ? FASConstants.DefaultCommandStepRetryBackoffInSeconds : int.Parse(commandStepRetryBackoffInSeconds); TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Read from config commandStepRetryBackoffInSeconds='{0}'", commandStepRetryBackoffInSecondsAsInt); string concurrentRequestsAsString = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.ConcurrentRequestsName); int concurrentRequests = string.IsNullOrEmpty(concurrentRequestsAsString) ? FASConstants.DefaultConcurrentRequests : int.Parse(concurrentRequestsAsString); TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Read from config concurrentRequests='{0}'", concurrentRequests); string dataLossCheckWaitDurationInSecondsAsString = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.DataLossCheckWaitDurationInSecondsName); int dataLossCheckWaitDurationInSeconds = string.IsNullOrEmpty(dataLossCheckWaitDurationInSecondsAsString) ? FASConstants.DefaultDataLossCheckWaitDurationInSeconds : int.Parse(dataLossCheckWaitDurationInSecondsAsString); TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Read from config dataLossCheckWaitDurationInSeconds='{0}'", dataLossCheckWaitDurationInSeconds); string dataLossCheckPollIntervalInSecondsAsString = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.DataLossCheckPollIntervalInSecondsName); int dataLossCheckPollIntervalInSeconds = string.IsNullOrEmpty(dataLossCheckPollIntervalInSecondsAsString) ? FASConstants.DefaultDataLossCheckPollIntervalInSeconds : int.Parse(dataLossCheckPollIntervalInSecondsAsString); TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Read from config dataLossCheckPollIntervalInSeconds='{0}'", dataLossCheckPollIntervalInSeconds); string replicaDropWaitDurationInSecondsAsString = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.ReplicaDropWaitDurationInSecondsName); int replicaDropWaitDurationInSeconds = string.IsNullOrEmpty(replicaDropWaitDurationInSecondsAsString) ? FASConstants.DefaultReplicaDropWaitDurationInSeconds : int.Parse(replicaDropWaitDurationInSecondsAsString); TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Read from config replicaDropWaitDurationInSeconds='{0}'", replicaDropWaitDurationInSeconds); string chaosTelemetrySamplingProbabilityAsString = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.ChaosTelemetrySamplingProbabilityConfigKeyName); double chaosTelemetrySamplingProbability = string.IsNullOrEmpty(chaosTelemetrySamplingProbabilityAsString) ? FASConstants.DefaultChaosTelemetrySamplingProbability : double.Parse(chaosTelemetrySamplingProbabilityAsString); TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Read from config chaosTelemetrySamplingProbability='{0}'", chaosTelemetrySamplingProbability); string chaosTelemetryReportPeriodInSecondsAsString = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.ChaostelemetryReportPeriodInSecondsConfigKeyName); int chaostelemetryReportPeriodInSeconds = string.IsNullOrEmpty(chaosTelemetryReportPeriodInSecondsAsString) ? FASConstants.DefaultTelemetryReportPeriodInSeconds : int.Parse(chaosTelemetryReportPeriodInSecondsAsString); TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Read from config chaostelemetryReportPeriodInSeconds='{0}'", chaostelemetryReportPeriodInSeconds); TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Creating action tables"); var actionTable = await this.StateManager.GetOrAddAsync <IReliableDictionary <Guid, byte[]> >(FASConstants.ActionReliableDictionaryName).ConfigureAwait(false); var historyTable = await this.StateManager.GetOrAddAsync <IReliableDictionary <Guid, byte[]> >(FASConstants.HistoryReliableDictionaryName).ConfigureAwait(false); var stoppedNodeTable = await this.StateManager.GetOrAddAsync <IReliableDictionary <string, bool> >(FASConstants.StoppedNodeTable); TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Creating ActionStore"); this.actionStore = new ActionStore( this.StateManager, this.Partition, actionTable, historyTable, stoppedNodeTable, maxStoredActionCountAsLong, storedActionCleanupIntervalInSecondsAsInt, completedActionKeepDurationInSecondsAsInt, this.IsTestMode, cancellationToken); FabricClient fabricClient = new FabricClient(); this.engine = new ReliableFaultsEngine(this.actionStore, this.IsTestMode, this.Partition, commandStepRetryBackoffInSecondsAsInt); this.messageProcessor = new FaultAnalysisServiceMessageProcessor( this.Partition, fabricClient, this.engine, this.actionStore, this.StateManager, stoppedNodeTable, TimeSpan.FromSeconds(requestTimeoutInSecondsAsInt), TimeSpan.FromSeconds(operationTimeoutInSecondsAsInt), concurrentRequests, dataLossCheckWaitDurationInSeconds, dataLossCheckPollIntervalInSeconds, replicaDropWaitDurationInSeconds, cancellationToken); string maxStoredChaosEventCleanupIntervalInSeconds = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.StoredChaosEventCleanupIntervalInSecondsConfigName); int maxStoredChaosEventCleanupIntervalInSecondsAsInt = string.IsNullOrEmpty(maxStoredChaosEventCleanupIntervalInSeconds) ? FASConstants.StoredChaosEventCleanupIntervalInSecondsDefault : int.Parse(maxStoredChaosEventCleanupIntervalInSeconds); string maxStoredChaosEventCount = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.MaxStoredChaosEventCountConfigName); int maxStoredChaosEventCountAsInt = string.IsNullOrEmpty(maxStoredChaosEventCount) ? FASConstants.MaxStoredChaosEventCountDefault : int.Parse(maxStoredChaosEventCount); this.chaosMessageProcessor = new ChaosMessageProcessor( this.StateManager, apiTestMode, maxStoredChaosEventCleanupIntervalInSecondsAsInt, maxStoredChaosEventCountAsInt, chaosTelemetrySamplingProbability, chaostelemetryReportPeriodInSeconds, this.Partition, fabricClient, this.scheduler, cancellationToken); await this.messageProcessor.ResumePendingActionsAsync(fabricClient, cancellationToken).ConfigureAwait(false); this.faultAnalysisServiceImpl.MessageProcessor = this.messageProcessor; this.faultAnalysisServiceImpl.ChaosMessageProcessor = this.chaosMessageProcessor; this.faultAnalysisServiceImpl.LoadActionMapping(); await this.scheduler.InitializeAsync(fabricClient, this.chaosMessageProcessor, cancellationToken).ConfigureAwait(false); TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Done initializing engine"); await this.InitClusterAnalysisEngineAsync(fabricClient, configStore, cancellationToken).ConfigureAwait(false); }