internal async Task InitializeAsync(FabricClient fabricClient, ChaosMessageProcessor chaosMessageProcessor, CancellationToken cancellationToken)
{
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "InitializeAsync - started");
this.fabricClient = fabricClient;
this.chaosMessageProcessor = chaosMessageProcessor;
await this.RestartRecoveryAsync(cancellationToken).ConfigureAwait(false);
#pragma warning disable 4014
Task.Run(() => this.StartAndRunAsync(cancellationToken)).ContinueWith(task =>
{
if (task.IsCanceled)
{
// This will print after the thread running the scheduler ends.
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "Scheduler task cancelled.");
}
});
#pragma warning restore 4014
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "InitializeAsync - ended");
}
private async Task InitializeAsync(CancellationToken cancellationToken)
{
System.Fabric.Common.NativeConfigStore configStore = System.Fabric.Common.NativeConfigStore.FabricGetConfigStore();
string isTestModeEnabled = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, "TestModeEnabled");
if (isTestModeEnabled.ToLowerInvariant() == "true")
{
TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Enabling internal test mode");
this.IsTestMode = true;
}
string apiTestModeString = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, "ApiTestMode");
int apiTestMode;
if (!int.TryParse(apiTestModeString, out apiTestMode))
{
apiTestMode = 0;
}
string requestTimeout = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.RequestTimeoutInSecondsName);
int requestTimeoutInSecondsAsInt = string.IsNullOrEmpty(requestTimeout) ? FASConstants.DefaultRequestTimeoutInSeconds : int.Parse(requestTimeout);
string operationTimeout = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.OperationTimeoutInSecondsName);
int operationTimeoutInSecondsAsInt = string.IsNullOrEmpty(operationTimeout) ? FASConstants.DefaultOperationTimeoutInSeconds : int.Parse(operationTimeout);
TestabilityTrace.TraceSource.WriteInfo(
FaultAnalysisServiceTraceType,
"Setting requestTimeout='{0}' and operationTimeout='{1}'",
TimeSpan.FromSeconds(requestTimeoutInSecondsAsInt),
TimeSpan.FromSeconds(operationTimeoutInSecondsAsInt));
string maxStoredActionCount = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.MaxStoredActionCountName);
long maxStoredActionCountAsLong = string.IsNullOrEmpty(maxStoredActionCount) ? FASConstants.DefaultMaxStoredActionCount : long.Parse(maxStoredActionCount);
string storedActionCleanupIntervalInSeconds = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.StoredActionCleanupIntervalInSecondsName);
int storedActionCleanupIntervalInSecondsAsInt = string.IsNullOrEmpty(storedActionCleanupIntervalInSeconds) ? FASConstants.DefaultStoredActionCleanupIntervalInSeconds : int.Parse(storedActionCleanupIntervalInSeconds);
string completedActionKeepDurationInSeconds = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.CompletedActionKeepDurationInSecondsName);
int completedActionKeepDurationInSecondsAsInt = string.IsNullOrEmpty(completedActionKeepDurationInSeconds) ? FASConstants.DefaultCompletedActionKeepDurationInSeconds : int.Parse(completedActionKeepDurationInSeconds);
TestabilityTrace.TraceSource.WriteInfo(
FaultAnalysisServiceTraceType,
"MaxStoredActionCount={0}, StoredActionCleanupIntervalInSeconds={1}, CompletedActionKeepDurationInSecondsName={2}",
maxStoredActionCountAsLong,
storedActionCleanupIntervalInSecondsAsInt,
completedActionKeepDurationInSecondsAsInt);
string commandStepRetryBackoffInSeconds = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.CommandStepRetryBackoffInSecondsName);
int commandStepRetryBackoffInSecondsAsInt = string.IsNullOrEmpty(commandStepRetryBackoffInSeconds) ? FASConstants.DefaultCommandStepRetryBackoffInSeconds : int.Parse(commandStepRetryBackoffInSeconds);
TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Read from config commandStepRetryBackoffInSeconds='{0}'", commandStepRetryBackoffInSecondsAsInt);
string concurrentRequestsAsString = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.ConcurrentRequestsName);
int concurrentRequests = string.IsNullOrEmpty(concurrentRequestsAsString) ? FASConstants.DefaultConcurrentRequests : int.Parse(concurrentRequestsAsString);
TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Read from config concurrentRequests='{0}'", concurrentRequests);
string dataLossCheckWaitDurationInSecondsAsString = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.DataLossCheckWaitDurationInSecondsName);
int dataLossCheckWaitDurationInSeconds = string.IsNullOrEmpty(dataLossCheckWaitDurationInSecondsAsString) ? FASConstants.DefaultDataLossCheckWaitDurationInSeconds : int.Parse(dataLossCheckWaitDurationInSecondsAsString);
TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Read from config dataLossCheckWaitDurationInSeconds='{0}'", dataLossCheckWaitDurationInSeconds);
string dataLossCheckPollIntervalInSecondsAsString = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.DataLossCheckPollIntervalInSecondsName);
int dataLossCheckPollIntervalInSeconds = string.IsNullOrEmpty(dataLossCheckPollIntervalInSecondsAsString) ? FASConstants.DefaultDataLossCheckPollIntervalInSeconds : int.Parse(dataLossCheckPollIntervalInSecondsAsString);
TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Read from config dataLossCheckPollIntervalInSeconds='{0}'", dataLossCheckPollIntervalInSeconds);
string replicaDropWaitDurationInSecondsAsString = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.ReplicaDropWaitDurationInSecondsName);
int replicaDropWaitDurationInSeconds = string.IsNullOrEmpty(replicaDropWaitDurationInSecondsAsString) ? FASConstants.DefaultReplicaDropWaitDurationInSeconds : int.Parse(replicaDropWaitDurationInSecondsAsString);
TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Read from config replicaDropWaitDurationInSeconds='{0}'", replicaDropWaitDurationInSeconds);
string chaosTelemetrySamplingProbabilityAsString = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.ChaosTelemetrySamplingProbabilityConfigKeyName);
double chaosTelemetrySamplingProbability = string.IsNullOrEmpty(chaosTelemetrySamplingProbabilityAsString) ? FASConstants.DefaultChaosTelemetrySamplingProbability : double.Parse(chaosTelemetrySamplingProbabilityAsString);
TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Read from config chaosTelemetrySamplingProbability='{0}'", chaosTelemetrySamplingProbability);
string chaosTelemetryReportPeriodInSecondsAsString = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.ChaostelemetryReportPeriodInSecondsConfigKeyName);
int chaostelemetryReportPeriodInSeconds = string.IsNullOrEmpty(chaosTelemetryReportPeriodInSecondsAsString) ? FASConstants.DefaultTelemetryReportPeriodInSeconds : int.Parse(chaosTelemetryReportPeriodInSecondsAsString);
TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Read from config chaostelemetryReportPeriodInSeconds='{0}'", chaostelemetryReportPeriodInSeconds);
TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Creating action tables");
var actionTable = await this.StateManager.GetOrAddAsync <IReliableDictionary <Guid, byte[]> >(FASConstants.ActionReliableDictionaryName).ConfigureAwait(false);
var historyTable = await this.StateManager.GetOrAddAsync <IReliableDictionary <Guid, byte[]> >(FASConstants.HistoryReliableDictionaryName).ConfigureAwait(false);
var stoppedNodeTable = await this.StateManager.GetOrAddAsync <IReliableDictionary <string, bool> >(FASConstants.StoppedNodeTable);
TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Creating ActionStore");
this.actionStore = new ActionStore(
this.StateManager,
this.Partition,
actionTable,
historyTable,
stoppedNodeTable,
maxStoredActionCountAsLong,
storedActionCleanupIntervalInSecondsAsInt,
completedActionKeepDurationInSecondsAsInt,
this.IsTestMode,
cancellationToken);
FabricClient fabricClient = new FabricClient();
this.engine = new ReliableFaultsEngine(this.actionStore, this.IsTestMode, this.Partition, commandStepRetryBackoffInSecondsAsInt);
this.messageProcessor = new FaultAnalysisServiceMessageProcessor(
this.Partition,
fabricClient,
this.engine,
this.actionStore,
this.StateManager,
stoppedNodeTable,
TimeSpan.FromSeconds(requestTimeoutInSecondsAsInt),
TimeSpan.FromSeconds(operationTimeoutInSecondsAsInt),
concurrentRequests,
dataLossCheckWaitDurationInSeconds,
dataLossCheckPollIntervalInSeconds,
replicaDropWaitDurationInSeconds,
cancellationToken);
string maxStoredChaosEventCleanupIntervalInSeconds = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.StoredChaosEventCleanupIntervalInSecondsConfigName);
int maxStoredChaosEventCleanupIntervalInSecondsAsInt = string.IsNullOrEmpty(maxStoredChaosEventCleanupIntervalInSeconds) ? FASConstants.StoredChaosEventCleanupIntervalInSecondsDefault : int.Parse(maxStoredChaosEventCleanupIntervalInSeconds);
string maxStoredChaosEventCount = configStore.ReadUnencryptedString(FASConstants.FaultAnalysisServiceConfigSectionName, FASConstants.MaxStoredChaosEventCountConfigName);
int maxStoredChaosEventCountAsInt = string.IsNullOrEmpty(maxStoredChaosEventCount) ? FASConstants.MaxStoredChaosEventCountDefault : int.Parse(maxStoredChaosEventCount);
this.chaosMessageProcessor = new ChaosMessageProcessor(
this.StateManager,
apiTestMode,
maxStoredChaosEventCleanupIntervalInSecondsAsInt,
maxStoredChaosEventCountAsInt,
chaosTelemetrySamplingProbability,
chaostelemetryReportPeriodInSeconds,
this.Partition,
fabricClient,
this.scheduler,
cancellationToken);
await this.messageProcessor.ResumePendingActionsAsync(fabricClient, cancellationToken).ConfigureAwait(false);
this.faultAnalysisServiceImpl.MessageProcessor = this.messageProcessor;
this.faultAnalysisServiceImpl.ChaosMessageProcessor = this.chaosMessageProcessor;
this.faultAnalysisServiceImpl.LoadActionMapping();
await this.scheduler.InitializeAsync(fabricClient, this.chaosMessageProcessor, cancellationToken).ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Done initializing engine");
await this.InitClusterAnalysisEngineAsync(fabricClient, configStore, cancellationToken).ConfigureAwait(false);
}