static async Task RunChaosTestScenarioAsync(string clusterConnection)
{
TimeSpan maxClusterStabilizationTimeout = TimeSpan.FromSeconds(180);
uint maxConcurrentFaults = 3;
bool enableMoveReplicaFaults = true;
FabricClient fabricClient = new FabricClient(clusterConnection);
TimeSpan timeToRun = TimeSpan.FromMinutes(60);
ChaosTestScenarioParameters scenarioParameters = new ChaosTestScenarioParameters(
maxClusterStabilizationTimeout,
maxConcurrentFaults,
enableMoveReplicaFaults,
timeToRun);
scenarioParameters.WaitTimeBetweenIterations = TimeSpan.FromSeconds(30);
scenarioParameters.WaitTimeBetweenFaults = TimeSpan.FromSeconds(10);
ChaosTestScenario chaosScenario = new ChaosTestScenario(fabricClient, scenarioParameters);
try
{
await chaosScenario.ExecuteAsync(CancellationToken.None);
}
catch (AggregateException ae)
{
throw ae.InnerException;
}
}
static async Task RunChaosTestScenarioAsync(string clusterConnection)
{
var maxClusterStabilizationTimeout = TimeSpan.FromSeconds(180);
uint maxConcurrentFaults = 3;
var enableMoveReplicaFaults = true;
// Create FabricClient with connection & security information here.
var fabricClient = new FabricClient(clusterConnection);
// The Chaos Test Scenario should run at least 60 minutes or up until it fails.
var timeToRun = TimeSpan.FromMinutes(60);
var scenarioParameters = new ChaosTestScenarioParameters(
maxClusterStabilizationTimeout,
maxConcurrentFaults,
enableMoveReplicaFaults,
timeToRun);
// Other related parameters:
// Pause between two iterations for a random duration bound by this value.
// scenarioParameters.WaitTimeBetweenIterations = TimeSpan.FromSeconds(30);
// Pause between concurrent actions for a random duration bound by this value.
// scenarioParameters.WaitTimeBetweenFaults = TimeSpan.FromSeconds(10);
// Create the scenario class and execute it asynchronously.
var chaosScenario = new ChaosTestScenario(fabricClient, scenarioParameters);
try
{
await chaosScenario.ExecuteAsync(CancellationToken.None);
}
catch (AggregateException ae)
{
throw ae.InnerException;
}
}
/// <summary>
/// Executes a chaos over test scenario.
/// </summary>
/// <param name="serviceName">Uri containing the service name.</param>
/// <param name="handler">Progress changed handler.</param>
/// <param name="duration">Duration of the test.</param>
/// <param name="stabilization">Duration of the stabilization period.</param>
/// <param name="token">CancellationToken instance.</param>
/// <returns>Task instance.</returns>
private async Task ExecuteChaosTestScenarioAsync(Uri serviceName, ProgressChangedEventHandler handler, TimeSpan stabilization, CancellationToken token)
{
const int MaxConcurrentFaults = 3;
const bool EnableMoveReplicaFaults = true;
Console.WriteLine($"Starting chaos scenario test on {serviceName.AbsoluteUri} lasting {_duration.TotalMinutes} minutes.");
ChaosTestScenarioParameters ctsp = new ChaosTestScenarioParameters(stabilization, MaxConcurrentFaults, EnableMoveReplicaFaults, _duration);
ChaosTestScenario cts = new ChaosTestScenario(_client, ctsp);
cts.ProgressChanged += handler;
try
{
await cts.ExecuteAsync(token);
}
catch (AggregateException ex) { CommonExceptionHandler.OutputInnerExceptions(ex); }
catch (Exception ex) { Console.WriteLine($"FaultTest.RunAsync Exception: {ex.Message} at {ex.StackTrace}"); }
}
protected override async Task RunAsync(CancellationToken runAsyncCancellationToken)
{
// This is to keep track of exceptions in the validation step at the end of
// each iteration of the ChaosTestScenario that is being used under the cover
//
bool validationExceptionCaught = false;
IReliableDictionary<string, CurrentState> chaosServiceState =
await this.StateManager.GetOrAddAsync<IReliableDictionary<string, CurrentState>>(StringResource.ChaosServiceStateKey);
using (ITransaction tx = this.StateManager.CreateTransaction())
{
if (!await chaosServiceState.ContainsKeyAsync(tx, StringResource.ChaosServiceStateKey, LockMode.Update))
{
await chaosServiceState.AddAsync(tx, StringResource.ChaosServiceStateKey, CurrentState.Stopped);
}
await tx.CommitAsync();
}
while (!runAsyncCancellationToken.IsCancellationRequested)
{
try
{
// check to see if we're in a "stop" or "start" state.
// this continues to poll until we're in a "start" state.
// a ReliableDictionary is used to store this information so that if the service
// fails over to another node, the state is preserved and the chaos test will continue to execute.
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue<CurrentState> currentStateResult =
await chaosServiceState.TryGetValueAsync(tx, StringResource.ChaosServiceStateKey);
if (currentStateResult.HasValue &&
(currentStateResult.Value == CurrentState.Stopped ||
currentStateResult.Value == CurrentState.None))
{
await Task.Delay(Constants.IntervalBetweenLoopIteration, runAsyncCancellationToken);
continue;
}
}
// this section runs the actual chaos test.
// the cancellation token source is linked to the token provided to RunAsync so that we
// can stop the test if the service needs to shut down.
using (FabricClient fabricClient = new FabricClient())
{
using (this.stopEventTokenSource = CancellationTokenSource.CreateLinkedTokenSource(runAsyncCancellationToken))
{
// when a validation exception is caught, this waits for a while to let the cluster stabilize before continuing.
if (validationExceptionCaught)
{
await Task.Delay(ChaosTestConfigSettings.MaxClusterStabilizationTimeout, this.stopEventTokenSource.Token);
validationExceptionCaught = false;
}
ChaosTestScenarioParameters chaosScenarioParameters =
new ChaosTestScenarioParameters(
ChaosTestConfigSettings.MaxClusterStabilizationTimeout,
ChaosTestConfigSettings.MaxConcurrentFaults,
ChaosTestConfigSettings.EnableMoveReplicaFaults,
TimeSpan.MaxValue)
{
WaitTimeBetweenFaults =
ChaosTestConfigSettings.WaitTimeBetweenFaults,
WaitTimeBetweenIterations =
ChaosTestConfigSettings.WaitTimeBetweenIterations
};
ChaosTestScenario chaosTestScenario = new ChaosTestScenario(fabricClient, chaosScenarioParameters);
// capture progress events so we can report them back
chaosTestScenario.ProgressChanged += this.TestScenarioProgressChanged;
// this continuously runs the chaos test until the CancellationToken is signaled.
await chaosTestScenario.ExecuteAsync(this.stopEventTokenSource.Token);
}
}
}
catch (TimeoutException e)
{
string message = $"Caught TimeoutException '{e.Message}'. Will wait for cluster to stabilize before continuing test";
ServiceEventSource.Current.ServiceMessage(this, message);
validationExceptionCaught = true;
await this.StoreEventAsync(message);
}
catch (FabricValidationException e)
{
string message = $"Caught FabricValidationException '{e.Message}'. Will wait for cluster to stabilize before continuing test";
ServiceEventSource.Current.ServiceMessage(this, message);
validationExceptionCaught = true;
await this.StoreEventAsync(message);
}
catch (OperationCanceledException)
{
if (runAsyncCancellationToken.IsCancellationRequested)
{
// if RunAsync is canceled then we need to quit.
throw;
}
ServiceEventSource.Current.ServiceMessage(
this,
"Caught OperationCanceledException Exception during test execution. This is expected if test was stopped");
}
catch (AggregateException e)
{
if (e.InnerException is OperationCanceledException)
{
if (runAsyncCancellationToken.IsCancellationRequested)
{
// if RunAsync is canceled then we need to quit.
throw;
}
ServiceEventSource.Current.ServiceMessage(
this,
"Caught OperationCanceledException Exception during test execution. This is expected if test was stopped");
}
else
{
string message = $"Caught unexpected Exception during test excecution {e.InnerException}";
ServiceEventSource.Current.ServiceMessage(this, message);
await this.StoreEventAsync(message);
}
}
catch (Exception e)
{
string message = $"Caught unexpected Exception during test excecution {e}";
ServiceEventSource.Current.ServiceMessage(this, message);
await this.StoreEventAsync(message);
}
}
}
protected override async Task RunAsync(CancellationToken runAsyncCancellationToken)
{
// This is to keep track of exceptions in the validation step at the end of
// each iteration of the ChaosTestScenario that is being used under the cover
//
bool validationExceptionCaught = false;
IReliableDictionary <string, CurrentState> chaosServiceState =
await this.StateManager.GetOrAddAsync <IReliableDictionary <string, CurrentState> >(StringResource.ChaosServiceStateKey);
using (ITransaction tx = this.StateManager.CreateTransaction())
{
if (!await chaosServiceState.ContainsKeyAsync(tx, StringResource.ChaosServiceStateKey, LockMode.Update))
{
await chaosServiceState.AddAsync(tx, StringResource.ChaosServiceStateKey, CurrentState.Stopped);
}
await tx.CommitAsync();
}
while (!runAsyncCancellationToken.IsCancellationRequested)
{
try
{
// check to see if we're in a "stop" or "start" state.
// this continues to poll until we're in a "start" state.
// a ReliableDictionary is used to store this information so that if the service
// fails over to another node, the state is preserved and the chaos test will continue to execute.
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <CurrentState> currentStateResult =
await chaosServiceState.TryGetValueAsync(tx, StringResource.ChaosServiceStateKey);
if (currentStateResult.HasValue &&
(currentStateResult.Value == CurrentState.Stopped ||
currentStateResult.Value == CurrentState.None))
{
await Task.Delay(Constants.IntervalBetweenLoopIteration, runAsyncCancellationToken);
continue;
}
}
// this section runs the actual chaos test.
// the cancellation token source is linked to the token provided to RunAsync so that we
// can stop the test if the service needs to shut down.
using (FabricClient fabricClient = new FabricClient())
{
using (this.stopEventTokenSource = CancellationTokenSource.CreateLinkedTokenSource(runAsyncCancellationToken))
{
// when a validation exception is caught, this waits for a while to let the cluster stabilize before continuing.
if (validationExceptionCaught)
{
await Task.Delay(ChaosTestConfigSettings.MaxClusterStabilizationTimeout, this.stopEventTokenSource.Token);
validationExceptionCaught = false;
}
ChaosTestScenarioParameters chaosScenarioParameters =
new ChaosTestScenarioParameters(
ChaosTestConfigSettings.MaxClusterStabilizationTimeout,
ChaosTestConfigSettings.MaxConcurrentFaults,
ChaosTestConfigSettings.EnableMoveReplicaFaults,
TimeSpan.MaxValue)
{
WaitTimeBetweenFaults =
ChaosTestConfigSettings.WaitTimeBetweenFaults,
WaitTimeBetweenIterations =
ChaosTestConfigSettings.WaitTimeBetweenIterations
};
ChaosTestScenario chaosTestScenario = new ChaosTestScenario(fabricClient, chaosScenarioParameters);
// capture progress events so we can report them back
chaosTestScenario.ProgressChanged += this.TestScenarioProgressChanged;
// this continuously runs the chaos test until the CancellationToken is signaled.
await chaosTestScenario.ExecuteAsync(this.stopEventTokenSource.Token);
}
}
}
catch (TimeoutException e)
{
string message = $"Caught TimeoutException '{e.Message}'. Will wait for cluster to stabilize before continuing test";
ServiceEventSource.Current.ServiceMessage(this, message);
validationExceptionCaught = true;
await this.StoreEventAsync(message);
}
catch (FabricValidationException e)
{
string message = $"Caught FabricValidationException '{e.Message}'. Will wait for cluster to stabilize before continuing test";
ServiceEventSource.Current.ServiceMessage(this, message);
validationExceptionCaught = true;
await this.StoreEventAsync(message);
}
catch (OperationCanceledException)
{
if (runAsyncCancellationToken.IsCancellationRequested)
{
// if RunAsync is canceled then we need to quit.
throw;
}
ServiceEventSource.Current.ServiceMessage(
this,
"Caught OperationCanceledException Exception during test execution. This is expected if test was stopped");
}
catch (AggregateException e)
{
if (e.InnerException is OperationCanceledException)
{
if (runAsyncCancellationToken.IsCancellationRequested)
{
// if RunAsync is canceled then we need to quit.
throw;
}
ServiceEventSource.Current.ServiceMessage(
this,
"Caught OperationCanceledException Exception during test execution. This is expected if test was stopped");
}
else
{
string message = $"Caught unexpected Exception during test excecution {e.InnerException}";
ServiceEventSource.Current.ServiceMessage(this, message);
await this.StoreEventAsync(message);
}
}
catch (Exception e)
{
string message = $"Caught unexpected Exception during test excecution {e}";
ServiceEventSource.Current.ServiceMessage(this, message);
await this.StoreEventAsync(message);
}
}
}
