コード例 #1
0
        static async Task RunChaosTestScenarioAsync(string clusterConnection)
        {
            TimeSpan maxClusterStabilizationTimeout = TimeSpan.FromSeconds(180);
            uint     maxConcurrentFaults            = 3;
            bool     enableMoveReplicaFaults        = true;

            FabricClient fabricClient = new FabricClient(clusterConnection);

            TimeSpan timeToRun = TimeSpan.FromMinutes(60);
            ChaosTestScenarioParameters scenarioParameters = new ChaosTestScenarioParameters(
                maxClusterStabilizationTimeout,
                maxConcurrentFaults,
                enableMoveReplicaFaults,
                timeToRun);

            scenarioParameters.WaitTimeBetweenIterations = TimeSpan.FromSeconds(30);
            scenarioParameters.WaitTimeBetweenFaults     = TimeSpan.FromSeconds(10);

            ChaosTestScenario chaosScenario = new ChaosTestScenario(fabricClient, scenarioParameters);

            try
            {
                await chaosScenario.ExecuteAsync(CancellationToken.None);
            }
            catch (AggregateException ae)
            {
                throw ae.InnerException;
            }
        }
コード例 #2
0
        static async Task RunChaosTestScenarioAsync(string clusterConnection)
        {
            var  maxClusterStabilizationTimeout = TimeSpan.FromSeconds(180);
            uint maxConcurrentFaults            = 3;
            var  enableMoveReplicaFaults        = true;

            // Create FabricClient with connection & security information here.
            var fabricClient = new FabricClient(clusterConnection);

            // The Chaos Test Scenario should run at least 60 minutes or up until it fails.
            var timeToRun          = TimeSpan.FromMinutes(60);
            var scenarioParameters = new ChaosTestScenarioParameters(
                maxClusterStabilizationTimeout,
                maxConcurrentFaults,
                enableMoveReplicaFaults,
                timeToRun);

            // Other related parameters:
            // Pause between two iterations for a random duration bound by this value.
            // scenarioParameters.WaitTimeBetweenIterations = TimeSpan.FromSeconds(30);
            // Pause between concurrent actions for a random duration bound by this value.
            // scenarioParameters.WaitTimeBetweenFaults = TimeSpan.FromSeconds(10);

            // Create the scenario class and execute it asynchronously.
            var chaosScenario = new ChaosTestScenario(fabricClient, scenarioParameters);

            try
            {
                await chaosScenario.ExecuteAsync(CancellationToken.None);
            }
            catch (AggregateException ae)
            {
                throw ae.InnerException;
            }
        }
コード例 #3
0
        /// <summary>
        /// Executes a chaos over test scenario.
        /// </summary>
        /// <param name="serviceName">Uri containing the service name.</param>
        /// <param name="handler">Progress changed handler.</param>
        /// <param name="duration">Duration of the test.</param>
        /// <param name="stabilization">Duration of the stabilization period.</param>
        /// <param name="token">CancellationToken instance.</param>
        /// <returns>Task instance.</returns>
        private async Task ExecuteChaosTestScenarioAsync(Uri serviceName, ProgressChangedEventHandler handler, TimeSpan stabilization, CancellationToken token)
        {
            const int  MaxConcurrentFaults     = 3;
            const bool EnableMoveReplicaFaults = true;

            Console.WriteLine($"Starting chaos scenario test on {serviceName.AbsoluteUri} lasting {_duration.TotalMinutes} minutes.");

            ChaosTestScenarioParameters ctsp = new ChaosTestScenarioParameters(stabilization, MaxConcurrentFaults, EnableMoveReplicaFaults, _duration);
            ChaosTestScenario           cts  = new ChaosTestScenario(_client, ctsp);

            cts.ProgressChanged += handler;

            try
            {
                await cts.ExecuteAsync(token);
            }
            catch (AggregateException ex) { CommonExceptionHandler.OutputInnerExceptions(ex); }
            catch (Exception ex) { Console.WriteLine($"FaultTest.RunAsync Exception: {ex.Message} at {ex.StackTrace}"); }
        }
        protected override async Task RunAsync(CancellationToken runAsyncCancellationToken)
        {
            // This is to keep track of exceptions in the validation step at the end of
            // each iteration of the ChaosTestScenario that is being used under the cover 
            //
            bool validationExceptionCaught = false;

            IReliableDictionary<string, CurrentState> chaosServiceState =
                await this.StateManager.GetOrAddAsync<IReliableDictionary<string, CurrentState>>(StringResource.ChaosServiceStateKey);

            using (ITransaction tx = this.StateManager.CreateTransaction())
            {
                if (!await chaosServiceState.ContainsKeyAsync(tx, StringResource.ChaosServiceStateKey, LockMode.Update))
                {
                    await chaosServiceState.AddAsync(tx, StringResource.ChaosServiceStateKey, CurrentState.Stopped);
                }
                await tx.CommitAsync();
            }


            while (!runAsyncCancellationToken.IsCancellationRequested)
            {
                try
                {
                    // check to see if we're in a "stop" or "start" state.
                    // this continues to poll until we're in a "start" state.
                    // a ReliableDictionary is used to store this information so that if the service
                    //   fails over to another node, the state is preserved and the chaos test will continue to execute.
                    using (ITransaction tx = this.StateManager.CreateTransaction())
                    {
                        ConditionalValue<CurrentState> currentStateResult =
                            await chaosServiceState.TryGetValueAsync(tx, StringResource.ChaosServiceStateKey);

                        if (currentStateResult.HasValue &&
                            (currentStateResult.Value == CurrentState.Stopped ||
                             currentStateResult.Value == CurrentState.None))
                        {
                            await Task.Delay(Constants.IntervalBetweenLoopIteration, runAsyncCancellationToken);
                            continue;
                        }
                    }

                    // this section runs the actual chaos test.
                    // the cancellation token source is linked to the token provided to RunAsync so that we
                    //   can stop the test if the service needs to shut down.
                    using (FabricClient fabricClient = new FabricClient())
                    {
                        using (this.stopEventTokenSource = CancellationTokenSource.CreateLinkedTokenSource(runAsyncCancellationToken))
                        {
                            // when a validation exception is caught, this waits for a while to let the cluster stabilize before continuing.
                            if (validationExceptionCaught)
                            {
                                await Task.Delay(ChaosTestConfigSettings.MaxClusterStabilizationTimeout, this.stopEventTokenSource.Token);
                                validationExceptionCaught = false;
                            }

                            ChaosTestScenarioParameters chaosScenarioParameters =
                                new ChaosTestScenarioParameters(
                                    ChaosTestConfigSettings.MaxClusterStabilizationTimeout,
                                    ChaosTestConfigSettings.MaxConcurrentFaults,
                                    ChaosTestConfigSettings.EnableMoveReplicaFaults,
                                    TimeSpan.MaxValue)
                                {
                                    WaitTimeBetweenFaults =
                                        ChaosTestConfigSettings.WaitTimeBetweenFaults,
                                    WaitTimeBetweenIterations =
                                        ChaosTestConfigSettings.WaitTimeBetweenIterations
                                };

                            ChaosTestScenario chaosTestScenario = new ChaosTestScenario(fabricClient, chaosScenarioParameters);

                            // capture progress events so we can report them back
                            chaosTestScenario.ProgressChanged += this.TestScenarioProgressChanged;

                            // this continuously runs the chaos test until the CancellationToken is signaled.
                            await chaosTestScenario.ExecuteAsync(this.stopEventTokenSource.Token);
                        }
                    }
                }
                catch (TimeoutException e)
                {
                    string message = $"Caught TimeoutException '{e.Message}'. Will wait for cluster to stabilize before continuing test";
                    ServiceEventSource.Current.ServiceMessage(this, message);
                    validationExceptionCaught = true;
                    await this.StoreEventAsync(message);
                }
                catch (FabricValidationException e)
                {
                    string message = $"Caught FabricValidationException '{e.Message}'. Will wait for cluster to stabilize before continuing test";
                    ServiceEventSource.Current.ServiceMessage(this, message);
                    validationExceptionCaught = true;
                    await this.StoreEventAsync(message);
                }
                catch (OperationCanceledException)
                {
                    if (runAsyncCancellationToken.IsCancellationRequested)
                    {
                        // if RunAsync is canceled then we need to quit.
                        throw;
                    }

                    ServiceEventSource.Current.ServiceMessage(
                        this,
                        "Caught OperationCanceledException Exception during test execution. This is expected if test was stopped");
                }
                catch (AggregateException e)
                {
                    if (e.InnerException is OperationCanceledException)
                    {
                        if (runAsyncCancellationToken.IsCancellationRequested)
                        {
                            // if RunAsync is canceled then we need to quit.
                            throw;
                        }

                        ServiceEventSource.Current.ServiceMessage(
                            this,
                            "Caught OperationCanceledException Exception during test execution. This is expected if test was stopped");
                    }
                    else
                    {
                        string message = $"Caught unexpected Exception during test excecution {e.InnerException}";
                        ServiceEventSource.Current.ServiceMessage(this, message);
                        await this.StoreEventAsync(message);
                    }
                }
                catch (Exception e)
                {
                    string message = $"Caught unexpected Exception during test excecution {e}";
                    ServiceEventSource.Current.ServiceMessage(this, message);
                    await this.StoreEventAsync(message);
                }
            }
        }
        protected override async Task RunAsync(CancellationToken runAsyncCancellationToken)
        {
            // This is to keep track of exceptions in the validation step at the end of
            // each iteration of the ChaosTestScenario that is being used under the cover
            //
            bool validationExceptionCaught = false;

            IReliableDictionary <string, CurrentState> chaosServiceState =
                await this.StateManager.GetOrAddAsync <IReliableDictionary <string, CurrentState> >(StringResource.ChaosServiceStateKey);

            using (ITransaction tx = this.StateManager.CreateTransaction())
            {
                if (!await chaosServiceState.ContainsKeyAsync(tx, StringResource.ChaosServiceStateKey, LockMode.Update))
                {
                    await chaosServiceState.AddAsync(tx, StringResource.ChaosServiceStateKey, CurrentState.Stopped);
                }
                await tx.CommitAsync();
            }


            while (!runAsyncCancellationToken.IsCancellationRequested)
            {
                try
                {
                    // check to see if we're in a "stop" or "start" state.
                    // this continues to poll until we're in a "start" state.
                    // a ReliableDictionary is used to store this information so that if the service
                    //   fails over to another node, the state is preserved and the chaos test will continue to execute.
                    using (ITransaction tx = this.StateManager.CreateTransaction())
                    {
                        ConditionalValue <CurrentState> currentStateResult =
                            await chaosServiceState.TryGetValueAsync(tx, StringResource.ChaosServiceStateKey);

                        if (currentStateResult.HasValue &&
                            (currentStateResult.Value == CurrentState.Stopped ||
                             currentStateResult.Value == CurrentState.None))
                        {
                            await Task.Delay(Constants.IntervalBetweenLoopIteration, runAsyncCancellationToken);

                            continue;
                        }
                    }

                    // this section runs the actual chaos test.
                    // the cancellation token source is linked to the token provided to RunAsync so that we
                    //   can stop the test if the service needs to shut down.
                    using (FabricClient fabricClient = new FabricClient())
                    {
                        using (this.stopEventTokenSource = CancellationTokenSource.CreateLinkedTokenSource(runAsyncCancellationToken))
                        {
                            // when a validation exception is caught, this waits for a while to let the cluster stabilize before continuing.
                            if (validationExceptionCaught)
                            {
                                await Task.Delay(ChaosTestConfigSettings.MaxClusterStabilizationTimeout, this.stopEventTokenSource.Token);

                                validationExceptionCaught = false;
                            }

                            ChaosTestScenarioParameters chaosScenarioParameters =
                                new ChaosTestScenarioParameters(
                                    ChaosTestConfigSettings.MaxClusterStabilizationTimeout,
                                    ChaosTestConfigSettings.MaxConcurrentFaults,
                                    ChaosTestConfigSettings.EnableMoveReplicaFaults,
                                    TimeSpan.MaxValue)
                            {
                                WaitTimeBetweenFaults =
                                    ChaosTestConfigSettings.WaitTimeBetweenFaults,
                                WaitTimeBetweenIterations =
                                    ChaosTestConfigSettings.WaitTimeBetweenIterations
                            };

                            ChaosTestScenario chaosTestScenario = new ChaosTestScenario(fabricClient, chaosScenarioParameters);

                            // capture progress events so we can report them back
                            chaosTestScenario.ProgressChanged += this.TestScenarioProgressChanged;

                            // this continuously runs the chaos test until the CancellationToken is signaled.
                            await chaosTestScenario.ExecuteAsync(this.stopEventTokenSource.Token);
                        }
                    }
                }
                catch (TimeoutException e)
                {
                    string message = $"Caught TimeoutException '{e.Message}'. Will wait for cluster to stabilize before continuing test";
                    ServiceEventSource.Current.ServiceMessage(this, message);
                    validationExceptionCaught = true;
                    await this.StoreEventAsync(message);
                }
                catch (FabricValidationException e)
                {
                    string message = $"Caught FabricValidationException '{e.Message}'. Will wait for cluster to stabilize before continuing test";
                    ServiceEventSource.Current.ServiceMessage(this, message);
                    validationExceptionCaught = true;
                    await this.StoreEventAsync(message);
                }
                catch (OperationCanceledException)
                {
                    if (runAsyncCancellationToken.IsCancellationRequested)
                    {
                        // if RunAsync is canceled then we need to quit.
                        throw;
                    }

                    ServiceEventSource.Current.ServiceMessage(
                        this,
                        "Caught OperationCanceledException Exception during test execution. This is expected if test was stopped");
                }
                catch (AggregateException e)
                {
                    if (e.InnerException is OperationCanceledException)
                    {
                        if (runAsyncCancellationToken.IsCancellationRequested)
                        {
                            // if RunAsync is canceled then we need to quit.
                            throw;
                        }

                        ServiceEventSource.Current.ServiceMessage(
                            this,
                            "Caught OperationCanceledException Exception during test execution. This is expected if test was stopped");
                    }
                    else
                    {
                        string message = $"Caught unexpected Exception during test excecution {e.InnerException}";
                        ServiceEventSource.Current.ServiceMessage(this, message);
                        await this.StoreEventAsync(message);
                    }
                }
                catch (Exception e)
                {
                    string message = $"Caught unexpected Exception during test excecution {e}";
                    ServiceEventSource.Current.ServiceMessage(this, message);
                    await this.StoreEventAsync(message);
                }
            }
        }