protected override async Task ExecuteActionAsync(FabricTestContext testContext, GetClusterStateSnapshotAction action, CancellationToken cancellationToken) { Dictionary <string, int> ExceptionHistory = new Dictionary <string, int>(); int retries = 0; GetClusterStateSnapshotAction.ServiceCount = 0; GetClusterStateSnapshotAction.PartitionCount = 0; GetClusterStateSnapshotAction.ReplicaCount = 0; Stopwatch stopWatch = Stopwatch.StartNew(); ClusterStateSnapshot clusterSnapshot = null; do { ++retries; await Task.Delay(Constants.DefaultChaosSnapshotRecaptureBackoffInterval, cancellationToken).ConfigureAwait(false); try { clusterSnapshot = await this.CaptureClusterStateSnapshotAndPopulateEntitiesAsync( testContext, action, cancellationToken).ConfigureAwait(false); } catch (Exception exception) when(exception is FabricException || exception is ChaosInconsistentClusterSnapshotException) { string exceptionString = exception.Message; if (ExceptionHistory.ContainsKey(exceptionString)) { ExceptionHistory[exceptionString]++; } else { ExceptionHistory[exceptionString] = 1; } } string allExceptions = string.Join(ExceptionDelimeter, ExceptionHistory); if (retries >= action.MaximumNumberOfRetries) { TestabilityTrace.TraceSource.WriteWarning(TraceType, "While taking a consistent cluster snapshot, following exceptions occurred: {0}", allExceptions); } ChaosUtility.ThrowOrAssertIfTrue( ChaosConstants.GetClusterSnapshotAction_MaximumNumberOfRetriesAchieved_TelemetryId, retries >= action.MaximumNumberOfRetries, string.Format(StringResources.ChaosEngineError_GetClusterSnapshotAction_MaximumNumberOfRetriesAchieved, action.MaximumNumberOfRetries, allExceptions)); }while (clusterSnapshot == null); stopWatch.Stop(); var elapsedInGatherSnapshot = stopWatch.Elapsed; stopWatch = Stopwatch.StartNew(); clusterSnapshot.ApplyChaosTargetFilter(action.ChaosTargetFilter); clusterSnapshot.MarkAllUnsafeEntities(); stopWatch.Stop(); var elapsedInMarkAllUnsafe = stopWatch.Elapsed; if (UniformRandomNumberGenerator.NextDouble() < action.TelemetrySamplingProbability) { FabricEvents.Events.ChaosSnapshot( Guid.NewGuid().ToString(), clusterSnapshot.Nodes.Count, clusterSnapshot.Applications.Count, GetClusterStateSnapshotAction.ServiceCount, GetClusterStateSnapshotAction.PartitionCount, GetClusterStateSnapshotAction.ReplicaCount, elapsedInGatherSnapshot.TotalSeconds, elapsedInMarkAllUnsafe.TotalSeconds, retries); } TestabilityTrace.TraceSource.WriteInfo(TraceType, "For '{0}' nodes, '{1}' apps, '{2}' services, '{3}' partitions, '{4}' replicas, snapshot took '{5}', mark unsafe took '{6}', took '{7}' retries.", clusterSnapshot.Nodes.Count, clusterSnapshot.Applications.Count, GetClusterStateSnapshotAction.ServiceCount, GetClusterStateSnapshotAction.PartitionCount, GetClusterStateSnapshotAction.ReplicaCount, elapsedInGatherSnapshot, elapsedInMarkAllUnsafe, retries); action.Result = clusterSnapshot; ResultTraceString = "GetClusterStateSnapshotAction succeeded"; }