protected override async Task ExecuteActionAsync(FabricTestContext testContext, GetClusterStateSnapshotAction action, CancellationToken cancellationToken)
{
Dictionary <string, int> ExceptionHistory = new Dictionary <string, int>();
int retries = 0;
GetClusterStateSnapshotAction.ServiceCount = 0;
GetClusterStateSnapshotAction.PartitionCount = 0;
GetClusterStateSnapshotAction.ReplicaCount = 0;
Stopwatch stopWatch = Stopwatch.StartNew();
ClusterStateSnapshot clusterSnapshot = null;
do
{
++retries;
await Task.Delay(Constants.DefaultChaosSnapshotRecaptureBackoffInterval, cancellationToken).ConfigureAwait(false);
try
{
clusterSnapshot = await this.CaptureClusterStateSnapshotAndPopulateEntitiesAsync(
testContext,
action,
cancellationToken).ConfigureAwait(false);
}
catch (Exception exception) when(exception is FabricException || exception is ChaosInconsistentClusterSnapshotException)
{
string exceptionString = exception.Message;
if (ExceptionHistory.ContainsKey(exceptionString))
{
ExceptionHistory[exceptionString]++;
}
else
{
ExceptionHistory[exceptionString] = 1;
}
}
string allExceptions = string.Join(ExceptionDelimeter, ExceptionHistory);
if (retries >= action.MaximumNumberOfRetries)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "While taking a consistent cluster snapshot, following exceptions occurred: {0}", allExceptions);
}
ChaosUtility.ThrowOrAssertIfTrue(
ChaosConstants.GetClusterSnapshotAction_MaximumNumberOfRetriesAchieved_TelemetryId,
retries >= action.MaximumNumberOfRetries,
string.Format(StringResources.ChaosEngineError_GetClusterSnapshotAction_MaximumNumberOfRetriesAchieved, action.MaximumNumberOfRetries, allExceptions));
}while (clusterSnapshot == null);
stopWatch.Stop();
var elapsedInGatherSnapshot = stopWatch.Elapsed;
stopWatch = Stopwatch.StartNew();
clusterSnapshot.ApplyChaosTargetFilter(action.ChaosTargetFilter);
clusterSnapshot.MarkAllUnsafeEntities();
stopWatch.Stop();
var elapsedInMarkAllUnsafe = stopWatch.Elapsed;
if (UniformRandomNumberGenerator.NextDouble() < action.TelemetrySamplingProbability)
{
FabricEvents.Events.ChaosSnapshot(
Guid.NewGuid().ToString(),
clusterSnapshot.Nodes.Count,
clusterSnapshot.Applications.Count,
GetClusterStateSnapshotAction.ServiceCount,
GetClusterStateSnapshotAction.PartitionCount,
GetClusterStateSnapshotAction.ReplicaCount,
elapsedInGatherSnapshot.TotalSeconds,
elapsedInMarkAllUnsafe.TotalSeconds,
retries);
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "For '{0}' nodes, '{1}' apps, '{2}' services, '{3}' partitions, '{4}' replicas, snapshot took '{5}', mark unsafe took '{6}', took '{7}' retries.",
clusterSnapshot.Nodes.Count,
clusterSnapshot.Applications.Count,
GetClusterStateSnapshotAction.ServiceCount,
GetClusterStateSnapshotAction.PartitionCount,
GetClusterStateSnapshotAction.ReplicaCount,
elapsedInGatherSnapshot,
elapsedInMarkAllUnsafe,
retries);
action.Result = clusterSnapshot;
ResultTraceString = "GetClusterStateSnapshotAction succeeded";
}
