public GetChaosReportDescription(
ChaosReportFilter filter,
string continuationToken)
{
this.Filter = filter;
this.ContinuationToken = continuationToken;
}
public async Task <ChaosReport> GetChaosReportAsync(
GetChaosReportDescription getChaosReportDescription,
TimeSpan timeout,
CancellationToken cancellationToken)
{
Guid activityId = Guid.NewGuid();
this.ThrowIfNotReady(() => this.ChaosMessageProcessor != null);
TestabilityTrace.TraceSource.WriteNoise(TraceType, "{0}:Enter GetChaosReportAsync, description={1}", activityId, getChaosReportDescription.ToString());
ChaosReport report = null;
try
{
ChaosReportFilter reportFilter = null;
var continuationToken = getChaosReportDescription.ContinuationToken;
if (!string.IsNullOrEmpty(continuationToken))
{
reportFilter = await this.ChaosMessageProcessor.GetReportFilterAsync(activityId, continuationToken).ConfigureAwait(false);
}
else
{
var continuationTokenGuid = Guid.NewGuid().ToString();
var fileTimeUtcTicks = DateTime.UtcNow.ToFileTimeUtc();
continuationToken = string.Format(FASConstants.ChaosReportContinuationTokenFormat, fileTimeUtcTicks, continuationTokenGuid);
}
reportFilter = reportFilter ?? getChaosReportDescription.Filter;
if (!string.IsNullOrEmpty(getChaosReportDescription.ClientType) &&
(getChaosReportDescription.ClientType.Equals(ChaosConstants.RestClientTypeName, StringComparison.OrdinalIgnoreCase) ||
getChaosReportDescription.ClientType.Equals(ChaosConstants.NativeClientTypeName, StringComparison.OrdinalIgnoreCase)))
{
ChaosUtility.DisableOptimizationForValidationFailedEvent = true;
}
else
{
ChaosUtility.DisableOptimizationForValidationFailedEvent = false;
}
report = await this.ChaosMessageProcessor.ProcessGetChaosReportAsync(
activityId,
reportFilter,
continuationToken).ConfigureAwait(false);
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0}:GetChaosReportAsync - Exception occurred: {1}", activityId, e.Message);
FaultAnalysisServiceUtility.ThrowTransientExceptionIfRetryable(e);
throw;
}
TestabilityTrace.TraceSource.WriteNoise(TraceType, "{0}:GetChaosReportAsync - returning report.", activityId);
return(report);
}
public static ChaosReportFilter GetReportFilterFromBytes(byte[] data)
{
ChaosReportFilter filter = new ChaosReportFilter();
filter.FromBytes(data);
return(filter);
}
private ChaosReportFilter CreateChaosReportFilter()
{
var filter = new ChaosReportFilter(
this.StartTimeUtc.HasValue ? this.StartTimeUtc.Value : DateTime.MinValue,
this.EndTimeUtc.HasValue ? this.EndTimeUtc.Value : DateTime.MaxValue);
return(filter);
}
static async Task Chaos()
{
var clusterConnectionString = "localhost:19000";
using (var client = new FabricClient(clusterConnectionString))
{
var startTimeUtc = DateTime.UtcNow;
var stabilizationTimeout = TimeSpan.FromSeconds(10.00);
var timeToRun = TimeSpan.FromMinutes(3);
var maxConcurrentFaults = 3;
var parameters = new ChaosParameters(
stabilizationTimeout,
maxConcurrentFaults,
true, /* EnableMoveReplicaFault */
timeToRun);
try
{
await client.TestManager.StartChaosAsync(parameters);
}
catch (FabricChaosAlreadyRunningException)
{
await client.TestManager.StopChaosAsync();
await client.TestManager.StartChaosAsync(parameters);
}
var filter = new ChaosReportFilter(startTimeUtc, DateTime.MaxValue);
var eventSet = new HashSet <ChaosEvent>(new ChaosEventComparer());
while (true)
{
var report = client.TestManager.GetChaosReportAsync(filter).GetAwaiter().GetResult();
foreach (var chaosEvent in report.History)
{
if (eventSet.Add(chaosEvent))
{
Console.WriteLine(chaosEvent);
}
}
// When Chaos stops, a StoppedEvent is created.
// If a StoppedEvent is found, exit the loop.
var lastEvent = report.History.LastOrDefault();
if (lastEvent is StoppedEvent)
{
break;
}
await Task.Delay(TimeSpan.FromSeconds(1.0));
}
}
}
static void Main(string[] args)
{
var clusterConnectionString = "localhost:19000";
using (var client = new FabricClient(clusterConnectionString))
{
var startTimeUtc = DateTime.UtcNow;
var stabilizationTimeout = TimeSpan.FromSeconds(30.0);
var timeToRun = TimeSpan.FromMinutes(60.0);
var maxConcurrentFaults = 3;
var parameters = new ChaosParameters(
stabilizationTimeout,
maxConcurrentFaults,
true, /* EnableMoveReplicaFault */
timeToRun);
try
{
client.TestManager.StartChaosAsync(parameters).GetAwaiter().GetResult();
}
catch (FabricChaosAlreadyRunningException)
{
Console.WriteLine("An instance of Chaos is already running in the cluster.");
}
var filter = new ChaosReportFilter(startTimeUtc, DateTime.MaxValue);
var eventSet = new HashSet <ChaosEvent>(new ChaosEventComparer());
while (true)
{
var report = client.TestManager.GetChaosReportAsync(filter).GetAwaiter().GetResult();
foreach (var chaosEvent in report.History)
{
if (eventSet.Add(chaosEvent))
{
Console.WriteLine(chaosEvent);
}
}
var lastEvent = report.History.LastOrDefault();
if (lastEvent is StoppedEvent)
{
break;
}
Task.Delay(TimeSpan.FromSeconds(1.0)).GetAwaiter().GetResult();
}
}
}
internal static unsafe GetChaosReportDescription CreateFromNative(IntPtr nativeRaw)
{
NativeTypes.FABRIC_GET_CHAOS_REPORT_DESCRIPTION native = *(NativeTypes.FABRIC_GET_CHAOS_REPORT_DESCRIPTION *)nativeRaw;
var filter = ChaosReportFilter.FromNative(native.Filter);
string continuationToken = NativeTypes.FromNativeString(native.ContinuationToken);
var description = new GetChaosReportDescription(filter, continuationToken);
if (native.Reserved != IntPtr.Zero)
{
var clientType = *((NativeTypes.FABRIC_CHAOS_CLIENT_TYPE *)native.Reserved);
if (clientType.ClientType != IntPtr.Zero)
{
description.ClientType = NativeTypes.FromNativeString(clientType.ClientType);
}
}
return(description);
}
static void Main(string[] args)
{
// README:
//
// Please ensure your cluster certificate is installed in
// the 'CurrentUser' certificate store.
//
// REQUIRED STEPS:
// - Paste your Service Fabric certificate's thumbprint below (line 52,53)
// - Update the cluster domain name to match your SF cluster (line 54)
// - Add your cluster node types to the inclusion list (line 102)
string clientCertThumb = "D6426E96E0169B60ED030E53FCD05EAC12AAA1E0";
string serverCertThumb = "D6426E96E0169B60ED030E53FCD05EAC12AAA1E0";
string clusterDomainName = "dotjson.westeurope.cloudapp.azure.com";
string commonName = $"www.{clusterDomainName}";
string clusterEndpoint = $"{clusterDomainName}:19000";
var creds = GetCredentials(clientCertThumb, serverCertThumb, commonName);
Console.WriteLine($"Connecting to cluster {clusterEndpoint} using certificate '{clientCertThumb}'.");
using (var client = new FabricClient(creds, clusterEndpoint))
{
var startTimeUtc = DateTime.UtcNow;
// The maximum amount of time to wait for all cluster entities to become stable and healthy.
// Chaos executes in iterations and at the start of each iteration it validates the health of cluster entities.
// During validation if a cluster entity is not stable and healthy within MaxClusterStabilizationTimeoutInSeconds, Chaos generates a validation failed event.
var maxClusterStabilizationTimeout = TimeSpan.FromSeconds(30.0);
var timeToRun = TimeSpan.FromMinutes(60.0);
// MaxConcurrentFaults is the maximum number of concurrent faults induced per iteration.
// Chaos executes in iterations and two consecutive iterations are separated by a validation phase.
// The higher the concurrency, the more aggressive the injection of faults -- inducing more complex series of states to uncover bugs.
// The recommendation is to start with a value of 2 or 3 and to exercise caution while moving up.
var maxConcurrentFaults = 3;
// Describes a map, which is a collection of (string, string) type key-value pairs. The map can be used to record information about
// the Chaos run. There cannot be more than 100 such pairs and each string (key or value) can be at most 4095 characters long.
// This map is set by the starter of the Chaos run to optionally store the context about the specific run.
var startContext = new Dictionary <string, string> {
{ "ReasonForStart", "Testing" }
};
// Time-separation (in seconds) between two consecutive iterations of Chaos. The larger the value, the lower the fault injection rate.
var waitTimeBetweenIterations = TimeSpan.FromSeconds(10);
// Wait time (in seconds) between consecutive faults within a single iteration.
// The larger the value, the lower the overlapping between faults and the simpler the sequence of state transitions that the cluster goes through.
// The recommendation is to start with a value between 1 and 5 and exercise caution while moving up.
var waitTimeBetweenFaults = TimeSpan.Zero;
// Passed-in cluster health policy is used to validate health of the cluster in between Chaos iterations.
var clusterHealthPolicy = new ClusterHealthPolicy
{
ConsiderWarningAsError = false,
MaxPercentUnhealthyApplications = 100,
MaxPercentUnhealthyNodes = 100
};
// All types of faults, restart node, restart code package, restart replica, move primary replica, and move secondary replica will happen
// for nodes of type 'FrontEndType'
var nodetypeInclusionList = new List <string> {
"nodetype0"
};
// In addition to the faults included by nodetypeInclusionList,
// restart code package, restart replica, move primary replica, move secondary replica faults will happen for 'fabric:/TestApp2'
// even if a replica or code package from 'fabric:/TestApp2' is residing on a node which is not of type included in nodeypeInclusionList.
var applicationInclusionList = new List <string> {
"fabric:/Exchange"
};
// List of cluster entities to target for Chaos faults.
var chaosTargetFilter = new ChaosTargetFilter
{
NodeTypeInclusionList = nodetypeInclusionList,
ApplicationInclusionList = applicationInclusionList
};
var parameters = new ChaosParameters(
maxClusterStabilizationTimeout,
maxConcurrentFaults,
true, /* EnableMoveReplicaFault */
timeToRun,
startContext,
waitTimeBetweenIterations,
waitTimeBetweenFaults,
clusterHealthPolicy)
{
ChaosTargetFilter = chaosTargetFilter
};
try
{
client.TestManager.StartChaosAsync(parameters).GetAwaiter().GetResult();
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(30)); // Allow enough time for Chaos engine to start
}
catch (FabricChaosAlreadyRunningException)
{
Console.WriteLine("An instance of Chaos is already running in the cluster");
}
var filter = new ChaosReportFilter(startTimeUtc, DateTime.MaxValue);
var eventSet = new HashSet <ChaosEvent>(new ChaosEventComparer());
string continuationToken = null;
while (true)
{
ChaosReport report;
try
{
report = string.IsNullOrEmpty(continuationToken)
? client.TestManager.GetChaosReportAsync(filter).GetAwaiter().GetResult()
: client.TestManager.GetChaosReportAsync(continuationToken).GetAwaiter().GetResult();
}
catch (Exception e)
{
if (e is FabricTransientException)
{
Console.WriteLine("A transient exception happened: '{0}'", e);
}
else if (e is TimeoutException)
{
Console.WriteLine("A timeout exception happened: '{0}'", e);
}
else
{
throw;
}
Task.Delay(TimeSpan.FromSeconds(1.0)).GetAwaiter().GetResult();
continue;
}
continuationToken = report.ContinuationToken;
foreach (var chaosEvent in report.History)
{
if (eventSet.Add(chaosEvent))
{
Console.WriteLine(chaosEvent);
}
}
// When Chaos stops, a StoppedEvent is created.
// If a StoppedEvent is found, exit the loop.
var lastEvent = report.History.LastOrDefault();
if (lastEvent is StoppedEvent)
{
break;
}
Task.Delay(TimeSpan.FromSeconds(1.0)).GetAwaiter().GetResult();
}
}
}
public static async Task <ChaosReport> RunTest(int minsTorun)
{
string clientCertThumb = "87b906f84a251c015d44ea188e2eff322d1c16f8";
string serverCertThumb = "87b906f84a251c015d44ea188e2eff322d1c16f8";
string CommonName = "memoryleak";
string connection = "sf-memoryleak.eastus.cloudapp.azure.com:19000";
var xc = GetCredentials(clientCertThumb, serverCertThumb, CommonName);
using (var client = new FabricClient(xc, connection))
{
var startTimeUtc = DateTime.UtcNow;
var maxClusterStabilizationTimeout = TimeSpan.FromSeconds(30.0);
var timeToRun = TimeSpan.FromMinutes(minsTorun);
// The recommendation is to start with a value of 2 or 3 and to exercise caution while moving up.
var maxConcurrentFaults = 3;
var startContext = new Dictionary <string, string> {
{ "ReasonForStart", "Testing" }
};
// Time-separation (in seconds) between two consecutive iterations of Chaos. The larger the value, the
// lower the fault injection rate.
var waitTimeBetweenIterations = TimeSpan.FromSeconds(1);
// Wait time (in seconds) between consecutive faults within a single iteration.
// The larger the value, the lower the overlapping between faults and the simpler the sequence of
// state transitions that the cluster goes through.
var waitTimeBetweenFaults = TimeSpan.FromSeconds(1);
// Passed-in cluster health policy is used to validate health of the cluster in between Chaos iterations.
var clusterHealthPolicy = new ClusterHealthPolicy
{
ConsiderWarningAsError = false,
MaxPercentUnhealthyApplications = 100,
MaxPercentUnhealthyNodes = 100
};
var nodetypeInclusionList = new List <string> {
"nt2vm", "nt3vm"
};
var applicationInclusionList = new List <string> {
"fabric:/RequestHandling"
};
// List of cluster entities to target for Chaos faults.
var chaosTargetFilter = new ChaosTargetFilter
{
NodeTypeInclusionList = nodetypeInclusionList,
//ApplicationInclusionList = applicationInclusionList,
};
var parameters = new ChaosParameters(
maxClusterStabilizationTimeout,
maxConcurrentFaults,
true, /* EnableMoveReplicaFault */
timeToRun,
startContext,
waitTimeBetweenIterations,
waitTimeBetweenFaults,
clusterHealthPolicy)
{
ChaosTargetFilter = chaosTargetFilter
};
try
{
await client.TestManager.StartChaosAsync(parameters);
}
catch (FabricChaosAlreadyRunningException)
{
Console.WriteLine("An instance of Chaos is already running in the cluster.");
await client.TestManager.StopChaosAsync();
throw new Exception("Chaos test already running");
}
var filter = new ChaosReportFilter(startTimeUtc, DateTime.MaxValue);
var eventSet = new HashSet <ChaosEvent>(new ChaosEventComparer());
string continuationToken = null;
while (true)
{
ChaosReport report;
try
{
report = string.IsNullOrEmpty(continuationToken)
? await client.TestManager.GetChaosReportAsync(filter)
: await client.TestManager.GetChaosReportAsync(continuationToken);
}
catch (Exception e)
{
if (e is FabricTransientException)
{
Console.WriteLine("A transient exception happened: '{0}'", e);
}
else if (e is TimeoutException)
{
Console.WriteLine("A timeout exception happened: '{0}'", e);
}
else
{
throw;
}
Task.Delay(TimeSpan.FromSeconds(1.0)).GetAwaiter().GetResult();
continue;
}
continuationToken = report.ContinuationToken;
foreach (var chaosEvent in report.History)
{
eventSet.Add(chaosEvent);
}
// When Chaos stops, a StoppedEvent is created.
// If a StoppedEvent is found, exit the loop.
var lastEvent = report.History.LastOrDefault();
if (lastEvent is StoppedEvent)
{
return(report);
}
Task.Delay(TimeSpan.FromSeconds(1.0)).GetAwaiter().GetResult();
}
}
}
