internal ChaosScheduleEventInstance(string chaosParametersReferenceName, ChaosParameters chaosParameters, DateTime start, DateTime end)
{
this.ChaosParametersReferenceName = chaosParametersReferenceName;
this.ChaosParameters = chaosParameters;
this.Start = start;
this.End = end;
}
/// <inheritdoc/>
protected override void ProcessRecordInternal()
{
var clusterHealthPolicy = new ClusterHealthPolicy(
considerWarningAsError: this.ConsiderWarningAsError,
maxPercentUnhealthyNodes: this.MaxPercentUnhealthyNodes,
maxPercentUnhealthyApplications: this.MaxPercentUnhealthyApplications,
applicationTypeHealthPolicyMap: this.ApplicationTypeHealthPolicyMap);
var chaosContext = new ChaosContext(
map: this.Map?.ToDictionary <string, string>());
var chaosTargetFilter = new ChaosTargetFilter(
nodeTypeInclusionList: this.NodeTypeInclusionList,
applicationInclusionList: this.ApplicationInclusionList);
var chaosParameters = new ChaosParameters(
timeToRunInSeconds: this.TimeToRunInSeconds,
maxClusterStabilizationTimeoutInSeconds: this.MaxClusterStabilizationTimeoutInSeconds,
maxConcurrentFaults: this.MaxConcurrentFaults,
enableMoveReplicaFaults: this.EnableMoveReplicaFaults,
waitTimeBetweenFaultsInSeconds: this.WaitTimeBetweenFaultsInSeconds,
waitTimeBetweenIterationsInSeconds: this.WaitTimeBetweenIterationsInSeconds,
clusterHealthPolicy: clusterHealthPolicy,
context: chaosContext,
chaosTargetFilter: chaosTargetFilter);
this.ServiceFabricClient.ChaosClient.StartChaosAsync(
chaosParameters: chaosParameters,
serverTimeout: this.ServerTimeout,
cancellationToken: this.CancellationToken).GetAwaiter().GetResult();
Console.WriteLine("Success!");
}
public StartChaosRequest(IFabricClient fabricClient, ChaosParameters chaosTestScenarioParameters, TimeSpan timeout)
: base(fabricClient, timeout)
{
this.ChaosParameters = chaosTestScenarioParameters;
this.RetryErrorCodes.Add((uint)NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_RECONFIGURATION_PENDING);
}
/// <summary>
/// Create a schedule that runs chaos 24/7 using a particular ChaosParameters. This is useful for create schedules that simulate the old StartChaos
/// </summary>
/// <param name="chaosParameters">Parameteres to run chaos with.</param>
/// <param name="startTime">Time for schedule to become active</param>
/// <param name="endTime">Time for schedule to expire</param>
/// <returns>A 24/7 schedule that uses one set of ChaosParameters.</returns>
public static ChaosSchedule Create247Schedule(ChaosParameters chaosParameters, DateTime startTime, DateTime endTime)
{
var chaosParametersDictionary = new Dictionary <string, ChaosParameters>();
chaosParametersDictionary.Add(ChaosConstants.ChaosScheduler_DefaultParameterKey, chaosParameters);
List <ChaosScheduleTimeRangeUtc> times = new List <ChaosScheduleTimeRangeUtc>()
{
ChaosScheduleTimeRangeUtc.WholeDay
};
var everyDay = new HashSet <DayOfWeek>()
{
DayOfWeek.Sunday, DayOfWeek.Monday, DayOfWeek.Tuesday, DayOfWeek.Wednesday, DayOfWeek.Thursday, DayOfWeek.Friday, DayOfWeek.Saturday
};
ChaosScheduleJobActiveDays activeEveryday = new ChaosScheduleJobActiveDays(everyDay);
ChaosScheduleJob job = new ChaosScheduleJob(ChaosConstants.ChaosScheduler_DefaultParameterKey, activeEveryday, times);
var chaosScheduleJobs = new List <ChaosScheduleJob>()
{
job
};
ChaosSchedule schedule = new ChaosSchedule(startTime, endTime, chaosParametersDictionary, chaosScheduleJobs);
return(schedule);
}
public static void WriteChaosStartedTraceEvent(ChaosParameters chaosParameters)
{
Requires.Argument <ChaosParameters>("chaosParameters", chaosParameters).NotNull();
Guid eventInstanceId = Guid.NewGuid();
long maxConcurrentFaults = chaosParameters.MaxConcurrentFaults;
double timeToRunInSeconds = chaosParameters.TimeToRun.TotalSeconds;
double maxClusterStabilizationTimeoutInSeconds = chaosParameters.MaxClusterStabilizationTimeout.TotalSeconds;
double waitTimeBetweenIterationsInSeconds = chaosParameters.WaitTimeBetweenIterations.TotalSeconds;
double waitTimeBetweenFaultsInSeconds = chaosParameters.WaitTimeBetweenFaults.TotalSeconds;
bool enableMoveReplicaFaults = chaosParameters.EnableMoveReplicaFaults;
ChaosTargetFilter targetFilter = chaosParameters.ChaosTargetFilter;
IList <string> nodeTypeList = targetFilter != null ? targetFilter.NodeTypeInclusionList : null;
IList <string> applicationList = targetFilter != null ? targetFilter.ApplicationInclusionList : null;
string nodeTypeListAsString = string.Empty;
string applicationListAsString = string.Empty;
if (targetFilter != null && nodeTypeList != null)
{
nodeTypeListAsString = string.Join(ListEntryDelimeter, nodeTypeList);
}
if (targetFilter != null && applicationList != null)
{
applicationListAsString = string.Join(ListEntryDelimeter, applicationList);
}
var clusterHealthPolicy = chaosParameters.ClusterHealthPolicy;
string clusterHealthPolicyAsString = string.Empty;
if (clusterHealthPolicy != null)
{
clusterHealthPolicyAsString = clusterHealthPolicy.ToString();
}
var context = chaosParameters.Context;
string contextAsString = string.Empty;
if (context != null)
{
contextAsString = string.Join(ListEntryDelimeter, context);
}
FabricEvents.Events.ChaosStartedEvent(
eventInstanceId,
maxConcurrentFaults,
timeToRunInSeconds,
maxClusterStabilizationTimeoutInSeconds,
waitTimeBetweenIterationsInSeconds,
waitTimeBetweenFaultsInSeconds,
enableMoveReplicaFaults,
nodeTypeListAsString,
applicationListAsString,
clusterHealthPolicyAsString,
contextAsString);
}
internal static unsafe StartChaosDescription CreateFromNative(IntPtr nativeRaw)
{
NativeTypes.FABRIC_START_CHAOS_DESCRIPTION native = *(NativeTypes.FABRIC_START_CHAOS_DESCRIPTION *)nativeRaw;
ChaosParameters chaosTestScenarioParameters = ChaosParameters.CreateFromNative(native.ChaosParameters);
return(new StartChaosDescription(chaosTestScenarioParameters));
}
static async Task Chaos()
{
var clusterConnectionString = "localhost:19000";
using (var client = new FabricClient(clusterConnectionString))
{
var startTimeUtc = DateTime.UtcNow;
var stabilizationTimeout = TimeSpan.FromSeconds(10.00);
var timeToRun = TimeSpan.FromMinutes(3);
var maxConcurrentFaults = 3;
var parameters = new ChaosParameters(
stabilizationTimeout,
maxConcurrentFaults,
true, /* EnableMoveReplicaFault */
timeToRun);
try
{
await client.TestManager.StartChaosAsync(parameters);
}
catch (FabricChaosAlreadyRunningException)
{
await client.TestManager.StopChaosAsync();
await client.TestManager.StartChaosAsync(parameters);
}
var filter = new ChaosReportFilter(startTimeUtc, DateTime.MaxValue);
var eventSet = new HashSet <ChaosEvent>(new ChaosEventComparer());
while (true)
{
var report = client.TestManager.GetChaosReportAsync(filter).GetAwaiter().GetResult();
foreach (var chaosEvent in report.History)
{
if (eventSet.Add(chaosEvent))
{
Console.WriteLine(chaosEvent);
}
}
// When Chaos stops, a StoppedEvent is created.
// If a StoppedEvent is found, exit the loop.
var lastEvent = report.History.LastOrDefault();
if (lastEvent is StoppedEvent)
{
break;
}
await Task.Delay(TimeSpan.FromSeconds(1.0));
}
}
}
/// <summary>
/// Serializes the object to JSON.
/// </summary>
/// <param name="writer">The <see cref="T: Newtonsoft.Json.JsonWriter" /> to write to.</param>
/// <param name="obj">The object to serialize to JSON.</param>
internal static void Serialize(JsonWriter writer, ChaosParameters obj)
{
// Required properties are always serialized, optional properties are serialized when not null.
writer.WriteStartObject();
if (obj.TimeToRunInSeconds != null)
{
writer.WriteProperty(obj.TimeToRunInSeconds, "TimeToRunInSeconds", JsonWriterExtensions.WriteStringValue);
}
if (obj.MaxClusterStabilizationTimeoutInSeconds != null)
{
writer.WriteProperty(obj.MaxClusterStabilizationTimeoutInSeconds, "MaxClusterStabilizationTimeoutInSeconds", JsonWriterExtensions.WriteLongValue);
}
if (obj.MaxConcurrentFaults != null)
{
writer.WriteProperty(obj.MaxConcurrentFaults, "MaxConcurrentFaults", JsonWriterExtensions.WriteLongValue);
}
if (obj.EnableMoveReplicaFaults != null)
{
writer.WriteProperty(obj.EnableMoveReplicaFaults, "EnableMoveReplicaFaults", JsonWriterExtensions.WriteBoolValue);
}
if (obj.WaitTimeBetweenFaultsInSeconds != null)
{
writer.WriteProperty(obj.WaitTimeBetweenFaultsInSeconds, "WaitTimeBetweenFaultsInSeconds", JsonWriterExtensions.WriteLongValue);
}
if (obj.WaitTimeBetweenIterationsInSeconds != null)
{
writer.WriteProperty(obj.WaitTimeBetweenIterationsInSeconds, "WaitTimeBetweenIterationsInSeconds", JsonWriterExtensions.WriteLongValue);
}
if (obj.ClusterHealthPolicy != null)
{
writer.WriteProperty(obj.ClusterHealthPolicy, "ClusterHealthPolicy", ClusterHealthPolicyConverter.Serialize);
}
if (obj.Context != null)
{
writer.WriteProperty(obj.Context, "Context", ChaosContextConverter.Serialize);
}
if (obj.ChaosTargetFilter != null)
{
writer.WriteProperty(obj.ChaosTargetFilter, "ChaosTargetFilter", ChaosTargetFilterConverter.Serialize);
}
writer.WriteEndObject();
}
static void Main(string[] args)
{
var clusterConnectionString = "localhost:19000";
using (var client = new FabricClient(clusterConnectionString))
{
var startTimeUtc = DateTime.UtcNow;
var stabilizationTimeout = TimeSpan.FromSeconds(30.0);
var timeToRun = TimeSpan.FromMinutes(60.0);
var maxConcurrentFaults = 3;
var parameters = new ChaosParameters(
stabilizationTimeout,
maxConcurrentFaults,
true, /* EnableMoveReplicaFault */
timeToRun);
try
{
client.TestManager.StartChaosAsync(parameters).GetAwaiter().GetResult();
}
catch (FabricChaosAlreadyRunningException)
{
Console.WriteLine("An instance of Chaos is already running in the cluster.");
}
var filter = new ChaosReportFilter(startTimeUtc, DateTime.MaxValue);
var eventSet = new HashSet <ChaosEvent>(new ChaosEventComparer());
while (true)
{
var report = client.TestManager.GetChaosReportAsync(filter).GetAwaiter().GetResult();
foreach (var chaosEvent in report.History)
{
if (eventSet.Add(chaosEvent))
{
Console.WriteLine(chaosEvent);
}
}
var lastEvent = report.History.LastOrDefault();
if (lastEvent is StoppedEvent)
{
break;
}
Task.Delay(TimeSpan.FromSeconds(1.0)).GetAwaiter().GetResult();
}
}
}
public void ChaosParametersSerializationTest()
{
TimeSpan maxClusterStabilizationTimeout = TimeSpan.FromSeconds(997);
long maxConcurrentFaults = 7;
TimeSpan waitTimeBetweenIterations = TimeSpan.FromSeconds(131);
TimeSpan waitTimeBetweenFaults = TimeSpan.FromSeconds(19);
TimeSpan timeToRun = TimeSpan.FromSeconds(104729);
bool enableMoveReplicaFaults = true;
var healthPolicy = new ClusterHealthPolicy
{
ConsiderWarningAsError = false,
MaxPercentUnhealthyNodes = 10,
MaxPercentUnhealthyApplications = 15
};
healthPolicy.ApplicationTypeHealthPolicyMap["TestApplicationTypePolicy"] = 11;
var context = new Dictionary <string, string> {
{ "key1", "value1" }, { "key2", "value2" }
};
var chaosParameters = new ChaosParameters(
maxClusterStabilizationTimeout,
maxConcurrentFaults,
enableMoveReplicaFaults,
timeToRun,
context,
waitTimeBetweenIterations,
waitTimeBetweenFaults,
healthPolicy);
var chaosTargetFilter = new ChaosTargetFilter();
var nodeTypeInclustionList = new List <string> {
"NodeType1", "NodeType2"
};
var applicationInclusionList = new List <string> {
"fabric:/app1", "fabric:/app2"
};
chaosTargetFilter.ApplicationInclusionList = applicationInclusionList;
chaosTargetFilter.NodeTypeInclusionList = nodeTypeInclustionList;
this.TestUsingSerializer(this.Serializer, chaosParameters);
}
private ChaosParameters CreateChaosScenarioParameters()
{
var chaosParametersObject = new ChaosParameters(
this.MaxClusterStabilizationTimeoutSec.HasValue
? TimeSpan.FromSeconds(this.MaxClusterStabilizationTimeoutSec.Value)
: ChaosConstants.DefaultClusterStabilizationTimeout,
this.MaxConcurrentFaults ?? ChaosConstants.MaxConcurrentFaultsDefault,
this.EnableMoveReplicaFaults.IsPresent,
this.TimeToRunMinute.HasValue ? TimeSpan.FromMinutes(this.TimeToRunMinute.Value) : TimeSpan.MaxValue,
this.GetContextDictionary(this.Context),
this.WaitTimeBetweenIterationsSec.HasValue ? TimeSpan.FromSeconds(this.WaitTimeBetweenIterationsSec.Value) : ChaosConstants.WaitTimeBetweenIterationsDefault,
this.WaitTimeBetweenFaultsSec.HasValue ? TimeSpan.FromSeconds(this.WaitTimeBetweenFaultsSec.Value) : ChaosConstants.WaitTimeBetweenFaultsDefault,
this.ClusterHealthPolicy);
chaosParametersObject.ChaosTargetFilter = this.ChaosTargetFilter;
return(chaosParametersObject);
}
internal ChaosScenario(
FabricClient fabricClient,
ChaosParameters chaosParameters,
IReliableDictionary <long, byte[]> eventsDictionary,
IReliableDictionary <string, byte[]> lastEventKeyDict,
IReliableStateManager stateManager,
TimeSpan timeToRun,
double chaosSnapshotTelemetrySamplingProbability,
IStatefulServicePartition partition,
bool testMode,
CancellationToken cancellationToken)
{
this.actionTranslator = new ActionTranslator();
this.random = new Random((int)DateTime.Now.Ticks);
this.rootActionGenerator = new RootActionGenerator(chaosParameters.ActionGeneratorParameters, this.random);
this.generatedActions = new List <FabricTestAction>();
this.FabricClient = fabricClient;
this.TestContext = this.FabricClient.FaultManager.TestContext;
this.ChaosParameters = chaosParameters;
this.EventsDictionary = eventsDictionary;
this.LastEventKeyDict = lastEventKeyDict;
this.StateManager = stateManager;
this.consecutiveFailures = 0;
this.chaosSnapshotTelemetrySamplingProbability = chaosSnapshotTelemetrySamplingProbability;
this.partition = partition;
this.cancellationToken = cancellationToken;
if (testMode)
{
this.testMode = testMode;
chaosParameters.OperationTimeout = FASConstants.TestOperationTimeout;
}
this.validationHelper = new ValidationHelper(chaosParameters, fabricClient);
this.ApplyInternalConfigFromContextIfPresent(this.ChaosParameters.Context);
}
/// <inheritdoc />
public Task StartChaosAsync(
ChaosParameters chaosParameters,
long?serverTimeout = 60,
CancellationToken cancellationToken = default(CancellationToken))
{
chaosParameters.ThrowIfNull(nameof(chaosParameters));
serverTimeout?.ThrowIfOutOfInclusiveRange("serverTimeout", 1, 4294967295);
var requestId = Guid.NewGuid().ToString();
var url = "Tools/Chaos/$/Start";
var queryParams = new List <string>();
// Append to queryParams if not null.
serverTimeout?.AddToQueryParameters(queryParams, $"timeout={serverTimeout}");
queryParams.Add("api-version=6.0");
url += "?" + string.Join("&", queryParams);
string content;
using (var sw = new StringWriter())
{
ChaosParametersConverter.Serialize(new JsonTextWriter(sw), chaosParameters);
content = sw.ToString();
}
HttpRequestMessage RequestFunc()
{
var request = new HttpRequestMessage()
{
Method = HttpMethod.Post,
Content = new StringContent(content, Encoding.UTF8)
};
request.Content.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json; charset=utf-8");
return(request);
}
return(this.httpClient.SendAsync(RequestFunc, url, requestId, cancellationToken));
}
public static async Task <ChaosReport> RunTest(int minsTorun)
{
string clientCertThumb = "87b906f84a251c015d44ea188e2eff322d1c16f8";
string serverCertThumb = "87b906f84a251c015d44ea188e2eff322d1c16f8";
string CommonName = "memoryleak";
string connection = "sf-memoryleak.eastus.cloudapp.azure.com:19000";
var xc = GetCredentials(clientCertThumb, serverCertThumb, CommonName);
using (var client = new FabricClient(xc, connection))
{
var startTimeUtc = DateTime.UtcNow;
var maxClusterStabilizationTimeout = TimeSpan.FromSeconds(30.0);
var timeToRun = TimeSpan.FromMinutes(minsTorun);
// The recommendation is to start with a value of 2 or 3 and to exercise caution while moving up.
var maxConcurrentFaults = 3;
var startContext = new Dictionary <string, string> {
{ "ReasonForStart", "Testing" }
};
// Time-separation (in seconds) between two consecutive iterations of Chaos. The larger the value, the
// lower the fault injection rate.
var waitTimeBetweenIterations = TimeSpan.FromSeconds(1);
// Wait time (in seconds) between consecutive faults within a single iteration.
// The larger the value, the lower the overlapping between faults and the simpler the sequence of
// state transitions that the cluster goes through.
var waitTimeBetweenFaults = TimeSpan.FromSeconds(1);
// Passed-in cluster health policy is used to validate health of the cluster in between Chaos iterations.
var clusterHealthPolicy = new ClusterHealthPolicy
{
ConsiderWarningAsError = false,
MaxPercentUnhealthyApplications = 100,
MaxPercentUnhealthyNodes = 100
};
var nodetypeInclusionList = new List <string> {
"nt2vm", "nt3vm"
};
var applicationInclusionList = new List <string> {
"fabric:/RequestHandling"
};
// List of cluster entities to target for Chaos faults.
var chaosTargetFilter = new ChaosTargetFilter
{
NodeTypeInclusionList = nodetypeInclusionList,
//ApplicationInclusionList = applicationInclusionList,
};
var parameters = new ChaosParameters(
maxClusterStabilizationTimeout,
maxConcurrentFaults,
true, /* EnableMoveReplicaFault */
timeToRun,
startContext,
waitTimeBetweenIterations,
waitTimeBetweenFaults,
clusterHealthPolicy)
{
ChaosTargetFilter = chaosTargetFilter
};
try
{
await client.TestManager.StartChaosAsync(parameters);
}
catch (FabricChaosAlreadyRunningException)
{
Console.WriteLine("An instance of Chaos is already running in the cluster.");
await client.TestManager.StopChaosAsync();
throw new Exception("Chaos test already running");
}
var filter = new ChaosReportFilter(startTimeUtc, DateTime.MaxValue);
var eventSet = new HashSet <ChaosEvent>(new ChaosEventComparer());
string continuationToken = null;
while (true)
{
ChaosReport report;
try
{
report = string.IsNullOrEmpty(continuationToken)
? await client.TestManager.GetChaosReportAsync(filter)
: await client.TestManager.GetChaosReportAsync(continuationToken);
}
catch (Exception e)
{
if (e is FabricTransientException)
{
Console.WriteLine("A transient exception happened: '{0}'", e);
}
else if (e is TimeoutException)
{
Console.WriteLine("A timeout exception happened: '{0}'", e);
}
else
{
throw;
}
Task.Delay(TimeSpan.FromSeconds(1.0)).GetAwaiter().GetResult();
continue;
}
continuationToken = report.ContinuationToken;
foreach (var chaosEvent in report.History)
{
eventSet.Add(chaosEvent);
}
// When Chaos stops, a StoppedEvent is created.
// If a StoppedEvent is found, exit the loop.
var lastEvent = report.History.LastOrDefault();
if (lastEvent is StoppedEvent)
{
return(report);
}
Task.Delay(TimeSpan.FromSeconds(1.0)).GetAwaiter().GetResult();
}
}
}
static void Main(string[] args)
{
// README:
//
// Please ensure your cluster certificate is installed in
// the 'CurrentUser' certificate store.
//
// REQUIRED STEPS:
// - Paste your Service Fabric certificate's thumbprint below (line 52,53)
// - Update the cluster domain name to match your SF cluster (line 54)
// - Add your cluster node types to the inclusion list (line 102)
string clientCertThumb = "D6426E96E0169B60ED030E53FCD05EAC12AAA1E0";
string serverCertThumb = "D6426E96E0169B60ED030E53FCD05EAC12AAA1E0";
string clusterDomainName = "dotjson.westeurope.cloudapp.azure.com";
string commonName = $"www.{clusterDomainName}";
string clusterEndpoint = $"{clusterDomainName}:19000";
var creds = GetCredentials(clientCertThumb, serverCertThumb, commonName);
Console.WriteLine($"Connecting to cluster {clusterEndpoint} using certificate '{clientCertThumb}'.");
using (var client = new FabricClient(creds, clusterEndpoint))
{
var startTimeUtc = DateTime.UtcNow;
// The maximum amount of time to wait for all cluster entities to become stable and healthy.
// Chaos executes in iterations and at the start of each iteration it validates the health of cluster entities.
// During validation if a cluster entity is not stable and healthy within MaxClusterStabilizationTimeoutInSeconds, Chaos generates a validation failed event.
var maxClusterStabilizationTimeout = TimeSpan.FromSeconds(30.0);
var timeToRun = TimeSpan.FromMinutes(60.0);
// MaxConcurrentFaults is the maximum number of concurrent faults induced per iteration.
// Chaos executes in iterations and two consecutive iterations are separated by a validation phase.
// The higher the concurrency, the more aggressive the injection of faults -- inducing more complex series of states to uncover bugs.
// The recommendation is to start with a value of 2 or 3 and to exercise caution while moving up.
var maxConcurrentFaults = 3;
// Describes a map, which is a collection of (string, string) type key-value pairs. The map can be used to record information about
// the Chaos run. There cannot be more than 100 such pairs and each string (key or value) can be at most 4095 characters long.
// This map is set by the starter of the Chaos run to optionally store the context about the specific run.
var startContext = new Dictionary <string, string> {
{ "ReasonForStart", "Testing" }
};
// Time-separation (in seconds) between two consecutive iterations of Chaos. The larger the value, the lower the fault injection rate.
var waitTimeBetweenIterations = TimeSpan.FromSeconds(10);
// Wait time (in seconds) between consecutive faults within a single iteration.
// The larger the value, the lower the overlapping between faults and the simpler the sequence of state transitions that the cluster goes through.
// The recommendation is to start with a value between 1 and 5 and exercise caution while moving up.
var waitTimeBetweenFaults = TimeSpan.Zero;
// Passed-in cluster health policy is used to validate health of the cluster in between Chaos iterations.
var clusterHealthPolicy = new ClusterHealthPolicy
{
ConsiderWarningAsError = false,
MaxPercentUnhealthyApplications = 100,
MaxPercentUnhealthyNodes = 100
};
// All types of faults, restart node, restart code package, restart replica, move primary replica, and move secondary replica will happen
// for nodes of type 'FrontEndType'
var nodetypeInclusionList = new List <string> {
"nodetype0"
};
// In addition to the faults included by nodetypeInclusionList,
// restart code package, restart replica, move primary replica, move secondary replica faults will happen for 'fabric:/TestApp2'
// even if a replica or code package from 'fabric:/TestApp2' is residing on a node which is not of type included in nodeypeInclusionList.
var applicationInclusionList = new List <string> {
"fabric:/Exchange"
};
// List of cluster entities to target for Chaos faults.
var chaosTargetFilter = new ChaosTargetFilter
{
NodeTypeInclusionList = nodetypeInclusionList,
ApplicationInclusionList = applicationInclusionList
};
var parameters = new ChaosParameters(
maxClusterStabilizationTimeout,
maxConcurrentFaults,
true, /* EnableMoveReplicaFault */
timeToRun,
startContext,
waitTimeBetweenIterations,
waitTimeBetweenFaults,
clusterHealthPolicy)
{
ChaosTargetFilter = chaosTargetFilter
};
try
{
client.TestManager.StartChaosAsync(parameters).GetAwaiter().GetResult();
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(30)); // Allow enough time for Chaos engine to start
}
catch (FabricChaosAlreadyRunningException)
{
Console.WriteLine("An instance of Chaos is already running in the cluster");
}
var filter = new ChaosReportFilter(startTimeUtc, DateTime.MaxValue);
var eventSet = new HashSet <ChaosEvent>(new ChaosEventComparer());
string continuationToken = null;
while (true)
{
ChaosReport report;
try
{
report = string.IsNullOrEmpty(continuationToken)
? client.TestManager.GetChaosReportAsync(filter).GetAwaiter().GetResult()
: client.TestManager.GetChaosReportAsync(continuationToken).GetAwaiter().GetResult();
}
catch (Exception e)
{
if (e is FabricTransientException)
{
Console.WriteLine("A transient exception happened: '{0}'", e);
}
else if (e is TimeoutException)
{
Console.WriteLine("A timeout exception happened: '{0}'", e);
}
else
{
throw;
}
Task.Delay(TimeSpan.FromSeconds(1.0)).GetAwaiter().GetResult();
continue;
}
continuationToken = report.ContinuationToken;
foreach (var chaosEvent in report.History)
{
if (eventSet.Add(chaosEvent))
{
Console.WriteLine(chaosEvent);
}
}
// When Chaos stops, a StoppedEvent is created.
// If a StoppedEvent is found, exit the loop.
var lastEvent = report.History.LastOrDefault();
if (lastEvent is StoppedEvent)
{
break;
}
Task.Delay(TimeSpan.FromSeconds(1.0)).GetAwaiter().GetResult();
}
}
}
public static async Task ValidateChaosTargetFilterAsync(
FabricClient fabricClient,
ChaosParameters chaosParameters,
TimeSpan requestTimeout,
TimeSpan operationTimeout,
CancellationToken cancellationToken)
{
// If chaosParameters == null, there is no input to validate
// If chaosParameters.Context == null, there is no DisableInputValidationForChaosTargetFilterKey internal key, so whether the input is valid depends on ChaosTargetFilter
// If chaosParameters.Context != null, we need to check if DisableInputValidationForChaosTargetFilterKey internal key is there, if yes we do not validate input, if on the other had
// DisableInputValidationForChaosTargetFilterKey is not present then validation depends on ChaosTargetFilter. If ChaosTargetFilter is null, there is nothing to validate.
if (chaosParameters != null && chaosParameters.Context != null && !chaosParameters.Context.ContainsKey(ChaosConstants.DisableInputValidationForChaosTargetFilterKey) && chaosParameters.ChaosTargetFilter != null)
{
var includedNodeTypes = chaosParameters.ChaosTargetFilter.NodeTypeInclusionList;
if (includedNodeTypes != null && includedNodeTypes.Any())
{
var nodeList =
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => fabricClient.QueryManager.GetNodeListAsync(null, requestTimeout, cancellationToken), operationTimeout, cancellationToken).ConfigureAwait(false);
HashSet <string> invalidNodeTypeSet = new HashSet <string>(includedNodeTypes);
foreach (var node in nodeList)
{
if (invalidNodeTypeSet.Contains(node.NodeType))
{
invalidNodeTypeSet.Remove(node.NodeType);
}
}
if (invalidNodeTypeSet.Any())
{
var invalidNodeTypes = string.Join(",", invalidNodeTypeSet);
throw new ArgumentException(string.Format(StringResources.ChaosApiError_InvalidNodeType, invalidNodeTypes));
}
}
var includedApplications = chaosParameters.ChaosTargetFilter.ApplicationInclusionList;
if (includedApplications != null && includedApplications.Any())
{
var applicationList =
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => fabricClient.QueryManager.GetApplicationListAsync(null, requestTimeout, cancellationToken), operationTimeout, cancellationToken).ConfigureAwait(false);
var nonexistentApplications = new List <string>();
foreach (var app in includedApplications)
{
var foundApplication =
applicationList.FirstOrDefault(a => a.ApplicationName.OriginalString.Equals(app, StringComparison.Ordinal));
if (foundApplication == null)
{
nonexistentApplications.Add(app);
}
}
if (nonexistentApplications.Any())
{
throw new ArgumentException(string.Format(StringResources.ChaosApiError_InvalidApplicationName, string.Join(",", nonexistentApplications)));
}
}
}
}
private async Task TryStartChaosAsync(CancellationToken cancellationToken)
{
// Must only be called when inside the semaphore
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "Enter TryStartChaosAsync");
ChaosParameters parameters = new ChaosParameters();
var peak = this.PeakMoveState(Command.StartChaos);
if (peak.Equals(SchedulerState.ChaosScheduleActive))
{
if (!this.eventInstancesEnumerator.HasEvents())
{
// schedule is empty, no point trying to check for events.
return;
}
DateTime datetimeNow = DateTime.UtcNow;
// if current time is after current event, check next event
while (this.eventInstancesEnumerator.Current.CompareTo(datetimeNow) == -1)
{
this.eventInstancesEnumerator.MoveNext();
}
if (this.eventInstancesEnumerator.Current.Start <= datetimeNow &&
datetimeNow <= this.eventInstancesEnumerator.Current.End)
{
// start chaos
parameters = new ChaosParameters(this.eventInstancesEnumerator.Current.ChaosParameters);
if (datetimeNow.Add(this.eventInstancesEnumerator.Current.End - datetimeNow) >= this.scheduleDescription.Schedule.ExpiryDate)
{
// Give Chaos time to run that will not go past expiry of schedule.
parameters.TimeToRun = this.scheduleDescription.Schedule.ExpiryDate - datetimeNow;
}
else
{
parameters.TimeToRun = this.eventInstancesEnumerator.Current.End - datetimeNow;
}
if (parameters.TimeToRun.TotalSeconds < 1)
{
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "TryStartChaosAsync not starting Chaos because time to run Chaos is less than 1 second");
return;
}
try
{
await this.chaosMessageProcessor.ProcessStartChaosOldAsync(parameters, cancellationToken).ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "Chaos started.");
}
catch (Exception ex)
{
string exceptionMessage = string.Format("TryStartChaosAsync - Failed to start chaos. Reason {0}", ex.Message);
TestabilityTrace.TraceSource.WriteError(TraceComponent, exceptionMessage);
await this.chaosMessageProcessor.ProcessStopChaosOldAsync(true).ConfigureAwait(false);
await this.TryMoveStateAsync(Command.StopChaos, cancellationToken).ConfigureAwait(false);
ChaosUtility.ThrowOrAssertIfTrue("ChaosScheduler::TryStartChaosAsync", true, exceptionMessage);
return;
}
await this.TryMoveStateAsync(Command.StartChaos, cancellationToken).ConfigureAwait(false);
this.CheckStateAndThrowOnError(SchedulerState.ChaosScheduleActive);
}
//// else we are before the start of the next event, so we just simply wait until then and do nothing.
}
}
public StartChaosDescription(
ChaosParameters chaosTestScenarioParameters)
{
Requires.Argument <ChaosParameters>("chaosTestScenarioParameters", chaosTestScenarioParameters).NotNull();
this.ChaosParameters = chaosTestScenarioParameters;
}
public ValidationHelper(ChaosParameters chaosParameters, FabricClient fabricClient)
{
this.chaosParameters = chaosParameters;
this.fabricClient = fabricClient;
}
