private static void FindScheduleConflicts(ChaosSchedule schedule)
{
bool hasError = false;
var squashedSchedule = ChaosSchedulerUtil.GetSortedChaosScheduleItemsList(schedule);
var errorString = new StringBuilder();
foreach (DayOfWeek day in ChaosSchedule.AllDaysOfWeek)
{
var conflicts = ChaosSchedulerUtil.FindScheduleConflictsForDay(squashedSchedule[day]);
bool dayHasError = conflicts.Count > 0;
hasError = hasError || dayHasError;
if (dayHasError)
{
errorString.AppendLine(string.Format("{0} :\n{1}", Enum.GetName(typeof(DayOfWeek), day), ChaosSchedulerUtil.PrettyPrintScheduleConflicts(conflicts)));
}
}
if (hasError)
{
var ret = string.Format("{0}\n \n{1}", errorString.ToString(), StringResources.ChaosScheduler_ScheduleConflict);
if (ret.Length > Constants.ErrorMessageMaxCharLength)
{
var moreString = string.Format("...\n \n{0}\n{1}\n", StringResources.ChaosScheduler_ScheduleConflict, StringResources.ChaosScheduler_MoreScheduleConflicts);
ret = ret.Substring(0, Constants.ErrorMessageMaxCharLength - moreString.Length) + moreString;
}
throw new System.ArgumentException(ret);
}
}
private static void FindMissingChaosParameterReferences(ChaosSchedule schedule)
{
bool hasError = false;
var squashedSchedule = ChaosSchedulerUtil.GetSortedChaosScheduleItemsList(schedule);
var errorStrings = new List <string>()
{
StringResources.ChaosScheduler_NonExistentChaosParameter
};
foreach (DayOfWeek day in ChaosSchedule.AllDaysOfWeek)
{
var dayErrors = ChaosSchedulerUtil.FindMissingChaosParameterReferencesForDay(schedule, squashedSchedule[day]);
var dayHasError = dayErrors.Count > 0;
hasError = hasError || dayHasError;
if (dayHasError)
{
errorStrings.Add(string.Format("{0}: {1}", Enum.GetName(typeof(DayOfWeek), day), JsonConvert.SerializeObject(dayErrors)));
}
}
if (hasError)
{
throw new System.ArgumentException(string.Join("\n", errorStrings).Replace(", [", ",\n["));
}
}
/// <summary>
/// Create a schedule that runs chaos 24/7 using a particular ChaosParameters. This is useful for create schedules that simulate the old StartChaos
/// </summary>
/// <param name="chaosParameters">Parameteres to run chaos with.</param>
/// <param name="startTime">Time for schedule to become active</param>
/// <param name="endTime">Time for schedule to expire</param>
/// <returns>A 24/7 schedule that uses one set of ChaosParameters.</returns>
public static ChaosSchedule Create247Schedule(ChaosParameters chaosParameters, DateTime startTime, DateTime endTime)
{
var chaosParametersDictionary = new Dictionary <string, ChaosParameters>();
chaosParametersDictionary.Add(ChaosConstants.ChaosScheduler_DefaultParameterKey, chaosParameters);
List <ChaosScheduleTimeRangeUtc> times = new List <ChaosScheduleTimeRangeUtc>()
{
ChaosScheduleTimeRangeUtc.WholeDay
};
var everyDay = new HashSet <DayOfWeek>()
{
DayOfWeek.Sunday, DayOfWeek.Monday, DayOfWeek.Tuesday, DayOfWeek.Wednesday, DayOfWeek.Thursday, DayOfWeek.Friday, DayOfWeek.Saturday
};
ChaosScheduleJobActiveDays activeEveryday = new ChaosScheduleJobActiveDays(everyDay);
ChaosScheduleJob job = new ChaosScheduleJob(ChaosConstants.ChaosScheduler_DefaultParameterKey, activeEveryday, times);
var chaosScheduleJobs = new List <ChaosScheduleJob>()
{
job
};
ChaosSchedule schedule = new ChaosSchedule(startTime, endTime, chaosParametersDictionary, chaosScheduleJobs);
return(schedule);
}
/// <summary>
/// Serializes the object to JSON.
/// </summary>
/// <param name="writer">The <see cref="T: Newtonsoft.Json.JsonWriter" /> to write to.</param>
/// <param name="obj">The object to serialize to JSON.</param>
internal static void Serialize(JsonWriter writer, ChaosSchedule obj)
{
// Required properties are always serialized, optional properties are serialized when not null.
writer.WriteStartObject();
if (obj.StartDate != null)
{
writer.WriteProperty(obj.StartDate, "StartDate", JsonWriterExtensions.WriteDateTimeValue);
}
if (obj.ExpiryDate != null)
{
writer.WriteProperty(obj.ExpiryDate, "ExpiryDate", JsonWriterExtensions.WriteDateTimeValue);
}
if (obj.ChaosParametersDictionary != null)
{
writer.WriteEnumerableProperty(obj.ChaosParametersDictionary, "ChaosParametersDictionary", ChaosParametersDictionaryItemConverter.Serialize);
}
if (obj.Jobs != null)
{
writer.WriteEnumerableProperty(obj.Jobs, "Jobs", ChaosScheduleJobConverter.Serialize);
}
writer.WriteEndObject();
}
/// <inheritdoc/>
protected override void ProcessRecordInternal()
{
var chaosSchedule = new ChaosSchedule(
startDate: this.StartDate,
expiryDate: this.ExpiryDate,
chaosParametersDictionary: this.ChaosParametersDictionary,
jobs: this.Jobs);
var chaosScheduleDescription = new ChaosScheduleDescription(
version: this.Version,
schedule: chaosSchedule);
this.ServiceFabricClient.ChaosClient.PostChaosScheduleAsync(
chaosSchedule: chaosScheduleDescription,
serverTimeout: this.ServerTimeout,
cancellationToken: this.CancellationToken).GetAwaiter().GetResult();
Console.WriteLine("Success!");
}
/// <summary>
/// Attempt to recover from saved Chaos parameters and history of events. Will only recover if Chaos was running and needs to run.
/// </summary>
/// <returns>boolean representing if the recovery was successful.</returns>
private async Task <bool> TryRecoveryFromHistory(CancellationToken cancellationToken)
{
bool chaosWasRunning = await this.chaosMessageProcessor.StateManager.ChaosWasRunningAsync(this.partition, cancellationToken).ConfigureAwait(false);
if (!chaosWasRunning)
{
// If chaos was not running then there is no need to try recovering Chaos.
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "TryRecoveryFromHistory failed to recover. Chaos was not running.");
return(false);
}
var chaosParams = await this.chaosMessageProcessor.GetChaosParametersAsync(Guid.NewGuid()).ConfigureAwait(false);
var chaosStartTime = await this.chaosMessageProcessor.GetChaosStartTimeAsync(Guid.NewGuid()).ConfigureAwait(false);
ChaosSchedule schedule = ChaosSchedulerUtil.Create247Schedule(chaosParams, chaosStartTime, chaosStartTime + chaosParams.TimeToRun);
await this.SetScheduleAndTryResumeAsync(new ChaosScheduleDescription(0, schedule), cancellationToken).ConfigureAwait(false);
return(true);
}
public static Dictionary <DayOfWeek, List <ChaosScheduleItem> > GetSortedChaosScheduleItemsList(ChaosSchedule schedule)
{
var result = new Dictionary <DayOfWeek, List <ChaosScheduleItem> >();
foreach (DayOfWeek day in Enum.GetValues(typeof(DayOfWeek)))
{
result.Add(day, new List <ChaosScheduleItem>());
}
for (int jobIndex = 0; jobIndex < schedule.Jobs.Count; jobIndex++)
{
var job = schedule.Jobs[jobIndex];
foreach (DayOfWeek day in Enum.GetValues(typeof(DayOfWeek)).Cast <DayOfWeek>())
{
if (job.Days.GetDayValueByEnum(day))
{
foreach (var time in job.Times)
{
result[day].Add(new ChaosScheduleItem(jobIndex, job.ChaosParameters, time));
}
}
}
}
// for each Day, sort the items by starttime from earliest to latest.
foreach (DayOfWeek day in Enum.GetValues(typeof(DayOfWeek)))
{
result[day].Sort((a, b) => a.Time.StartTime.CompareTo(b.Time.StartTime));
}
return(result);
}
private static List <ChaosScheduleItem> FindMissingChaosParameterReferencesForDay(ChaosSchedule schedule, List <ChaosScheduleItem> dayItems)
{
var result = new List <ChaosScheduleItem>();
foreach (var scheduleItem in dayItems)
{
if (!schedule.ChaosParametersDictionary.ContainsKey(scheduleItem.ChaosParameters))
{
result.Add(scheduleItem);
}
}
return(result);
}
public static async Task VerifyChaosScheduleAsync(
ChaosSchedule schedule,
FabricClient fabricClient,
CancellationToken cancellationToken)
{
if (schedule == null)
{
throw new System.ArgumentNullException("Schedule", StringResources.ChaosScheduler_ScheduleIsNull);
}
if (schedule.StartDate == null)
{
throw new System.ArgumentNullException("StartDate", StringResources.ChaosScheduler_ScheduleStartDateIsNull);
}
if (schedule.ExpiryDate == null)
{
throw new System.ArgumentNullException("ExpiryDate", StringResources.ChaosScheduler_ScheduleExpiryDateIsNull);
}
if (schedule.StartDate < ChaosConstants.FileTimeMinDateTime)
{
throw new System.ArgumentException(string.Format(StringResources.ChaosScheduler_ScheduleStartDateBeforeFileTimeEpoch, schedule.StartDate), "StartDate");
}
if (schedule.ExpiryDate < ChaosConstants.FileTimeMinDateTime)
{
throw new System.ArgumentException(string.Format(StringResources.ChaosScheduler_ScheduleExpiryDateBeforeFileTimeEpoch, schedule.ExpiryDate), "ExpiryDate");
}
if (schedule.ExpiryDate < schedule.StartDate)
{
throw new System.ArgumentException(string.Format(StringResources.ChaosScheduler_ScheduleExpiryDateBeforeStartDate, schedule.ExpiryDate, schedule.StartDate), "ExpiryDate");
}
if (schedule.ChaosParametersDictionary == null)
{
throw new System.ArgumentNullException("ChaosParametersDictionary", StringResources.ChaosScheduler_ScheduleParametersDictionaryIsNull);
}
foreach (var chaosParamDictionaryEntry in schedule.ChaosParametersDictionary)
{
await ChaosUtil.ValidateChaosTargetFilterAsync(
fabricClient,
chaosParamDictionaryEntry.Value,
new TimeSpan(0, 1, 0),
new TimeSpan(0, 1, 0),
cancellationToken).ConfigureAwait(false);
}
if (schedule.Jobs == null)
{
throw new System.ArgumentNullException("Jobs", StringResources.ChaosScheduler_ScheduleJobsIsNull);
}
// Validate each of the items before validating the combination of the items
foreach (var job in schedule.Jobs)
{
ChaosSchedulerUtil.VerifyChaosScheduleJob(job);
}
ChaosSchedulerUtil.FindMissingChaosParameterReferences(schedule);
ChaosSchedulerUtil.FindScheduleConflicts(schedule);
}
internal ChaosScheduleEventInstancesEnumerator(ChaosSchedule schedule, DateTime dayInWeek)
{
/*
* ChaosScheduleEventInstancesEnumerator that returns event instances based on the schedule definition.
* Schedules repeat on a weekly basis starting with Sunday as the first day of the week.
*
* In order to generate the event instances:
* 1. Determine the starting Sunday of the week for which dayInWeek is in. This is the starting datetime of that week's repetition cycle.
* 2. Convert the ChaosScheduleJobs of the schedule to a list of ChaosScheduleItems for each day of the week.
* 3. Turn each schedule item into an ChaosScheduleEventInstance by using the day of the week and time as an offset to the starting Sunday
* 4. Left join event instances if the start time of the future event is within one minute of the previous event's end time and they use the same ChaosParameters
* 5. If there are 2 or more event instances left, join the last item to the first item if the last item's endtime is within a minute of the first item's start time plus a week (next cycle start).
* 6. If there is 1 event instance left and it's end time is within one minute of it's start time plus 7 days (one cycle), then this event continues itself and
* should be just one long run until the schedule expires, so set the end time to the schedule's expiry date
*/
this.position = -1;
this.eventInstances = new List <ChaosScheduleEventInstance>();
while (dayInWeek.DayOfWeek != DayOfWeek.Sunday)
{
dayInWeek = dayInWeek.AddDays(-1);
}
// dayInWeek is now the start of Sunday of the week.
dayInWeek = new DateTime(dayInWeek.Year, dayInWeek.Month, dayInWeek.Day, 0, 0, 0, 0, DateTimeKind.Utc);
var squashedSchedule = ChaosSchedulerUtil.GetSortedChaosScheduleItemsList(schedule);
var fullEventInstances = new List <ChaosScheduleEventInstance>();
foreach (DayOfWeek dayOfWeek in new List <DayOfWeek>()
{
DayOfWeek.Sunday, DayOfWeek.Monday, DayOfWeek.Tuesday, DayOfWeek.Wednesday, DayOfWeek.Thursday, DayOfWeek.Friday, DayOfWeek.Saturday
})
{
foreach (ChaosScheduleItem scheduleEvent in squashedSchedule[dayOfWeek])
{
fullEventInstances.Add(
new ChaosScheduleEventInstance(
scheduleEvent.ChaosParameters,
schedule.ChaosParametersDictionary[scheduleEvent.ChaosParameters],
dayInWeek.Add(new TimeSpan(scheduleEvent.Time.StartTime.Hour, scheduleEvent.Time.StartTime.Minute, 0)),
dayInWeek.Add(new TimeSpan(scheduleEvent.Time.EndTime.Hour, scheduleEvent.Time.EndTime.Minute, 0))));
}
dayInWeek = dayInWeek.AddDays(1);
}
foreach (var eventInstance in fullEventInstances)
{
if (this.eventInstances.Count == 0)
{
this.eventInstances.Add(eventInstance);
}
else
{
var lastEventInstance = this.eventInstances[this.eventInstances.Count - 1];
if (eventInstance.Start - lastEventInstance.End <= new TimeSpan(0, 1, 0) && lastEventInstance.ChaosParametersReferenceName == eventInstance.ChaosParametersReferenceName)
{
lastEventInstance.End = eventInstance.End;
}
else
{
this.eventInstances.Add(eventInstance);
}
}
}
if (this.eventInstances.Count > 1)
{
if (this.eventInstances[0].Start.AddDays(7) - this.eventInstances[this.eventInstances.Count - 1].End <= new TimeSpan(0, 1, 0))
{
this.eventInstances[0].Start = this.eventInstances[this.eventInstances.Count - 1].Start.AddDays(-7);
this.eventInstances.RemoveAt(this.eventInstances.Count - 1);
}
}
if (this.eventInstances.Count == 1 && this.eventInstances[0].Start.AddDays(7) - this.eventInstances[0].End <= new TimeSpan(0, 1, 0))
{
this.eventInstances[0].End = new DateTime(schedule.ExpiryDate.Ticks, DateTimeKind.Utc);
}
}
