private static void FindScheduleConflicts(ChaosSchedule schedule)
{
bool hasError = false;
var squashedSchedule = ChaosSchedulerUtil.GetSortedChaosScheduleItemsList(schedule);
var errorString = new StringBuilder();
foreach (DayOfWeek day in ChaosSchedule.AllDaysOfWeek)
{
var conflicts = ChaosSchedulerUtil.FindScheduleConflictsForDay(squashedSchedule[day]);
bool dayHasError = conflicts.Count > 0;
hasError = hasError || dayHasError;
if (dayHasError)
{
errorString.AppendLine(string.Format("{0} :\n{1}", Enum.GetName(typeof(DayOfWeek), day), ChaosSchedulerUtil.PrettyPrintScheduleConflicts(conflicts)));
}
}
if (hasError)
{
var ret = string.Format("{0}\n \n{1}", errorString.ToString(), StringResources.ChaosScheduler_ScheduleConflict);
if (ret.Length > Constants.ErrorMessageMaxCharLength)
{
var moreString = string.Format("...\n \n{0}\n{1}\n", StringResources.ChaosScheduler_ScheduleConflict, StringResources.ChaosScheduler_MoreScheduleConflicts);
ret = ret.Substring(0, Constants.ErrorMessageMaxCharLength - moreString.Length) + moreString;
}
throw new System.ArgumentException(ret);
}
}
private async Task SetScheduleInternalAsync(ChaosScheduleDescription scheduleDescription, CancellationToken cancellationToken)
{
// Must only be called when inside the semaphore
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "Enter SetScheduleInternalAsync");
var peak = this.PeakMoveState(Command.SetSchedule);
if (peak.Equals(SchedulerState.NoChaosSchedulePending))
{
await ChaosSchedulerUtil.VerifyChaosScheduleAsync(scheduleDescription.Schedule, this.fabricClient, cancellationToken);
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "schedule verified");
this.scheduleDescription = scheduleDescription;
await this.WriteScheduleToReliableStoreAsync(scheduleDescription, cancellationToken).ConfigureAwait(false);
this.eventInstancesEnumerator = new ChaosScheduleEventInstancesEnumerator(this.scheduleDescription.Schedule, DateTime.UtcNow);
this.eventInstancesEnumerator.MoveNext();
await this.TryMoveStateAsync(Command.SetSchedule, cancellationToken).ConfigureAwait(false);
this.CheckStateAndThrowOnError(SchedulerState.NoChaosSchedulePending);
}
else if (peak.Equals(SchedulerState.ChaosScheduleActive))
{
TestabilityTrace.TraceSource.WriteWarning(TraceComponent, "Attempting to set schedule when chaos was running");
ChaosUtil.ThrowAlreadyRunning();
}
}
private static void FindMissingChaosParameterReferences(ChaosSchedule schedule)
{
bool hasError = false;
var squashedSchedule = ChaosSchedulerUtil.GetSortedChaosScheduleItemsList(schedule);
var errorStrings = new List <string>()
{
StringResources.ChaosScheduler_NonExistentChaosParameter
};
foreach (DayOfWeek day in ChaosSchedule.AllDaysOfWeek)
{
var dayErrors = ChaosSchedulerUtil.FindMissingChaosParameterReferencesForDay(schedule, squashedSchedule[day]);
var dayHasError = dayErrors.Count > 0;
hasError = hasError || dayHasError;
if (dayHasError)
{
errorStrings.Add(string.Format("{0}: {1}", Enum.GetName(typeof(DayOfWeek), day), JsonConvert.SerializeObject(dayErrors)));
}
}
if (hasError)
{
throw new System.ArgumentException(string.Join("\n", errorStrings).Replace(", [", ",\n["));
}
}
private static void VerifyChaosScheduleJob(ChaosScheduleJob job)
{
if (job.Days == null)
{
throw new System.ArgumentNullException("Days", StringResources.ChaosScheduler_ScheduleJobDaysIsNull);
}
if (job.Days.NoActiveDays())
{
throw new System.ArgumentException(StringResources.ChaosScheduler_ScheduleJobNoDaysSet, "Days");
}
if (job.Times == null)
{
throw new System.ArgumentNullException("Times", StringResources.ChaosScheduler_ScheduleJobTimesIsNull);
}
if (job.Times.Count < 1)
{
throw new System.ArgumentException(StringResources.ChaosScheduler_ScheduleJobEndTimeAfterEndOfDay, "Times");
}
foreach (var time in job.Times)
{
ChaosSchedulerUtil.VerifyChaosScheduleTimeRangeUtc(time);
}
}
/// <summary>
/// Recover by setting schedule to empty and status to stopped. No Chaos will be running.
/// </summary>
private async Task RecoverFromDefault(CancellationToken cancellationToken)
{
// Can't restart Chaos if it was running, make sure a stop event is registered.
await this.chaosMessageProcessor.RegisterStoppedEventForRestartFailureAsync().ConfigureAwait(false);
SchedulerState schedulerState = new SchedulerState(SchedulerState.NoChaosScheduleStopped);
ChaosScheduleDescription chaosScheduleDescription = new ChaosScheduleDescription();
await ChaosSchedulerUtil.VerifyChaosScheduleAsync(chaosScheduleDescription.Schedule, this.fabricClient, cancellationToken);
// Normally, the schedule and status should not be set directly but this is an exception as
// this is a setup step and we are only setting the state to the initial entry state
this.scheduleDescription = chaosScheduleDescription;
this.state = schedulerState;
await this.WriteScheduleToReliableStoreAsync(this.scheduleDescription, cancellationToken).ConfigureAwait(false);
await this.WriteStateToReliableStoreAsync(this.state, cancellationToken).ConfigureAwait(false);
}
private async Task SetScheduleAndTryResumeAsync(ChaosScheduleDescription scheduleDescription, CancellationToken cancellationToken)
{
// Must only be called when inside the semaphore
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "Enter SetScheduleAndTryResumeAsync");
await ChaosSchedulerUtil.VerifyChaosScheduleAsync(scheduleDescription.Schedule, this.fabricClient, cancellationToken);
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "schedule verified");
this.scheduleDescription = scheduleDescription;
await this.WriteScheduleToReliableStoreAsync(scheduleDescription, cancellationToken).ConfigureAwait(false);
this.eventInstancesEnumerator = new ChaosScheduleEventInstancesEnumerator(this.scheduleDescription.Schedule, DateTime.UtcNow);
this.eventInstancesEnumerator.MoveNext();
if (this.eventInstancesEnumerator.HasEvents())
{
while (this.eventInstancesEnumerator.Current.CompareTo(DateTime.UtcNow) == -1)
{
this.eventInstancesEnumerator.MoveNext();
}
}
await this.TryMoveStateAsync(Command.SetSchedule, cancellationToken).ConfigureAwait(false);
await this.TryMoveStateAsync(Command.MatureSchedule, cancellationToken).ConfigureAwait(false);
bool didResume = await this.chaosMessageProcessor.ResumeChaosAsync(cancellationToken);
if (didResume)
{
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "Chaos did resume");
// chaos was resumed and is running. We have lock protecting the state so we can move between these states without worrying about scheduler trying to take action based on states between these transitions.
await this.TryMoveStateAsync(Command.StartChaos, cancellationToken).ConfigureAwait(false);
}
else
{
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "Chaos did not resume");
}
}
/// <summary>
/// Attempt to recover from saved Chaos parameters and history of events. Will only recover if Chaos was running and needs to run.
/// </summary>
/// <returns>boolean representing if the recovery was successful.</returns>
private async Task <bool> TryRecoveryFromHistory(CancellationToken cancellationToken)
{
bool chaosWasRunning = await this.chaosMessageProcessor.StateManager.ChaosWasRunningAsync(this.partition, cancellationToken).ConfigureAwait(false);
if (!chaosWasRunning)
{
// If chaos was not running then there is no need to try recovering Chaos.
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "TryRecoveryFromHistory failed to recover. Chaos was not running.");
return(false);
}
var chaosParams = await this.chaosMessageProcessor.GetChaosParametersAsync(Guid.NewGuid()).ConfigureAwait(false);
var chaosStartTime = await this.chaosMessageProcessor.GetChaosStartTimeAsync(Guid.NewGuid()).ConfigureAwait(false);
ChaosSchedule schedule = ChaosSchedulerUtil.Create247Schedule(chaosParams, chaosStartTime, chaosStartTime + chaosParams.TimeToRun);
await this.SetScheduleAndTryResumeAsync(new ChaosScheduleDescription(0, schedule), cancellationToken).ConfigureAwait(false);
return(true);
}
private static void VerifyChaosScheduleTimeRangeUtc(ChaosScheduleTimeRangeUtc timerange)
{
if (timerange.StartTime == null)
{
throw new System.ArgumentNullException(
"StartTime",
StringResources.ChaosScheduler_ScheduleJobTimeIsNull);
}
if (timerange.EndTime == null)
{
throw new System.ArgumentNullException(
"EndTime",
StringResources.ChaosScheduler_ScheduleJobTimeIsNull);
}
ChaosSchedulerUtil.VerifyChaosScheduleTimeUtc(timerange.StartTime);
ChaosSchedulerUtil.VerifyChaosScheduleTimeUtc(timerange.EndTime);
if (timerange.StartTime > timerange.EndTime)
{
throw new System.ArgumentException(
string.Format(
StringResources.ChaosScheduler_ScheduleJobStartTimeAfterEndTime,
timerange.StartTime.ToString(),
timerange.EndTime.ToString()),
"StartTime");
}
if (timerange.StartTime < ChaosScheduleTimeUtc.StartOfDay)
{
throw new System.ArgumentException(
string.Format(
StringResources.ChaosScheduler_ScheduleJobStartTimeBeforeStartOfDay,
timerange.StartTime.ToString(),
ChaosScheduleTimeUtc.StartOfDay.ToString()),
"StartTime");
}
if (timerange.StartTime > ChaosScheduleTimeUtc.EndOfDay)
{
throw new System.ArgumentException(
string.Format(
StringResources.ChaosScheduler_ScheduleJobStartTimeAfterEndOfDay,
timerange.StartTime.ToString(),
ChaosScheduleTimeUtc.EndOfDay.ToString()),
"StartTime");
}
if (timerange.EndTime < ChaosScheduleTimeUtc.StartOfDay)
{
throw new System.ArgumentException(
string.Format(
StringResources.ChaosScheduler_ScheduleJobEndTimeBeforeStartOfDay,
timerange.EndTime.ToString(),
ChaosScheduleTimeUtc.StartOfDay.ToString()),
"EndTime");
}
if (timerange.EndTime > ChaosScheduleTimeUtc.EndOfDay)
{
throw new System.ArgumentException(
string.Format(
StringResources.ChaosScheduler_ScheduleJobEndTimeAfterEndOfDay,
timerange.EndTime.ToString(),
ChaosScheduleTimeUtc.EndOfDay.ToString()),
"EndTime");
}
}
public static async Task VerifyChaosScheduleAsync(
ChaosSchedule schedule,
FabricClient fabricClient,
CancellationToken cancellationToken)
{
if (schedule == null)
{
throw new System.ArgumentNullException("Schedule", StringResources.ChaosScheduler_ScheduleIsNull);
}
if (schedule.StartDate == null)
{
throw new System.ArgumentNullException("StartDate", StringResources.ChaosScheduler_ScheduleStartDateIsNull);
}
if (schedule.ExpiryDate == null)
{
throw new System.ArgumentNullException("ExpiryDate", StringResources.ChaosScheduler_ScheduleExpiryDateIsNull);
}
if (schedule.StartDate < ChaosConstants.FileTimeMinDateTime)
{
throw new System.ArgumentException(string.Format(StringResources.ChaosScheduler_ScheduleStartDateBeforeFileTimeEpoch, schedule.StartDate), "StartDate");
}
if (schedule.ExpiryDate < ChaosConstants.FileTimeMinDateTime)
{
throw new System.ArgumentException(string.Format(StringResources.ChaosScheduler_ScheduleExpiryDateBeforeFileTimeEpoch, schedule.ExpiryDate), "ExpiryDate");
}
if (schedule.ExpiryDate < schedule.StartDate)
{
throw new System.ArgumentException(string.Format(StringResources.ChaosScheduler_ScheduleExpiryDateBeforeStartDate, schedule.ExpiryDate, schedule.StartDate), "ExpiryDate");
}
if (schedule.ChaosParametersDictionary == null)
{
throw new System.ArgumentNullException("ChaosParametersDictionary", StringResources.ChaosScheduler_ScheduleParametersDictionaryIsNull);
}
foreach (var chaosParamDictionaryEntry in schedule.ChaosParametersDictionary)
{
await ChaosUtil.ValidateChaosTargetFilterAsync(
fabricClient,
chaosParamDictionaryEntry.Value,
new TimeSpan(0, 1, 0),
new TimeSpan(0, 1, 0),
cancellationToken).ConfigureAwait(false);
}
if (schedule.Jobs == null)
{
throw new System.ArgumentNullException("Jobs", StringResources.ChaosScheduler_ScheduleJobsIsNull);
}
// Validate each of the items before validating the combination of the items
foreach (var job in schedule.Jobs)
{
ChaosSchedulerUtil.VerifyChaosScheduleJob(job);
}
ChaosSchedulerUtil.FindMissingChaosParameterReferences(schedule);
ChaosSchedulerUtil.FindScheduleConflicts(schedule);
}
internal ChaosScheduleEventInstancesEnumerator(ChaosSchedule schedule, DateTime dayInWeek)
{
/*
* ChaosScheduleEventInstancesEnumerator that returns event instances based on the schedule definition.
* Schedules repeat on a weekly basis starting with Sunday as the first day of the week.
*
* In order to generate the event instances:
* 1. Determine the starting Sunday of the week for which dayInWeek is in. This is the starting datetime of that week's repetition cycle.
* 2. Convert the ChaosScheduleJobs of the schedule to a list of ChaosScheduleItems for each day of the week.
* 3. Turn each schedule item into an ChaosScheduleEventInstance by using the day of the week and time as an offset to the starting Sunday
* 4. Left join event instances if the start time of the future event is within one minute of the previous event's end time and they use the same ChaosParameters
* 5. If there are 2 or more event instances left, join the last item to the first item if the last item's endtime is within a minute of the first item's start time plus a week (next cycle start).
* 6. If there is 1 event instance left and it's end time is within one minute of it's start time plus 7 days (one cycle), then this event continues itself and
* should be just one long run until the schedule expires, so set the end time to the schedule's expiry date
*/
this.position = -1;
this.eventInstances = new List <ChaosScheduleEventInstance>();
while (dayInWeek.DayOfWeek != DayOfWeek.Sunday)
{
dayInWeek = dayInWeek.AddDays(-1);
}
// dayInWeek is now the start of Sunday of the week.
dayInWeek = new DateTime(dayInWeek.Year, dayInWeek.Month, dayInWeek.Day, 0, 0, 0, 0, DateTimeKind.Utc);
var squashedSchedule = ChaosSchedulerUtil.GetSortedChaosScheduleItemsList(schedule);
var fullEventInstances = new List <ChaosScheduleEventInstance>();
foreach (DayOfWeek dayOfWeek in new List <DayOfWeek>()
{
DayOfWeek.Sunday, DayOfWeek.Monday, DayOfWeek.Tuesday, DayOfWeek.Wednesday, DayOfWeek.Thursday, DayOfWeek.Friday, DayOfWeek.Saturday
})
{
foreach (ChaosScheduleItem scheduleEvent in squashedSchedule[dayOfWeek])
{
fullEventInstances.Add(
new ChaosScheduleEventInstance(
scheduleEvent.ChaosParameters,
schedule.ChaosParametersDictionary[scheduleEvent.ChaosParameters],
dayInWeek.Add(new TimeSpan(scheduleEvent.Time.StartTime.Hour, scheduleEvent.Time.StartTime.Minute, 0)),
dayInWeek.Add(new TimeSpan(scheduleEvent.Time.EndTime.Hour, scheduleEvent.Time.EndTime.Minute, 0))));
}
dayInWeek = dayInWeek.AddDays(1);
}
foreach (var eventInstance in fullEventInstances)
{
if (this.eventInstances.Count == 0)
{
this.eventInstances.Add(eventInstance);
}
else
{
var lastEventInstance = this.eventInstances[this.eventInstances.Count - 1];
if (eventInstance.Start - lastEventInstance.End <= new TimeSpan(0, 1, 0) && lastEventInstance.ChaosParametersReferenceName == eventInstance.ChaosParametersReferenceName)
{
lastEventInstance.End = eventInstance.End;
}
else
{
this.eventInstances.Add(eventInstance);
}
}
}
if (this.eventInstances.Count > 1)
{
if (this.eventInstances[0].Start.AddDays(7) - this.eventInstances[this.eventInstances.Count - 1].End <= new TimeSpan(0, 1, 0))
{
this.eventInstances[0].Start = this.eventInstances[this.eventInstances.Count - 1].Start.AddDays(-7);
this.eventInstances.RemoveAt(this.eventInstances.Count - 1);
}
}
if (this.eventInstances.Count == 1 && this.eventInstances[0].Start.AddDays(7) - this.eventInstances[0].End <= new TimeSpan(0, 1, 0))
{
this.eventInstances[0].End = new DateTime(schedule.ExpiryDate.Ticks, DateTimeKind.Utc);
}
}