private async Task SetScheduleInternalAsync(ChaosScheduleDescription scheduleDescription, CancellationToken cancellationToken)
{
// Must only be called when inside the semaphore
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "Enter SetScheduleInternalAsync");
var peak = this.PeakMoveState(Command.SetSchedule);
if (peak.Equals(SchedulerState.NoChaosSchedulePending))
{
await ChaosSchedulerUtil.VerifyChaosScheduleAsync(scheduleDescription.Schedule, this.fabricClient, cancellationToken);
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "schedule verified");
this.scheduleDescription = scheduleDescription;
await this.WriteScheduleToReliableStoreAsync(scheduleDescription, cancellationToken).ConfigureAwait(false);
this.eventInstancesEnumerator = new ChaosScheduleEventInstancesEnumerator(this.scheduleDescription.Schedule, DateTime.UtcNow);
this.eventInstancesEnumerator.MoveNext();
await this.TryMoveStateAsync(Command.SetSchedule, cancellationToken).ConfigureAwait(false);
this.CheckStateAndThrowOnError(SchedulerState.NoChaosSchedulePending);
}
else if (peak.Equals(SchedulerState.ChaosScheduleActive))
{
TestabilityTrace.TraceSource.WriteWarning(TraceComponent, "Attempting to set schedule when chaos was running");
ChaosUtil.ThrowAlreadyRunning();
}
}
/// <summary>
/// Recover by setting schedule to empty and status to stopped. No Chaos will be running.
/// </summary>
private async Task RecoverFromDefault(CancellationToken cancellationToken)
{
// Can't restart Chaos if it was running, make sure a stop event is registered.
await this.chaosMessageProcessor.RegisterStoppedEventForRestartFailureAsync().ConfigureAwait(false);
SchedulerState schedulerState = new SchedulerState(SchedulerState.NoChaosScheduleStopped);
ChaosScheduleDescription chaosScheduleDescription = new ChaosScheduleDescription();
await ChaosSchedulerUtil.VerifyChaosScheduleAsync(chaosScheduleDescription.Schedule, this.fabricClient, cancellationToken);
// Normally, the schedule and status should not be set directly but this is an exception as
// this is a setup step and we are only setting the state to the initial entry state
this.scheduleDescription = chaosScheduleDescription;
this.state = schedulerState;
await this.WriteScheduleToReliableStoreAsync(this.scheduleDescription, cancellationToken).ConfigureAwait(false);
await this.WriteStateToReliableStoreAsync(this.state, cancellationToken).ConfigureAwait(false);
}
private async Task SetScheduleAndTryResumeAsync(ChaosScheduleDescription scheduleDescription, CancellationToken cancellationToken)
{
// Must only be called when inside the semaphore
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "Enter SetScheduleAndTryResumeAsync");
await ChaosSchedulerUtil.VerifyChaosScheduleAsync(scheduleDescription.Schedule, this.fabricClient, cancellationToken);
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "schedule verified");
this.scheduleDescription = scheduleDescription;
await this.WriteScheduleToReliableStoreAsync(scheduleDescription, cancellationToken).ConfigureAwait(false);
this.eventInstancesEnumerator = new ChaosScheduleEventInstancesEnumerator(this.scheduleDescription.Schedule, DateTime.UtcNow);
this.eventInstancesEnumerator.MoveNext();
if (this.eventInstancesEnumerator.HasEvents())
{
while (this.eventInstancesEnumerator.Current.CompareTo(DateTime.UtcNow) == -1)
{
this.eventInstancesEnumerator.MoveNext();
}
}
await this.TryMoveStateAsync(Command.SetSchedule, cancellationToken).ConfigureAwait(false);
await this.TryMoveStateAsync(Command.MatureSchedule, cancellationToken).ConfigureAwait(false);
bool didResume = await this.chaosMessageProcessor.ResumeChaosAsync(cancellationToken);
if (didResume)
{
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "Chaos did resume");
// chaos was resumed and is running. We have lock protecting the state so we can move between these states without worrying about scheduler trying to take action based on states between these transitions.
await this.TryMoveStateAsync(Command.StartChaos, cancellationToken).ConfigureAwait(false);
}
else
{
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "Chaos did not resume");
}
}