private async Task SetScheduleInternalAsync(ChaosScheduleDescription scheduleDescription, CancellationToken cancellationToken)
{
// Must only be called when inside the semaphore
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "Enter SetScheduleInternalAsync");
var peak = this.PeakMoveState(Command.SetSchedule);
if (peak.Equals(SchedulerState.NoChaosSchedulePending))
{
await ChaosSchedulerUtil.VerifyChaosScheduleAsync(scheduleDescription.Schedule, this.fabricClient, cancellationToken);
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "schedule verified");
this.scheduleDescription = scheduleDescription;
await this.WriteScheduleToReliableStoreAsync(scheduleDescription, cancellationToken).ConfigureAwait(false);
this.eventInstancesEnumerator = new ChaosScheduleEventInstancesEnumerator(this.scheduleDescription.Schedule, DateTime.UtcNow);
this.eventInstancesEnumerator.MoveNext();
await this.TryMoveStateAsync(Command.SetSchedule, cancellationToken).ConfigureAwait(false);
this.CheckStateAndThrowOnError(SchedulerState.NoChaosSchedulePending);
}
else if (peak.Equals(SchedulerState.ChaosScheduleActive))
{
TestabilityTrace.TraceSource.WriteWarning(TraceComponent, "Attempting to set schedule when chaos was running");
ChaosUtil.ThrowAlreadyRunning();
}
}
/// <inheritdoc />
public Task PostChaosScheduleAsync(
ChaosScheduleDescription chaosSchedule,
CancellationToken cancellationToken = default(CancellationToken))
{
chaosSchedule.ThrowIfNull(nameof(chaosSchedule));
var requestId = Guid.NewGuid().ToString();
var url = "Tools/Chaos/Schedule";
var queryParams = new List <string>();
// Append to queryParams if not null.
queryParams.Add("api-version=6.2");
url += "?" + string.Join("&", queryParams);
string content;
using (var sw = new StringWriter())
{
ChaosScheduleDescriptionConverter.Serialize(new JsonTextWriter(sw), chaosSchedule);
content = sw.ToString();
}
HttpRequestMessage RequestFunc()
{
var request = new HttpRequestMessage()
{
Method = HttpMethod.Post,
Content = new StringContent(content, Encoding.UTF8)
};
request.Content.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json; charset=utf-8");
return(request);
}
return(this.httpClient.SendAsync(RequestFunc, url, requestId, cancellationToken));
}
protected override void ProcessRecord()
{
var clusterConnection = this.GetClusterConnection();
try
{
ChaosScheduleDescription description = null;
description = clusterConnection.GetChaosScheduleAsync(
this.GetTimeout(),
this.GetCancellationToken()).Result;
this.WriteObject(this.FormatOutput(description));
}
catch (AggregateException aggregateException)
{
aggregateException.Handle((ae) =>
{
this.ThrowTerminatingError(
ae,
Constants.GetChaosScheduleCommandErrorId,
clusterConnection);
return(true);
});
}
}
internal static unsafe SetChaosScheduleDescription CreateFromNative(IntPtr nativeRaw)
{
NativeTypes.FABRIC_CHAOS_SERVICE_SCHEDULE_DESCRIPTION native = *(NativeTypes.FABRIC_CHAOS_SERVICE_SCHEDULE_DESCRIPTION *)nativeRaw;
ChaosScheduleDescription chaosScheduleDescription = ChaosScheduleDescription.FromNative(native.ChaosScheduleDescription);
return(new SetChaosScheduleDescription(chaosScheduleDescription));
}
private async Task WriteScheduleToReliableStoreHelperAsync(ChaosScheduleDescription scheduleDescription, CancellationToken cancellationToken)
{
using (ITransaction tx = this.StateManager.CreateTransaction())
{
await this.StatusDictionary.SetAsync(tx, FASConstants.ChaosSchedulerStatusDictionaryScheduleKey, scheduleDescription.ToBytes()).ConfigureAwait(false);
await tx.CommitAsync();
}
}
internal ChaosScheduler(IReliableStateManager stateManager, IStatefulServicePartition partition)
{
this.StateManager = stateManager;
this.partition = partition;
this.stateSemaphore = new SemaphoreSlim(1, 1);
this.state = new SchedulerState(SchedulerState.NoChaosScheduleStopped);
this.scheduleDescription = new ChaosScheduleDescription();
}
internal static unsafe ChaosScheduleDescription CreateFromNative(NativeClient.IFabricChaosScheduleDescriptionResult nativeResult)
{
if (nativeResult == null)
{
return(null);
}
var chaosScheduleDescription = ChaosScheduleDescription.FromNative(nativeResult.get_ChaosScheduleDescriptionResult());
GC.KeepAlive(nativeResult);
return(chaosScheduleDescription);
}
private async Task WriteScheduleToReliableStoreAsync(ChaosScheduleDescription scheduleDescription, CancellationToken cancellationToken)
{
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "Enter WriteScheduleToReliableStoreAsync with scheduleDescription={0}", scheduleDescription.ToString());
this.StatusDictionary = this.StatusDictionary ?? await this.StateManager.GetOrAddAsync <IReliableDictionary <string, byte[]> >(FASConstants.ChaosSchedulerStateName).ConfigureAwait(false);
await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
Guid.Empty,
() => this.WriteScheduleToReliableStoreHelperAsync(scheduleDescription, cancellationToken),
this.partition,
"WriteScheduleToReliableStoreHelperAsync",
FASConstants.MaxRetriesForReliableDictionary,
cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Serializes the object to JSON.
/// </summary>
/// <param name="writer">The <see cref="T: Newtonsoft.Json.JsonWriter" /> to write to.</param>
/// <param name="obj">The object to serialize to JSON.</param>
internal static void Serialize(JsonWriter writer, ChaosScheduleDescription obj)
{
// Required properties are always serialized, optional properties are serialized when not null.
writer.WriteStartObject();
if (obj.Version != null)
{
writer.WriteProperty(obj.Version, "Version", JsonWriterExtensions.WriteIntValue);
}
if (obj.Schedule != null)
{
writer.WriteProperty(obj.Schedule, "Schedule", ChaosScheduleConverter.Serialize);
}
writer.WriteEndObject();
}
/// <inheritdoc/>
protected override void ProcessRecordInternal()
{
var chaosSchedule = new ChaosSchedule(
startDate: this.StartDate,
expiryDate: this.ExpiryDate,
chaosParametersDictionary: this.ChaosParametersDictionary,
jobs: this.Jobs);
var chaosScheduleDescription = new ChaosScheduleDescription(
version: this.Version,
schedule: chaosSchedule);
this.ServiceFabricClient.ChaosClient.PostChaosScheduleAsync(
chaosSchedule: chaosScheduleDescription,
serverTimeout: this.ServerTimeout,
cancellationToken: this.CancellationToken).GetAwaiter().GetResult();
Console.WriteLine("Success!");
}
/// <summary>
/// Recover by setting schedule to empty and status to stopped. No Chaos will be running.
/// </summary>
private async Task RecoverFromDefault(CancellationToken cancellationToken)
{
// Can't restart Chaos if it was running, make sure a stop event is registered.
await this.chaosMessageProcessor.RegisterStoppedEventForRestartFailureAsync().ConfigureAwait(false);
SchedulerState schedulerState = new SchedulerState(SchedulerState.NoChaosScheduleStopped);
ChaosScheduleDescription chaosScheduleDescription = new ChaosScheduleDescription();
await ChaosSchedulerUtil.VerifyChaosScheduleAsync(chaosScheduleDescription.Schedule, this.fabricClient, cancellationToken);
// Normally, the schedule and status should not be set directly but this is an exception as
// this is a setup step and we are only setting the state to the initial entry state
this.scheduleDescription = chaosScheduleDescription;
this.state = schedulerState;
await this.WriteScheduleToReliableStoreAsync(this.scheduleDescription, cancellationToken).ConfigureAwait(false);
await this.WriteStateToReliableStoreAsync(this.state, cancellationToken).ConfigureAwait(false);
}
private async Task SetScheduleAndTryResumeAsync(ChaosScheduleDescription scheduleDescription, CancellationToken cancellationToken)
{
// Must only be called when inside the semaphore
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "Enter SetScheduleAndTryResumeAsync");
await ChaosSchedulerUtil.VerifyChaosScheduleAsync(scheduleDescription.Schedule, this.fabricClient, cancellationToken);
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "schedule verified");
this.scheduleDescription = scheduleDescription;
await this.WriteScheduleToReliableStoreAsync(scheduleDescription, cancellationToken).ConfigureAwait(false);
this.eventInstancesEnumerator = new ChaosScheduleEventInstancesEnumerator(this.scheduleDescription.Schedule, DateTime.UtcNow);
this.eventInstancesEnumerator.MoveNext();
if (this.eventInstancesEnumerator.HasEvents())
{
while (this.eventInstancesEnumerator.Current.CompareTo(DateTime.UtcNow) == -1)
{
this.eventInstancesEnumerator.MoveNext();
}
}
await this.TryMoveStateAsync(Command.SetSchedule, cancellationToken).ConfigureAwait(false);
await this.TryMoveStateAsync(Command.MatureSchedule, cancellationToken).ConfigureAwait(false);
bool didResume = await this.chaosMessageProcessor.ResumeChaosAsync(cancellationToken);
if (didResume)
{
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "Chaos did resume");
// chaos was resumed and is running. We have lock protecting the state so we can move between these states without worrying about scheduler trying to take action based on states between these transitions.
await this.TryMoveStateAsync(Command.StartChaos, cancellationToken).ConfigureAwait(false);
}
else
{
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "Chaos did not resume");
}
}
internal async Task SetScheduleAsync(ChaosScheduleDescription scheduleDescription, bool useAutoVersion, CancellationToken cancellationToken)
{
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "Enter SetScheduleAsync with scheduleDescription={0} and useAutoVersion={1}", scheduleDescription.ToString(), useAutoVersion);
if (!this.stateSemaphore.Wait(ChaosConstants.SchedulerLockWaitMilliseconds, cancellationToken))
{
TestabilityTrace.TraceSource.WriteWarning(TraceComponent, "SetScheduleAsync - did not acquire lock in provisioned time. Not progressing.");
return;
}
try
{
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "stateSemaphore Acquired by SetScheduleAsync");
if (useAutoVersion || scheduleDescription.Version == this.scheduleDescription.Version)
{
var newScheduleDescription = new ChaosScheduleDescription(this.scheduleDescription.Version, scheduleDescription.Schedule);
newScheduleDescription.IncrementVersion();
await this.SetScheduleInternalAsync(newScheduleDescription, cancellationToken).ConfigureAwait(false);
}
else
{
throw new System.ArgumentException(
string.Format(
StringResources.ChaosScheduler_ScheduleVersionMismatch,
this.scheduleDescription.Version,
scheduleDescription.Version),
"ChaosScheduleDescription.Version");
}
}
finally
{
this.stateSemaphore.Release();
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "stateSemaphore Released by SetScheduleAsync");
}
}
public SetChaosScheduleDescription(
ChaosScheduleDescription chaosScheduleDescription)
{
Requires.Argument <ChaosScheduleDescription>("chaosScheduleDescription", chaosScheduleDescription).NotNull();
this.ChaosScheduleDescription = chaosScheduleDescription;
}
public static string SerializeChaosScheduleDescription(ChaosScheduleDescription description)
{
return(JsonConvert.SerializeObject(
description,
chaosScheduleDescriptionSerializationSettings));
}
/// <summary>
/// Attempt to recover from status of the Chaos schedule and Chaos scheduler status from RD. Chaos will be running if it should be running.
/// </summary>
/// <returns>boolean representing if the recovery was successful.</returns>
private async Task <bool> TryRecoveryFromSchedule(CancellationToken cancellationToken)
{
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "TryRecoveryFromSchedule entered.");
SchedulerState schedulerState = new SchedulerState(SchedulerState.NoChaosScheduleStopped);
ChaosScheduleDescription chaosScheduleDescription = new ChaosScheduleDescription();
this.StatusDictionary = this.StatusDictionary ?? await this.StateManager.GetOrAddAsync <IReliableDictionary <string, byte[]> >(FASConstants.ChaosSchedulerStateName).ConfigureAwait(false);
using (ITransaction tx = this.StateManager.CreateTransaction())
{
var schedulerResult = await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure <ConditionalValue <byte[]> >(
Guid.Empty,
() => this.StatusDictionary.TryGetValueAsync(tx, FASConstants.ChaosSchedulerStatusDictionaryScheduleKey),
this.partition,
"RestartRecoveryAsync",
FASConstants.MaxRetriesForReliableDictionary,
cancellationToken).ConfigureAwait(false);
var schedulerStateResult = await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure <ConditionalValue <byte[]> >(
Guid.Empty,
() => this.StatusDictionary.TryGetValueAsync(tx, FASConstants.ChaosSchedulerStatusDictionaryStateKey),
this.partition,
"RestartRecoveryAsync",
FASConstants.MaxRetriesForReliableDictionary,
cancellationToken).ConfigureAwait(false);
if (!schedulerResult.HasValue || !schedulerStateResult.HasValue)
{
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "TryRecoveryFromSchedule failed to recover. Scheduler state or schedule was non existent.");
return(false);
}
chaosScheduleDescription.FromBytes(schedulerResult.Value);
schedulerState.FromBytes(schedulerStateResult.Value);
await tx.CommitAsync().ConfigureAwait(false);
}
try
{
if (schedulerState.ScheduleStatus.Equals(ChaosScheduleStatus.Pending))
{
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "TryRecoveryFromSchedule scheduler state is pending");
await this.SetScheduleInternalAsync(chaosScheduleDescription, cancellationToken).ConfigureAwait(false);
}
else if (schedulerState.ScheduleStatus.Equals(ChaosScheduleStatus.Active))
{
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "TryRecoveryFromSchedule scheduler state is active");
await this.SetScheduleAndTryResumeAsync(chaosScheduleDescription, cancellationToken).ConfigureAwait(false);
}
// expire and stopped ChaosScheduleStatus will result in no action being taken
// the schedule and status is still correct in the RD because that is where the values were read from
}
catch (System.ArgumentException ex)
{
string exceptionMessage = string.Format("RestartRecoveryAsync - failed to recover chaos schedule. Reason {0}", ex.Message);
TestabilityTrace.TraceSource.WriteError(TraceComponent, exceptionMessage);
ChaosUtility.ThrowOrAssertIfTrue("ChaosScheduler::RestartRecoveryAsync", true, exceptionMessage);
}
return(true);
}
