public async Task <IEnumerable <ActionStateBase> > GetSelectedActionsAsync(TestCommandListDescription description)
{
TestCommandStateFilter stateFilter = description.TestCommandStateFilter;
TestCommandTypeFilter typeFilter = description.TestCommandTypeFilter;
TestabilityTrace.TraceSource.WriteInfo(TraceType, "GetSelectedActionsAsync() stateFilter={0}, typeFilter={1}", stateFilter.ToString(), typeFilter.ToString());
List <ActionStateBase> selectedActions = new List <ActionStateBase>();
try
{
await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
Guid.Empty,
() => this.GetSelectedActionsInnerAsync(selectedActions, stateFilter, typeFilter),
this.partition,
"Replace",
ActionStore.MaxRetries,
this.cancellationToken).ConfigureAwait(false);
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteError(TraceType, e.ToString());
throw;
}
return(selectedActions);
}
public async Task UpdateActionStateAsync(ActionStateBase actionState, bool wasCalledFromCancelApiPath)
{
await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
actionState.OperationId,
() => this.UpdateActionStateInnerAsync(actionState, wasCalledFromCancelApiPath),
this.partition,
"UpdateActionStateAsync",
ActionStore.MaxRetries,
this.cancellationToken).ConfigureAwait(false);
}
private Task <RollbackState> CheckUserCancellationAndUpdateIfNeededAsync(ActionStateBase actionState, CancellationToken cancellationToken, string location)
{
return(FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
actionState.OperationId,
() => this.CheckUserCancellationAndUpdateIfNeededInnerAsync(actionState, cancellationToken, location),
this.partition,
"ReliableFaultsEngine.CheckUserCancellationAndUpdateIfNeededAsync",
3,
cancellationToken));
}
// This method should only run one at a time (controlled by timer)
public async Task TruncateCallbackInnerAsync()
{
bool observedException = false;
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter truncate callback");
try
{
long count = await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
Guid.Empty,
() => this.GetActionCountAsync(true),
this.partition,
"GetActionCountAsync",
ActionStore.MaxRetries,
this.cancellationToken).ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Action store size is {0}", count);
if (count > this.maxStoredActionCount)
{
await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
Guid.Empty,
() => this.TruncateAsync(count),
this.partition,
"TruncateCallbackInnerAsync",
ActionStore.MaxRetries,
this.cancellationToken).ConfigureAwait(false);
}
}
catch (FabricNotPrimaryException fnp)
{
FaultAnalysisServiceUtility.TraceFabricNotPrimary(Guid.Empty, fnp);
observedException = true;
}
catch (FabricObjectClosedException foc)
{
FaultAnalysisServiceUtility.TraceFabricObjectClosed(Guid.Empty, foc);
observedException = true;
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Error: {0}", e.ToString());
observedException = true;
throw;
}
if (!observedException)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Rescheduling timer for {0} seconds", this.storedActionCleanupIntervalInSeconds);
this.truncateTimer.Change(TimeSpan.FromSeconds(this.storedActionCleanupIntervalInSeconds), Timeout.InfiniteTimeSpan);
}
}
public static async Task RegisterCurrentStatusAsync(
this IReliableStateManager stateManager,
ChaosStatus currentStatus,
IStatefulServicePartition partition,
CancellationToken cancellationToken)
{
await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
ChaosOperationID,
() => RegisterCurrentStatusPrivateAsync(stateManager, currentStatus, cancellationToken),
partition,
"RegisterCurrentStatusPrivateAsync",
FASConstants.MaxRetriesForReliableDictionary,
cancellationToken).ConfigureAwait(false);
}
public static async Task <long> GetLastStartTimeInTicksAsync(
this IReliableDictionary <string, byte[]> startimeDictionary,
ITransaction tx,
IStatefulServicePartition partition,
CancellationToken cancellationToken)
{
return(await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
Guid.NewGuid(),
() => GetLastStartTimeInTicksPrivateAsync(startimeDictionary, tx, cancellationToken),
partition,
"GetLastStartTimeInTicksPrivateAsync",
FASConstants.MaxRetriesForReliableDictionary,
cancellationToken).ConfigureAwait(false));
}
public static async Task <ChaosStatus> GetCurrentStatusAsync(
this IReliableDictionary <string, byte[]> statusDictionary,
ITransaction tx,
IStatefulServicePartition partition,
CancellationToken cancellationToken)
{
return(await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
ChaosOperationID,
() => GetCurrentStatusPrivateAsync(statusDictionary, tx, cancellationToken),
partition,
"GetCurrentStatusPrivateAsync",
FASConstants.MaxRetriesForReliableDictionary,
cancellationToken).ConfigureAwait(false));
}
private async Task WriteScheduleToReliableStoreAsync(ChaosScheduleDescription scheduleDescription, CancellationToken cancellationToken)
{
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "Enter WriteScheduleToReliableStoreAsync with scheduleDescription={0}", scheduleDescription.ToString());
this.StatusDictionary = this.StatusDictionary ?? await this.StateManager.GetOrAddAsync <IReliableDictionary <string, byte[]> >(FASConstants.ChaosSchedulerStateName).ConfigureAwait(false);
await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
Guid.Empty,
() => this.WriteScheduleToReliableStoreHelperAsync(scheduleDescription, cancellationToken),
this.partition,
"WriteScheduleToReliableStoreHelperAsync",
FASConstants.MaxRetriesForReliableDictionary,
cancellationToken).ConfigureAwait(false);
}
public static async Task <bool> ChaosWasRunningAsync(
this IReliableStateManager stateManager,
IStatefulServicePartition partition,
CancellationToken cancellationToken)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter ChaosWasRunningAsync");
return(await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
ChaosOperationID,
() => ChaosWasRunningAsyncPrivate(stateManager, cancellationToken),
partition,
"ShouldStopAsyncPrivate",
FASConstants.MaxRetriesForReliableDictionary,
cancellationToken));
}
public static async Task RemoveKeyFromReliableDictionaryAsync(
this IReliableStateManager stateManager,
string dictionaryName,
string key,
IStatefulServicePartition partition,
CancellationToken cancellationToken)
{
await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
Guid.NewGuid(),
() => RemoveKeyFromReliableDictionaryPrivateAsync(stateManager, dictionaryName, key, cancellationToken),
partition,
"RemoveKeyFromReliableDictionaryAsync",
FASConstants.MaxRetriesForReliableDictionary,
cancellationToken).ConfigureAwait(false);
}
public static async Task <DateTime> GetUtcTimeStampAsync(
this IReliableStateManager stateManager,
string timeStampRDName,
string timeStampKey,
IStatefulServicePartition partition,
CancellationToken cancellationToken)
{
return(await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
Guid.NewGuid(),
() => GetUtcTimeStampPrivateAsync(stateManager, timeStampRDName, timeStampKey, cancellationToken),
partition,
"GetTimestampPrivateAsync",
FASConstants.MaxRetriesForReliableDictionary,
cancellationToken).ConfigureAwait(false));
}
public static async Task AddOrUpdateAsync <TKey>(
this IReliableDictionary <TKey, byte[]> dictionary,
IReliableStateManager stateManager,
TKey key,
byte[] value,
IStatefulServicePartition partition,
CancellationToken cancellationToken) where TKey : IComparable <TKey>, IEquatable <TKey>
{
await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
ChaosOperationID,
() => AddOrUpdatePrivateAsync(dictionary, stateManager, key, value, cancellationToken),
partition,
"GetCurrentStatusPrivateAsync",
FASConstants.MaxRetriesForReliableDictionary,
cancellationToken).ConfigureAwait(false);
}
private async Task WriteStateToReliableStoreAsync(SchedulerState schedulerState, CancellationToken cancellationToken)
{
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "Enter WriteStateToReliableStoreAsync");
this.StatusDictionary = this.StatusDictionary ?? await this.StateManager.GetOrAddAsync <IReliableDictionary <string, byte[]> >(FASConstants.ChaosSchedulerStateName).ConfigureAwait(false);
using (ITransaction tx = this.StateManager.CreateTransaction())
{
await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
Guid.Empty,
() => this.StatusDictionary.SetAsync(tx, FASConstants.ChaosSchedulerStatusDictionaryStateKey, schedulerState.ToBytes()),
this.partition,
"WriteStateToReliableStoreAsync",
FASConstants.MaxRetriesForReliableDictionary,
cancellationToken).ConfigureAwait(false);
await tx.CommitAsync().ConfigureAwait(false);
}
}
/// <summary>
/// Attempt to recover from status of the Chaos schedule and Chaos scheduler status from RD. Chaos will be running if it should be running.
/// </summary>
/// <returns>boolean representing if the recovery was successful.</returns>
private async Task <bool> TryRecoveryFromSchedule(CancellationToken cancellationToken)
{
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "TryRecoveryFromSchedule entered.");
SchedulerState schedulerState = new SchedulerState(SchedulerState.NoChaosScheduleStopped);
ChaosScheduleDescription chaosScheduleDescription = new ChaosScheduleDescription();
this.StatusDictionary = this.StatusDictionary ?? await this.StateManager.GetOrAddAsync <IReliableDictionary <string, byte[]> >(FASConstants.ChaosSchedulerStateName).ConfigureAwait(false);
using (ITransaction tx = this.StateManager.CreateTransaction())
{
var schedulerResult = await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure <ConditionalValue <byte[]> >(
Guid.Empty,
() => this.StatusDictionary.TryGetValueAsync(tx, FASConstants.ChaosSchedulerStatusDictionaryScheduleKey),
this.partition,
"RestartRecoveryAsync",
FASConstants.MaxRetriesForReliableDictionary,
cancellationToken).ConfigureAwait(false);
var schedulerStateResult = await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure <ConditionalValue <byte[]> >(
Guid.Empty,
() => this.StatusDictionary.TryGetValueAsync(tx, FASConstants.ChaosSchedulerStatusDictionaryStateKey),
this.partition,
"RestartRecoveryAsync",
FASConstants.MaxRetriesForReliableDictionary,
cancellationToken).ConfigureAwait(false);
if (!schedulerResult.HasValue || !schedulerStateResult.HasValue)
{
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "TryRecoveryFromSchedule failed to recover. Scheduler state or schedule was non existent.");
return(false);
}
chaosScheduleDescription.FromBytes(schedulerResult.Value);
schedulerState.FromBytes(schedulerStateResult.Value);
await tx.CommitAsync().ConfigureAwait(false);
}
try
{
if (schedulerState.ScheduleStatus.Equals(ChaosScheduleStatus.Pending))
{
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "TryRecoveryFromSchedule scheduler state is pending");
await this.SetScheduleInternalAsync(chaosScheduleDescription, cancellationToken).ConfigureAwait(false);
}
else if (schedulerState.ScheduleStatus.Equals(ChaosScheduleStatus.Active))
{
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "TryRecoveryFromSchedule scheduler state is active");
await this.SetScheduleAndTryResumeAsync(chaosScheduleDescription, cancellationToken).ConfigureAwait(false);
}
// expire and stopped ChaosScheduleStatus will result in no action being taken
// the schedule and status is still correct in the RD because that is where the values were read from
}
catch (System.ArgumentException ex)
{
string exceptionMessage = string.Format("RestartRecoveryAsync - failed to recover chaos schedule. Reason {0}", ex.Message);
TestabilityTrace.TraceSource.WriteError(TraceComponent, exceptionMessage);
ChaosUtility.ThrowOrAssertIfTrue("ChaosScheduler::RestartRecoveryAsync", true, exceptionMessage);
}
return(true);
}
