public async Task <ChaosReport> GetChaosReportAsync(
GetChaosReportDescription getChaosReportDescription,
TimeSpan timeout,
CancellationToken cancellationToken)
{
Guid activityId = Guid.NewGuid();
this.ThrowIfNotReady(() => this.ChaosMessageProcessor != null);
TestabilityTrace.TraceSource.WriteNoise(TraceType, "{0}:Enter GetChaosReportAsync, description={1}", activityId, getChaosReportDescription.ToString());
ChaosReport report = null;
try
{
ChaosReportFilter reportFilter = null;
var continuationToken = getChaosReportDescription.ContinuationToken;
if (!string.IsNullOrEmpty(continuationToken))
{
reportFilter = await this.ChaosMessageProcessor.GetReportFilterAsync(activityId, continuationToken).ConfigureAwait(false);
}
else
{
var continuationTokenGuid = Guid.NewGuid().ToString();
var fileTimeUtcTicks = DateTime.UtcNow.ToFileTimeUtc();
continuationToken = string.Format(FASConstants.ChaosReportContinuationTokenFormat, fileTimeUtcTicks, continuationTokenGuid);
}
reportFilter = reportFilter ?? getChaosReportDescription.Filter;
if (!string.IsNullOrEmpty(getChaosReportDescription.ClientType) &&
(getChaosReportDescription.ClientType.Equals(ChaosConstants.RestClientTypeName, StringComparison.OrdinalIgnoreCase) ||
getChaosReportDescription.ClientType.Equals(ChaosConstants.NativeClientTypeName, StringComparison.OrdinalIgnoreCase)))
{
ChaosUtility.DisableOptimizationForValidationFailedEvent = true;
}
else
{
ChaosUtility.DisableOptimizationForValidationFailedEvent = false;
}
report = await this.ChaosMessageProcessor.ProcessGetChaosReportAsync(
activityId,
reportFilter,
continuationToken).ConfigureAwait(false);
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0}:GetChaosReportAsync - Exception occurred: {1}", activityId, e.Message);
FaultAnalysisServiceUtility.ThrowTransientExceptionIfRetryable(e);
throw;
}
TestabilityTrace.TraceSource.WriteNoise(TraceType, "{0}:GetChaosReportAsync - returning report.", activityId);
return(report);
}
public async Task <long> GetActionCountAsync(bool excludeRunningActions)
{
long count = 0;
try
{
using (var tx = this.stateManager.CreateTransaction())
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter transaction inside {0} - operationId={1}, operationId.GetHashCode={2}, tx.TranslationId={3}, tx.GetHashCode={4}", "GetActionCountAsync", Guid.Empty, 0, tx.TransactionId, tx.GetHashCode());
if (!excludeRunningActions)
{
count = await this.actionTable.GetCountAsync(tx).ConfigureAwait(false);
}
count += await this.historyTable.GetCountAsync(tx).ConfigureAwait(false);
await tx.CommitAsync().ConfigureAwait(false);
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter transaction inside {0} - operationId={1}, operationId.GetHashCode={2}", "GetActionCountAsync", Guid.Empty, 0);
}
catch (FabricNotPrimaryException fnp)
{
FaultAnalysisServiceUtility.TraceFabricNotPrimary(Guid.Empty, fnp);
throw;
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteError(TraceType, "GetActionCountAsync: {0}", e.ToString());
throw;
}
return(count);
}
public async Task <string> CallSystemService(
string action,
string inputBlob,
TimeSpan timeout,
CancellationToken cancellationToken)
{
this.ThrowIfNotReady();
string outputBlob = null;
try
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter CallSystemService for {0}.", action);
if (this.actionMapping.ContainsKey(action))
{
outputBlob = await this.actionMapping[action](inputBlob, timeout, cancellationToken).ConfigureAwait(false);
}
else
{
throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.E_NOTIMPL, "{0} is not implemented", action);
}
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "CallSystemService error: {0}", e);
FaultAnalysisServiceUtility.ThrowTransientExceptionIfRetryable(e);
throw;
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Exit CallSystemService for {0} with output {1}.", action, outputBlob);
return(outputBlob);
}
private static void ThrowIfRestartPartitionModeInvalid(RestartPartitionMode restartPartitionMode)
{
if (restartPartitionMode == RestartPartitionMode.Invalid)
{
throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.E_INVALIDARG, Strings.StringResources.Error_UnsupportedRestartPartitionMode);
}
}
public async Task <IEnumerable <ActionStateBase> > GetSelectedActionsAsync(TestCommandListDescription description)
{
TestCommandStateFilter stateFilter = description.TestCommandStateFilter;
TestCommandTypeFilter typeFilter = description.TestCommandTypeFilter;
TestabilityTrace.TraceSource.WriteInfo(TraceType, "GetSelectedActionsAsync() stateFilter={0}, typeFilter={1}", stateFilter.ToString(), typeFilter.ToString());
List <ActionStateBase> selectedActions = new List <ActionStateBase>();
try
{
await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
Guid.Empty,
() => this.GetSelectedActionsInnerAsync(selectedActions, stateFilter, typeFilter),
this.partition,
"Replace",
ActionStore.MaxRetries,
this.cancellationToken).ConfigureAwait(false);
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteError(TraceType, e.ToString());
throw;
}
return(selectedActions);
}
public async Task InvokeDataLossAsync(
InvokeDataLossDescription invokeDataLossDescription,
TimeSpan timeout,
CancellationToken cancellationToken)
{
this.ThrowIfNotReady();
TestabilityTrace.TraceSource.WriteInfo(
TraceType,
"{0} - Processing {1} PartitionSelector: {2}, DataLossMode: {3}",
invokeDataLossDescription.OperationId,
ActionType.InvokeDataLoss,
invokeDataLossDescription.PartitionSelector,
invokeDataLossDescription.DataLossMode);
try
{
await this.MessageProcessor.ProcessDataLossCommandAsync(
invokeDataLossDescription.OperationId,
invokeDataLossDescription.PartitionSelector,
invokeDataLossDescription.DataLossMode,
timeout,
null);
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Exception occurred: {1}", invokeDataLossDescription.OperationId, e.Message);
FaultAnalysisServiceUtility.ThrowTransientExceptionIfRetryable(e);
throw;
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - Intent saved for {1}", invokeDataLossDescription.OperationId, ActionType.InvokeDataLoss);
}
public Task InitializeNewActionAsync(ActionStateBase actionState, TimeSpan timeout)
{
return(FaultAnalysisServiceUtility.RetryOnTimeout(
actionState.OperationId,
() => this.InitializeNewActionInnerAsync(actionState),
timeout,
this.cancellationToken));
}
public Task <ActionStateBase> ProcessGetProgressAsync(Guid operationId, TimeSpan timeout, CancellationToken cancellationToken)
{
return(FaultAnalysisServiceUtility.RetryOnTimeout(
operationId,
() => this.actionStore.FindByOperationIdAsync(operationId, cancellationToken),
timeout,
cancellationToken));
}
private Task <RollbackState> CheckUserCancellationAndUpdateIfNeededAsync(ActionStateBase actionState, CancellationToken cancellationToken, string location)
{
return(FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
actionState.OperationId,
() => this.CheckUserCancellationAndUpdateIfNeededInnerAsync(actionState, cancellationToken, location),
this.partition,
"ReliableFaultsEngine.CheckUserCancellationAndUpdateIfNeededAsync",
3,
cancellationToken));
}
public async Task UpdateActionStateAsync(ActionStateBase actionState, bool wasCalledFromCancelApiPath)
{
await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
actionState.OperationId,
() => this.UpdateActionStateInnerAsync(actionState, wasCalledFromCancelApiPath),
this.partition,
"UpdateActionStateAsync",
ActionStore.MaxRetries,
this.cancellationToken).ConfigureAwait(false);
}
public async Task <PartitionDataLossProgress> GetInvokeDataLossProgressAsync(
Guid operationId,
TimeSpan timeout,
CancellationToken cancellationToken)
{
this.ThrowIfNotReady();
PartitionDataLossProgress progress = null;
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Inside GetInvokeDataLossProgressAsync, operationId = {0}", operationId);
try
{
ActionStateBase actionState = await this.MessageProcessor.ProcessGetProgressAsync(operationId, timeout, cancellationToken);
StepStateNames stateName = actionState.StateProgress.Peek();
TestCommandProgressState state = FaultAnalysisServiceUtility.ConvertState(actionState, TraceType);
InvokeDataLossState invokeDataLossState = actionState as InvokeDataLossState;
if (invokeDataLossState == null)
{
throw new InvalidCastException("State object could not be converted");
}
StepStateNames stepState = actionState.StateProgress.Peek();
var selectedPartition = new SelectedPartition
{
ServiceName = invokeDataLossState.Info.PartitionSelector.ServiceName,
PartitionId = invokeDataLossState.Info.PartitionId
};
PartitionDataLossResult result = new PartitionDataLossResult(selectedPartition, actionState.ErrorCausingRollback);
progress = new PartitionDataLossProgress(state, result);
TestabilityTrace.TraceSource.WriteInfo(
TraceType,
"{0} - {1} progress - {2}, Exception - {3}",
operationId,
ActionType.InvokeDataLoss,
progress.Result != null ? progress.Result.SelectedPartition.ToString() : FASConstants.UnavailableMessage,
(progress.Result != null && progress.Result.Exception != null) ? progress.Result.Exception.ToString() : FASConstants.UnavailableMessage);
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Caught {1}", operationId, e.ToString());
FaultAnalysisServiceUtility.ThrowTransientExceptionIfRetryable(e);
throw;
}
return(progress);
}
public async Task RunAsync(FabricClient fc, FabricTestAction action, ActionStateBase actionState, ServiceInternalFaultInfo serviceInternalFaultInfo, CancellationToken cancellationToken)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - Inside RunAsync of Engine, entering state machine", actionState.OperationId);
try
{
do
{
cancellationToken.ThrowIfCancellationRequested();
RollbackState readRollbackState = await this.CheckUserCancellationAndUpdateIfNeededAsync(actionState, cancellationToken, FASConstants.OuterLoop).ConfigureAwait(false);
// For the non-force case we need to cleanup, so that is why there's no break statement in that case.
if (readRollbackState == RollbackState.RollingBackForce)
{
actionState.StateProgress.Push(StepStateNames.Failed);
await this.actionStore.UpdateActionStateAsync(actionState).ConfigureAwait(false);
break;
}
await this.RunStateMachineAsync(fc, action, actionState, serviceInternalFaultInfo, cancellationToken).ConfigureAwait(false);
if (actionState.RollbackState == RollbackState.RollingBackAndWillRetryAction)
{
actionState.ErrorCausingRollback = 0;
int pauseTime = this.random.Next(10, 60);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - Pausing for {1} seconds before retrying", actionState.OperationId, pauseTime);
// Clear the rollback state so it will go forward when it resumes.
actionState.RollbackState = RollbackState.NotRollingBack;
await this.actionStore.UpdateActionStateAsync(actionState).ConfigureAwait(false);
await Task.Delay(TimeSpan.FromSeconds(pauseTime), cancellationToken).ConfigureAwait(false);
}
}while (actionState.StateProgress.Peek() != StepStateNames.CompletedSuccessfully &&
actionState.StateProgress.Peek() != StepStateNames.Failed);
}
catch (FabricNotPrimaryException notPrimary)
{
FaultAnalysisServiceUtility.TraceFabricNotPrimary(actionState.OperationId, notPrimary);
}
catch (FabricObjectClosedException objectClosed)
{
FaultAnalysisServiceUtility.TraceFabricObjectClosed(actionState.OperationId, objectClosed);
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} caught exception - {1}", actionState.OperationId, e);
throw;
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - Exiting state machine", actionState.OperationId);
}
private void ThrowIfNotReady(Func <bool> readinessCriteria = null)
{
if (this.MessageProcessor == null || (readinessCriteria != null && !readinessCriteria()))
{
throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.E_ABORT, "Service is not accepting client calls yet");
}
if (this.MessageProcessor.Partition.WriteStatus != PartitionAccessStatus.Granted)
{
// not primary, reconfiguration pending, and no write quorum all get translated to E_ABORT by the gateway, so there's no point in throwing a specific exception for each here.
throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.E_ABORT, string.Empty);
}
}
// This method should only run one at a time (controlled by timer)
public async Task TruncateCallbackInnerAsync()
{
bool observedException = false;
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter truncate callback");
try
{
long count = await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
Guid.Empty,
() => this.GetActionCountAsync(true),
this.partition,
"GetActionCountAsync",
ActionStore.MaxRetries,
this.cancellationToken).ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Action store size is {0}", count);
if (count > this.maxStoredActionCount)
{
await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
Guid.Empty,
() => this.TruncateAsync(count),
this.partition,
"TruncateCallbackInnerAsync",
ActionStore.MaxRetries,
this.cancellationToken).ConfigureAwait(false);
}
}
catch (FabricNotPrimaryException fnp)
{
FaultAnalysisServiceUtility.TraceFabricNotPrimary(Guid.Empty, fnp);
observedException = true;
}
catch (FabricObjectClosedException foc)
{
FaultAnalysisServiceUtility.TraceFabricObjectClosed(Guid.Empty, foc);
observedException = true;
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Error: {0}", e.ToString());
observedException = true;
throw;
}
if (!observedException)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Rescheduling timer for {0} seconds", this.storedActionCleanupIntervalInSeconds);
this.truncateTimer.Change(TimeSpan.FromSeconds(this.storedActionCleanupIntervalInSeconds), Timeout.InfiniteTimeSpan);
}
}
public static async Task <long> GetLastStartTimeInTicksAsync(
this IReliableDictionary <string, byte[]> startimeDictionary,
ITransaction tx,
IStatefulServicePartition partition,
CancellationToken cancellationToken)
{
return(await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
Guid.NewGuid(),
() => GetLastStartTimeInTicksPrivateAsync(startimeDictionary, tx, cancellationToken),
partition,
"GetLastStartTimeInTicksPrivateAsync",
FASConstants.MaxRetriesForReliableDictionary,
cancellationToken).ConfigureAwait(false));
}
public void Add(string nodeName)
{
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "Enter Add '{0}'", nodeName);
lock (this.NodeCommandLock)
{
if (this.nodeCommandInProgress.Contains(nodeName))
{
throw FaultAnalysisServiceUtility.CreateException(StepBase.TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_NODE_TRANSITION_IN_PROGRESS, Strings.StringResources.Error_NodeTransitionInProgress);
}
this.nodeCommandInProgress.Add(nodeName);
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "Done - Add '{0}'", nodeName);
}
}
public async Task <PartitionRestartProgress> GetRestartPartitionProgressAsync(
Guid operationId,
TimeSpan timeout,
CancellationToken cancellationToken)
{
this.ThrowIfNotReady();
PartitionRestartProgress progress = null;
try
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "GetRestartPartitionProgressAsync calling message processor");
ActionStateBase actionState = await this.MessageProcessor.ProcessGetProgressAsync(operationId, timeout, cancellationToken);
StepStateNames stateName = actionState.StateProgress.Peek();
TestCommandProgressState state = FaultAnalysisServiceUtility.ConvertState(actionState, TraceType);
RestartPartitionState restartPartitionState = actionState as RestartPartitionState;
TestabilityTrace.TraceSource.WriteInfo(
TraceType,
"RestartPartition - serviceName={0}, partitionId={1}",
restartPartitionState.Info.PartitionSelector.ServiceName.ToString(),
restartPartitionState.Info.PartitionId);
var selectedPartition = new SelectedPartition
{
ServiceName = restartPartitionState.Info.PartitionSelector.ServiceName,
PartitionId = restartPartitionState.Info.PartitionId
};
PartitionRestartResult result = new PartitionRestartResult(selectedPartition, actionState.ErrorCausingRollback);
progress = new PartitionRestartProgress(state, result);
TestabilityTrace.TraceSource.WriteInfo(
TraceType,
"{0} - {1} progress - {2}, Exception - {3}",
operationId,
ActionType.RestartPartition,
progress.Result != null ? progress.Result.SelectedPartition.ToString() : FASConstants.UnavailableMessage,
(progress.Result != null && progress.Result.Exception != null) ? progress.Result.Exception.ToString() : FASConstants.UnavailableMessage);
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Caught {1}", operationId, e.ToString());
FaultAnalysisServiceUtility.ThrowTransientExceptionIfRetryable(e);
throw;
}
return(progress);
}
public static async Task <ChaosStatus> GetCurrentStatusAsync(
this IReliableDictionary <string, byte[]> statusDictionary,
ITransaction tx,
IStatefulServicePartition partition,
CancellationToken cancellationToken)
{
return(await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
ChaosOperationID,
() => GetCurrentStatusPrivateAsync(statusDictionary, tx, cancellationToken),
partition,
"GetCurrentStatusPrivateAsync",
FASConstants.MaxRetriesForReliableDictionary,
cancellationToken).ConfigureAwait(false));
}
private async Task WriteScheduleToReliableStoreAsync(ChaosScheduleDescription scheduleDescription, CancellationToken cancellationToken)
{
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "Enter WriteScheduleToReliableStoreAsync with scheduleDescription={0}", scheduleDescription.ToString());
this.StatusDictionary = this.StatusDictionary ?? await this.StateManager.GetOrAddAsync <IReliableDictionary <string, byte[]> >(FASConstants.ChaosSchedulerStateName).ConfigureAwait(false);
await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
Guid.Empty,
() => this.WriteScheduleToReliableStoreHelperAsync(scheduleDescription, cancellationToken),
this.partition,
"WriteScheduleToReliableStoreHelperAsync",
FASConstants.MaxRetriesForReliableDictionary,
cancellationToken).ConfigureAwait(false);
}
public static async Task RegisterCurrentStatusAsync(
this IReliableStateManager stateManager,
ChaosStatus currentStatus,
IStatefulServicePartition partition,
CancellationToken cancellationToken)
{
await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
ChaosOperationID,
() => RegisterCurrentStatusPrivateAsync(stateManager, currentStatus, cancellationToken),
partition,
"RegisterCurrentStatusPrivateAsync",
FASConstants.MaxRetriesForReliableDictionary,
cancellationToken).ConfigureAwait(false);
}
public async Task <ActionStateBase> FindByOperationIdAsync(Guid operationId, CancellationToken cancellationToken)
{
ActionStateBase state = null;
try
{
using (var tx = this.stateManager.CreateTransaction())
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter transaction inside {0} - operationId={1}", "FindByOperationId", operationId);
var data = await this.actionTable.TryGetValueAsync(tx, operationId, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false);
if (data.HasValue)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "FindOperationId: {0}, actiontable has value", operationId);
state = this.ReadData(data.Value);
}
else
{
data = await this.historyTable.TryGetValueAsync(tx, operationId, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false);
if (data.HasValue)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "FindOperationId: {0}, history table has value", operationId);
state = this.ReadData(data.Value);
}
else
{
string error = string.Format(CultureInfo.InvariantCulture, "OperationId '{0}' was not found", operationId);
TestabilityTrace.TraceSource.WriteWarning(TraceType, error);
throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_KEY_NOT_FOUND, error);
}
}
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Exit transaction inside {0} - operationId={1}", "FindByOperationId", operationId);
}
catch (FabricNotPrimaryException fnp)
{
FaultAnalysisServiceUtility.TraceFabricNotPrimary(operationId, fnp);
throw;
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "FindOperationId: {0}", e.ToString());
throw;
}
return(state);
}
// Select all P, S, I that are not Down and not Dropped
private static List <StatefulServiceReplica> GetReplicasForFullDataLoss(List <StatefulServiceReplica> replicaList)
{
List <StatefulServiceReplica> targetReplicas = new List <StatefulServiceReplica>();
foreach (StatefulServiceReplica replica in replicaList)
{
if ((FaultAnalysisServiceUtility.IsPrimaryOrSecondary(replica) || (replica.ReplicaRole == ReplicaRole.IdleSecondary)) &&
FaultAnalysisServiceUtility.IsReplicaUp(replica))
{
targetReplicas.Add(replica);
}
}
return(targetReplicas);
}
public static async Task <DateTime> GetUtcTimeStampAsync(
this IReliableStateManager stateManager,
string timeStampRDName,
string timeStampKey,
IStatefulServicePartition partition,
CancellationToken cancellationToken)
{
return(await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
Guid.NewGuid(),
() => GetUtcTimeStampPrivateAsync(stateManager, timeStampRDName, timeStampKey, cancellationToken),
partition,
"GetTimestampPrivateAsync",
FASConstants.MaxRetriesForReliableDictionary,
cancellationToken).ConfigureAwait(false));
}
public static async Task <bool> ChaosWasRunningAsync(
this IReliableStateManager stateManager,
IStatefulServicePartition partition,
CancellationToken cancellationToken)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter ChaosWasRunningAsync");
return(await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
ChaosOperationID,
() => ChaosWasRunningAsyncPrivate(stateManager, cancellationToken),
partition,
"ShouldStopAsyncPrivate",
FASConstants.MaxRetriesForReliableDictionary,
cancellationToken));
}
public static async Task RemoveKeyFromReliableDictionaryAsync(
this IReliableStateManager stateManager,
string dictionaryName,
string key,
IStatefulServicePartition partition,
CancellationToken cancellationToken)
{
await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
Guid.NewGuid(),
() => RemoveKeyFromReliableDictionaryPrivateAsync(stateManager, dictionaryName, key, cancellationToken),
partition,
"RemoveKeyFromReliableDictionaryAsync",
FASConstants.MaxRetriesForReliableDictionary,
cancellationToken).ConfigureAwait(false);
}
public static async Task AddOrUpdateAsync <TKey>(
this IReliableDictionary <TKey, byte[]> dictionary,
IReliableStateManager stateManager,
TKey key,
byte[] value,
IStatefulServicePartition partition,
CancellationToken cancellationToken) where TKey : IComparable <TKey>, IEquatable <TKey>
{
await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
ChaosOperationID,
() => AddOrUpdatePrivateAsync(dictionary, stateManager, key, value, cancellationToken),
partition,
"GetCurrentStatusPrivateAsync",
FASConstants.MaxRetriesForReliableDictionary,
cancellationToken).ConfigureAwait(false);
}
public async Task CancelTestCommandAsync(
CancelTestCommandDescription description,
TimeSpan timeout,
CancellationToken cancellationToken)
{
this.ThrowIfNotReady();
try
{
await this.MessageProcessor.CancelTestCommandAsync(description.OperationId, description.Force).ConfigureAwait(false);
}
catch (Exception e)
{
FaultAnalysisServiceUtility.ThrowTransientExceptionIfRetryable(e);
throw;
}
}
private async Task WriteStateToReliableStoreAsync(SchedulerState schedulerState, CancellationToken cancellationToken)
{
TestabilityTrace.TraceSource.WriteInfo(TraceComponent, "Enter WriteStateToReliableStoreAsync");
this.StatusDictionary = this.StatusDictionary ?? await this.StateManager.GetOrAddAsync <IReliableDictionary <string, byte[]> >(FASConstants.ChaosSchedulerStateName).ConfigureAwait(false);
using (ITransaction tx = this.StateManager.CreateTransaction())
{
await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
Guid.Empty,
() => this.StatusDictionary.SetAsync(tx, FASConstants.ChaosSchedulerStatusDictionaryStateKey, schedulerState.ToBytes()),
this.partition,
"WriteStateToReliableStoreAsync",
FASConstants.MaxRetriesForReliableDictionary,
cancellationToken).ConfigureAwait(false);
await tx.CommitAsync().ConfigureAwait(false);
}
}
public async Task StopChaosAsync(
TimeSpan timeout,
CancellationToken cancellationToken)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter StopChaosAsync, datetimeutc={0}", DateTime.UtcNow);
this.ThrowIfNotReady(() => this.ChaosMessageProcessor != null);
try
{
await this.ChaosMessageProcessor.ProcessStopChaosAsync(cancellationToken).ConfigureAwait(false);
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "StopChaosAsync - Exception occurred: {0}", e.Message);
FaultAnalysisServiceUtility.ThrowTransientExceptionIfRetryable(e);
throw;
}
}
public async Task <string> GetStoppedNodeListAsync(
TimeSpan timeout,
CancellationToken cancellationToken)
{
this.ThrowIfNotReady();
string result = null;
try
{
result = await this.MessageProcessor.ProcessGetStoppedNodeListAsync().ConfigureAwait(false);
}
catch (Exception e)
{
FaultAnalysisServiceUtility.ThrowTransientExceptionIfRetryable(e);
throw;
}
return(result);
}
