internal unsafe void ToNative(PinCollection pin, ref NativeTypes.FABRIC_SERVICE_CORRELATION_DESCRIPTION description)
{
description.ServiceName = pin.AddObject(this.ServiceName);
switch (this.Scheme)
{
case ServiceCorrelationScheme.Affinity:
description.Scheme = NativeTypes.FABRIC_SERVICE_CORRELATION_SCHEME.FABRIC_SERVICE_CORRELATION_SCHEME_AFFINITY;
break;
case ServiceCorrelationScheme.AlignedAffinity:
description.Scheme = NativeTypes.FABRIC_SERVICE_CORRELATION_SCHEME.FABRIC_SERVICE_CORRELATION_SCHEME_ALIGNED_AFFINITY;
break;
case ServiceCorrelationScheme.NonAlignedAffinity:
description.Scheme = NativeTypes.FABRIC_SERVICE_CORRELATION_SCHEME.FABRIC_SERVICE_CORRELATION_SCHEME_NONALIGNED_AFFINITY;
break;
default:
ReleaseAssert.Failfast(string.Format(CultureInfo.CurrentCulture, StringResources.Error_InvalidServiceCorrelationScheme_Formatted, this.Scheme));
break;
}
}
internal static unsafe PartitionSchemeDescription CreateFromNative(NativeTypes.FABRIC_PARTITION_SCHEME scheme, IntPtr ptr)
{
ReleaseAssert.AssertIfNot(scheme != NativeTypes.FABRIC_PARTITION_SCHEME.FABRIC_PARTITION_SCHEME_INVALID, StringResources.Error_PartitionSchemeNotSupported);
switch (scheme)
{
case NativeTypes.FABRIC_PARTITION_SCHEME.FABRIC_PARTITION_SCHEME_SINGLETON:
return(SingletonPartitionSchemeDescription.CreateFromNative(ptr));
case NativeTypes.FABRIC_PARTITION_SCHEME.FABRIC_PARTITION_SCHEME_UNIFORM_INT64_RANGE:
return(UniformInt64RangePartitionSchemeDescription.CreateFromNative(ptr));
case NativeTypes.FABRIC_PARTITION_SCHEME.FABRIC_PARTITION_SCHEME_NAMED:
return(NamedPartitionSchemeDescription.CreateFromNative(ptr));
default:
AppTrace.TraceSource.WriteError("ServicePartitionDescription.CreateFromNative", "Unknown scheme: {0}", scheme);
ReleaseAssert.Failfast(string.Format(CultureInfo.CurrentCulture, StringResources.Error_PartitionSchemeUnknown_Formatted, scheme));
break;
}
return(null);
}
private async Task CancelStateProviderInitializationAsync()
{
if (this.stateProviderInitCts != null &&
this.stateProviderInitCts.IsCancellationRequested == false)
{
ActorTrace.Source.WriteInfoWithId(TraceType, this.traceId, "Canceling state provider initialization...");
this.stateProviderInitCts.Cancel();
try
{
await this.stateProviderInitTask;
}
catch (Exception ex)
{
// Code should never come here.
ReleaseAssert.Failfast(
"CancelStateProviderInitializationAsync() unexpected exception: {0}.",
ex.ToString());
}
finally
{
this.stateProviderInitCts = null;
this.stateProviderInitTask = null;
}
}
// Stop logical timer if it is running
if (this.isLogicalTimeManagerInitialized == true)
{
ActorTrace.Source.WriteInfoWithId(TraceType, this.traceId, "Stopping logical time manager...");
this.logicalTimeManager.Stop();
this.isLogicalTimeManagerInitialized = false;
}
}
internal override Task <BufferedClientStream> AcquireClientStreamAsync(CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
NamedPipeClientStream pipeClientStream = null;
BufferedClientStream clientStream = null;
try
{
pipeClientStream = new NamedPipeClientStream(this.serverName, this.pipeName);
pipeClientStream.Connect(ClientConnectionTimeoutMilliSeconds);
clientStream = new BufferedClientStream(pipeClientStream);
}
catch (ArgumentOutOfRangeException ex)
{
ReleaseAssert.Failfast($"Connecting to pipe encountered: {ex}.");
}
catch (InvalidOperationException ex)
{
ReleaseAssert.Failfast($"Connecting to pipe encountered: {ex}.");
}
catch (Exception ex)
{
if (pipeClientStream != null)
{
pipeClientStream.Dispose();
}
var errMsg = this.GetConnectionFailedErrorMessage(ex);
AppTrace.TraceSource.WriteWarning(TraceType, errMsg);
throw new FabricException(errMsg, FabricErrorCode.OperationTimedOut);
}
return(Task.FromResult(clientStream));
}
private static ContainerEventType EventTypeFromString(string eventTypeStr)
{
var eventTypeStrLower = eventTypeStr.ToLowerInvariant().Trim();
var eventType = ContainerEventType.None;
if (eventTypeStrLower == "stop")
{
eventType = ContainerEventType.Stop;
}
else if (eventTypeStrLower == "die")
{
eventType = ContainerEventType.Die;
}
else if (eventTypeStrLower.StartsWith("health_status"))
{
eventType = ContainerEventType.Health;
}
else
{
ReleaseAssert.Failfast("Unexpected container event received: {0}", eventTypeStr);
}
return(eventType);
}
public override async Task <ActionStateBase> RunAsync(CancellationToken cancellationToken, ServiceInternalFaultInfo serviceInternalFaultInfo)
{
NodeCommandState state = Convert(this.State);
// The return value is ignored, this is just being used to check if the RemoveNodeState was called.
Node queriedNode = await FaultAnalysisServiceUtility.GetNodeInfoAsync(
this.State.OperationId,
this.FabricClient,
state.Info.NodeName,
this.action.Partition,
this.action.StateManager,
this.action.StoppedNodeTable,
this.RequestTimeout,
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - calling StartNodeUsingNodeNameAsync, ApiInputNodeInstanceId={1}", this.State.OperationId, state.Info.InputNodeInstanceId);
Exception exception = null;
try
{
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.FaultManager.StartNodeUsingNodeNameAsync(
state.Info.NodeName,
state.Info.InputNodeInstanceId,
null,
0,
this.RequestTimeout,
cancellationToken),
FabricClientRetryErrors.StartNodeErrors.Value,
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(StepBase.TraceType, "{0} - StartNodeUsingNodeNameAsync threw {1}", this.State.OperationId, e);
exception = e;
}
cancellationToken.ThrowIfCancellationRequested();
SuccessRetryOrFail status = SuccessRetryOrFail.Invalid;
if (exception != null)
{
FabricException fe = exception as FabricException;
if (fe != null)
{
status = this.HandleFabricException(fe, state);
}
else
{
TestabilityTrace.TraceSource.WriteWarning(StepBase.TraceType, "{0} - StartNodeUsingNodeNameAsync threw non-FabricException with ErrorCode={1}", this.State.OperationId, exception);
status = SuccessRetryOrFail.RetryStep;
}
}
else
{
// success
status = SuccessRetryOrFail.Success;
await FaultAnalysisServiceUtility.SetStoppedNodeStateAsync(
this.action.State.OperationId,
this.action.Partition,
this.action.StateManager,
this.action.StoppedNodeTable,
state.Info.NodeName,
false,
cancellationToken).ConfigureAwait(false);
}
ActionTest.PerformInternalServiceFaultIfRequested(this.State.OperationId, serviceInternalFaultInfo, this.State, cancellationToken, true);
if (status == SuccessRetryOrFail.RetryStep)
{
throw new RetrySameStepException("retrystep", exception);
}
else if (status == SuccessRetryOrFail.Fail)
{
throw new FatalException("fatal", exception);
}
else if (status == SuccessRetryOrFail.Success)
{
// no-op
}
else
{
ReleaseAssert.Failfast(string.Format(CultureInfo.InvariantCulture, "This condition should not have been hit. OperationId: {0}", this.State.OperationId));
}
await this.ValidateAsync(this.FabricClient, state, cancellationToken).ConfigureAwait(false);
state.StateProgress.Push(StepStateNames.CompletedSuccessfully);
return(state);
}
private bool HandleRollback(Guid operationId, Exception exception, List <FabricErrorCode> expectedErrorCodes)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Enter HandleRollback", operationId);
if (exception == null)
{
string error = string.Format(CultureInfo.InvariantCulture, "{0} - exception is null", operationId);
ReleaseAssert.Failfast(error);
}
if (exception is FatalException)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - HandleRollback - FatalException", operationId);
return(false);
}
if (exception is RollbackAndRetryCommandException)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - HandleRollback - RollbackAndRetryCommandException", operationId);
return(true);
}
// Use for internal testing
if (exception is InvalidOperationException)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - HandleRollback - InvalidOperationException", operationId);
return(false);
}
if (exception is OperationCanceledException)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Retrying after OperationCanceledException", operationId);
return(true);
}
if (exception is FabricInvalidAddressException)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Retrying after {1}", operationId, exception);
return(true);
}
// Add to this list
if (exception is FabricException)
{
FabricException fabricException = exception as FabricException;
if ((fabricException.ErrorCode == FabricErrorCode.InvalidReplicaStateForReplicaOperation) || // eg restart of down replica
(fabricException.ErrorCode == FabricErrorCode.InvalidAddress) ||
(fabricException.ErrorCode == FabricErrorCode.NotReady) ||
(fabricException.ErrorCode == FabricErrorCode.OperationTimedOut) ||
(fabricException.ErrorCode == FabricErrorCode.ReplicaDoesNotExist))
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Retrying after {1}", operationId, exception);
return(true);
}
}
FabricException innerException = exception as FabricException;
if (innerException == null)
{
// malformed exception
string error = string.Format(CultureInfo.InvariantCulture, "{0} - Exception type was not expected: '{1}'", operationId, exception);
TestabilityTrace.TraceSource.WriteError(TraceType, "{0}", error);
return(false);
}
if (innerException is FabricTransientException)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Retrying after inner ex {1}", operationId, exception);
return(true);
}
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Bottom of HandleRollback, printing exception info", operationId);
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Exception: {1}", operationId, exception);
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Inner ex {1}", operationId, innerException);
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Inner ex error code {1}", operationId, innerException.ErrorCode);
// nonretryable
return(false);
}
private async Task RunStateMachineAsync(
FabricClient fabricClient,
FabricTestAction action,
ActionStateBase actionState,
ServiceInternalFaultInfo serviceInternalFaultInfo,
CancellationToken cancellationToken)
{
if (actionState.StateProgress == null || actionState.StateProgress.Count == 0)
{
ReleaseAssert.AssertIf(actionState.StateProgress == null || actionState.StateProgress.Count == 0, "ActionProgress should not be null or empty");
}
Exception actionError = null;
if (actionState.RollbackState == RollbackState.NotRollingBack ||
(actionState.RollbackState != RollbackState.RollingBackForce && actionState.RetryStepWithoutRollingBackOnFailure))
{
// TODO: should also include Error
while (actionState.StateProgress.Peek() != StepStateNames.CompletedSuccessfully)
{
cancellationToken.ThrowIfCancellationRequested();
RollbackState readRollbackState = await this.CheckUserCancellationAndUpdateIfNeededAsync(actionState, cancellationToken, FASConstants.ForwardLoop).ConfigureAwait(false);
if ((readRollbackState == RollbackState.RollingBackForce) ||
((readRollbackState == RollbackState.RollingBackDueToUserCancel) && !actionState.RetryStepWithoutRollingBackOnFailure))
{
break;
}
try
{
await this.RunStepAsync(fabricClient, action, actionState, cancellationToken, serviceInternalFaultInfo).ConfigureAwait(false);
ActionTest.PerformInternalServiceFaultIfRequested(actionState.OperationId, serviceInternalFaultInfo, actionState, cancellationToken);
if (actionState.StateProgress.Peek() == StepStateNames.CompletedSuccessfully)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - completed successfully, clearing ErrorCausingRollback", actionState.OperationId);
actionState.ErrorCausingRollback = 0;
}
actionState.TimeStopped = DateTime.UtcNow;
await this.actionStore.UpdateActionStateAsync(actionState).ConfigureAwait(false);
}
catch (RetrySameStepException)
{
// Retry the command in the same step - do not rollback or go forward, and do not call ActionStore.UpdateActionStateAsync().
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - threw RetrySameStepException, retrying state {1} ", actionState.StateProgress.Peek());
}
catch (FabricNotPrimaryException)
{
throw;
}
catch (FabricObjectClosedException)
{
throw;
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - RunStateMachineAsync caught: {1}", actionState.OperationId, e.ToString());
readRollbackState = this.CheckUserCancellationAndUpdateIfNeededAsync(actionState, cancellationToken, FASConstants.ForwardLoopExceptionBlock).GetAwaiter().GetResult();
// 1st line: if this is a force rollback (RollingBackForce), just exit
// 2nd line: if !RetryStepWithoutRollingBackOnFailure and there was a graceful cancel then exit this block and proceed to the rollback code block below.
// If RetryStepWithoutRollingBackOnFailure is true, which it is only for the node steps today, then first call HandleRollback to translate the exception.
if ((readRollbackState == RollbackState.RollingBackForce) ||
((readRollbackState == RollbackState.RollingBackDueToUserCancel) && !actionState.RetryStepWithoutRollingBackOnFailure))
{
break;
}
else
{
bool isRetryable = this.HandleRollback(actionState.OperationId, e);
if (isRetryable)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - observed retryable exception, will retry action. Exception: {1}", actionState.OperationId, e.ToString());
actionState.RollbackState = RollbackState.RollingBackAndWillRetryAction;
}
else
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - observed non-retryable exception. Exception: {1}", actionState.OperationId, e.ToString());
actionState.RollbackState = RollbackState.RollingBackAndWillFailAction;
}
}
actionError = e;
break;
}
}
}
if (actionState.RollbackState == RollbackState.RollingBackAndWillRetryAction ||
actionState.RollbackState == RollbackState.RollingBackAndWillFailAction ||
(actionState.RollbackState == RollbackState.RollingBackDueToUserCancel && (actionState.StateProgress.Peek() != StepStateNames.CompletedSuccessfully)))
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Rollingback type={1}", actionState.OperationId, actionState.ActionType);
if (!this.isTestMode && actionState.StateProgress.Peek() == StepStateNames.CompletedSuccessfully)
{
string error = string.Format(CultureInfo.InvariantCulture, "{0} - state should not be CompletedSuccessfully", actionState.OperationId);
TestabilityTrace.TraceSource.WriteError(TraceType, error);
ReleaseAssert.Failfast(error);
}
// If actionError is not null it means we are currently running a resumed rollback. In that case the ErrorCausingRollback must have
// already been set.
if (actionError != null)
{
actionState.ErrorCausingRollback = TranslateRollbackError(actionError.HResult);
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Translated ErrorCausingRollback ={1}", actionState.OperationId, actionState.ErrorCausingRollback);
}
if (this.isTestMode && actionState.StateProgress.Peek() == StepStateNames.CompletedSuccessfully)
{
// In test mode it's intentionally possible to fault an action after it's completed its work, but before the state name has been updated.
actionState.StateProgress.Pop();
}
await this.actionStore.UpdateActionStateAsync(actionState).ConfigureAwait(false);
try
{
while (actionState.StateProgress.Peek() != StepStateNames.IntentSaved &&
actionState.StateProgress.Peek() != StepStateNames.Failed)
{
cancellationToken.ThrowIfCancellationRequested();
RollbackState readRollbackState = await this.CheckUserCancellationAndUpdateIfNeededAsync(actionState, cancellationToken, FASConstants.OuterCleanupLoop).ConfigureAwait(false);
if (readRollbackState == RollbackState.RollingBackDueToUserCancel)
{
// Do nothing, already rolling back - debug only
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Read RollingBackDueToUserCancel in outer rollback loop", actionState.OperationId);
}
else if (readRollbackState == RollbackState.RollingBackForce)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Read RollingBackForce in outer rollback loop", actionState.OperationId);
break;
}
StepStateNames currentStateName = actionState.StateProgress.Peek();
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - DEBUG - Rollback path loop, current state {1}", actionState.OperationId, actionState.StateProgress.Peek());
try
{
await this.CleanupStepAsync(fabricClient, action, actionState, cancellationToken, serviceInternalFaultInfo).ConfigureAwait(false);
await this.actionStore.UpdateActionStateAsync(actionState).ConfigureAwait(false);
}
catch (FabricNotPrimaryException)
{
throw;
}
catch (FabricObjectClosedException)
{
throw;
}
catch (Exception e)
{
ReleaseAssert.Failfast("Unexpected exception, RunStateAsync for cleanup should have handled {0}", e);
}
}
// If this is true rollback is finished. If it is retryable set the state to LookingUpState
if (actionState.StateProgress.Peek() == StepStateNames.IntentSaved)
{
if (actionState.RollbackState == RollbackState.RollingBackAndWillRetryAction)
{
actionState.StateProgress.Push(StepStateNames.LookingUpState);
actionState.ClearInfo();
}
else if (actionState.RollbackState == RollbackState.RollingBackAndWillFailAction)
{
actionState.StateProgress.Push(StepStateNames.Failed);
actionState.RollbackState = RollbackState.NotRollingBack;
actionState.TimeStopped = DateTime.UtcNow;
}
else if (actionState.RollbackState == RollbackState.RollingBackDueToUserCancel)
{
actionState.StateProgress.Push(StepStateNames.Failed);
actionState.TimeStopped = DateTime.UtcNow;
}
else if (actionState.RollbackState == RollbackState.RollingBackForce)
{
actionState.StateProgress.Push(StepStateNames.Failed);
actionState.TimeStopped = DateTime.UtcNow;
}
else
{
string error = string.Format(CultureInfo.InvariantCulture, "{0} - RollbackState == NotRollingBack not expected", actionState.OperationId);
ReleaseAssert.Failfast(error);
}
}
else if (actionState.RollbackState == RollbackState.RollingBackForce)
{
actionState.StateProgress.Push(StepStateNames.Failed);
actionState.TimeStopped = DateTime.UtcNow;
}
}
catch (OperationCanceledException)
{
// This means the cancellation token is set, not that an api call observed an E_ABORT
throw;
}
catch (FabricNotPrimaryException)
{
throw;
}
catch (FabricObjectClosedException)
{
throw;
}
catch (Exception e)
{
ReleaseAssert.Failfast("Unexpected exception, RunStateAsync for cleanup should have handled {0}", e);
}
TestabilityTrace.TraceSource.WriteInfo(
TraceType,
"{0} - Action failed, type='{1}', will retry={2}, RollbackState={3}",
actionState.OperationId,
actionState.ActionType,
actionState.RollbackState == RollbackState.RollingBackAndWillRetryAction ? "true" : "false",
actionState.RollbackState);
await this.actionStore.UpdateActionStateAsync(actionState).ConfigureAwait(false);
}
else if (actionState.StateProgress.Peek() == StepStateNames.CompletedSuccessfully)
{
// user cancelled, but action/command completed anyways before cancellation was checked.
TestabilityTrace.TraceSource.WriteInfo(TraceType, "DEBUG {0} - Action type '{1}' completed successfully, not updating again ", actionState.OperationId, actionState.ActionType);
}
else if ((actionState.StateProgress.Peek() == StepStateNames.IntentSaved) &&
(actionState.RollbackState == RollbackState.RollingBackDueToUserCancel))
{
actionState.StateProgress.Push(StepStateNames.Failed);
actionState.TimeStopped = DateTime.UtcNow;
await this.actionStore.UpdateActionStateAsync(actionState).ConfigureAwait(false);
}
else if (actionState.RollbackState == RollbackState.RollingBackForce)
{
// Note: unlike the case above this does not have a state of IntentSaved as a requirement since a force rollback is an abort and does run the steps in reverse.
// It is possible for the StateProgress to be CompletedSuccessfully here, since we want to exit as quickly as possible. In that case, the block 2 blocks above handles it -
// we do nothing extra, and the command finishes executing. If the user calls an api for information on this command, we translate the state to ForceCancelled if state is a terminal state
// and RollbackState is RollingBackForce. See ActionStore.MatchesStateFilter().
actionState.TimeStopped = DateTime.UtcNow;
TestabilityTrace.TraceSource.WriteWarning(TraceType, "Bottom of Engine.RunAsync() - state is={0}, rollbackState={1}", actionState.StateProgress.Peek().ToString(), actionState.RollbackState.ToString());
actionState.StateProgress.Push(StepStateNames.Failed);
await this.actionStore.UpdateActionStateAsync(actionState).ConfigureAwait(false);
}
else
{
string unexpectedError = string.Format(CultureInfo.InvariantCulture, "Unexpected case reached, state is={0}, rollbackState={1}", actionState.StateProgress.Peek().ToString(), actionState.RollbackState.ToString());
TestabilityTrace.TraceSource.WriteError(TraceType, "{0}", unexpectedError);
ReleaseAssert.Failfast(unexpectedError);
}
}
internal static unsafe NamedProperty FromNative(NativeClient.IFabricPropertyValueResult nativeResult, bool includesValue)
{
if (nativeResult == null)
{
return(null);
}
NativeTypes.FABRIC_NAMED_PROPERTY *nativeNamedProperty = (NativeTypes.FABRIC_NAMED_PROPERTY *)nativeResult.get_Property();
if (nativeNamedProperty->Metadata == IntPtr.Zero)
{
AppTrace.TraceSource.WriteError("NamedProperty.GetValueInternal", "Property has no metadata");
ReleaseAssert.Failfast(StringResources.Error_PropertyHasNoMetadata);
}
NativeTypes.FABRIC_NAMED_PROPERTY_METADATA *nativePropertyMetaData = (NativeTypes.FABRIC_NAMED_PROPERTY_METADATA *)nativeNamedProperty->Metadata;
NamedPropertyMetadata namedPropertyMetadata = NamedPropertyMetadata.FromNative(*nativePropertyMetaData);
if (nativeNamedProperty->Value == IntPtr.Zero)
{
includesValue = false;
}
NamedProperty returnValue = null;
if (includesValue)
{
object value = null;
switch (namedPropertyMetadata.TypeId)
{
case PropertyTypeId.Binary:
value = NativeTypes.FromNativeBytes(nativeNamedProperty->Value, (uint)namedPropertyMetadata.ValueSize);
break;
case PropertyTypeId.Int64:
value = (Int64)(*((Int64 *)nativeNamedProperty->Value));
break;
case PropertyTypeId.Double:
value = (double)(*((double *)nativeNamedProperty->Value));
break;
case PropertyTypeId.String:
value = new string((char *)nativeNamedProperty->Value);
break;
case PropertyTypeId.Guid:
value = (Guid)(*((Guid *)nativeNamedProperty->Value));
break;
default:
AppTrace.TraceSource.WriteError("NamedProperty.GetValueInternal", "Unknown propertyTypeId: {0}", namedPropertyMetadata.TypeId);
throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, StringResources.Error_PropertyTypeIDUnknown_Formatted, namedPropertyMetadata.TypeId));
}
returnValue = new NamedProperty(namedPropertyMetadata, value);
}
else
{
returnValue = new NamedProperty(namedPropertyMetadata);
}
GC.KeepAlive(nativeResult);
return(returnValue);
}
public Task InsertCommandAsync(Command command)
{
Uri failoverManagerUri = new Uri("fabric:/System/FailoverManagerService");
TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, "****Adding command: " + command);
Task task = null;
if (command == Command.FailoverManagerDataLoss)
{
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
Guid id = MockClientCommandInfo[Command.FailoverManagerDataLoss];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, null);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.InvokeDataLossMidActionTestFatal)
{
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.RollbackAction);
Guid id = MockClientCommandInfo[Command.InvokeDataLossMidActionTestFatal];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed);
}
else if (command == Command.InvokeDataLossMidActionTestTransient)
{
// rollback and retry then success
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.RollbackActionAndRetry);
Guid id = MockClientCommandInfo[Command.InvokeDataLossMidActionTestTransient];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.InvokeDataLossMidActionTestFailover)
{
// failover then success
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.KillProcess);
Guid id = MockClientCommandInfo[Command.InvokeDataLossMidActionTestFailover];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.FailoverManagerDataLossCauseActionRollbackFatal)
{
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.CompletedSuccessfully, ServiceInternalFaultType.RollbackAction);
Guid id = MockClientCommandInfo[Command.FailoverManagerDataLossCauseActionRollbackFatal];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed);
}
else if (command == Command.FailoverManagerDataLossCauseActionRollbackWithSuccessOnRetry)
{
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.CompletedSuccessfully, ServiceInternalFaultType.RollbackActionAndRetry);
Guid id = MockClientCommandInfo[Command.FailoverManagerDataLossCauseActionRollbackWithSuccessOnRetry];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.FailoverManagerDataLossFailoverFaultAnalysisService)
{
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.PerformingActions, ServiceInternalFaultType.KillProcess);
Guid id = MockClientCommandInfo[Command.FailoverManagerDataLossFailoverFaultAnalysisService];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.FailoverManagerDataLossCauseActionRollbackFatalBeforeActionStep)
{
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.PerformingActions, ServiceInternalFaultType.RollbackAction);
Guid id = MockClientCommandInfo[Command.FailoverManagerDataLossCauseActionRollbackFatalBeforeActionStep];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed);
}
else if (command == Command.FailoverManagerDataLossCauseActionRollbackWithSuccessOnRetryBeforeActionStep)
{
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.PerformingActions, ServiceInternalFaultType.RollbackActionAndRetry);
Guid id = MockClientCommandInfo[Command.FailoverManagerDataLossCauseActionRollbackWithSuccessOnRetryBeforeActionStep];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.InvokeQuorumLossMidActionTestFatal)
{
Uri uri = new Uri("fabric:/System/NamingService");
PartitionSelector ps = PartitionSelector.RandomOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.RollbackAction);
Guid id = MockClientCommandInfo[Command.InvokeQuorumLossMidActionTestFatal];
task = this.messageProcessor.ProcessQuorumLossCommandAsync(id, ps, QuorumLossMode.AllReplicas, TimeSpan.FromSeconds(10.0d), FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed);
}
else if (command == Command.InvokeQuorumLossMidActionTestFailover)
{
Uri uri = new Uri("fabric:/System/NamingService");
PartitionSelector ps = PartitionSelector.RandomOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.KillProcess);
Guid id = MockClientCommandInfo[Command.InvokeQuorumLossMidActionTestFailover];
task = this.messageProcessor.ProcessQuorumLossCommandAsync(id, ps, QuorumLossMode.AllReplicas, TimeSpan.FromSeconds(10.0d), FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.InvokeQuorumLossMidActionTestTransient)
{
Uri uri = new Uri("fabric:/System/NamingService");
PartitionSelector ps = PartitionSelector.RandomOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.RollbackActionAndRetry);
Guid id = MockClientCommandInfo[Command.InvokeQuorumLossMidActionTestTransient];
task = this.messageProcessor.ProcessQuorumLossCommandAsync(id, ps, QuorumLossMode.AllReplicas, TimeSpan.FromSeconds(10.0d), FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.RestartPartitionMidActionTestFatal)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.RollbackAction);
Guid id = MockClientCommandInfo[Command.RestartPartitionMidActionTestFatal];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed);
}
else if (command == Command.RestartPartitionMidActionTestFailover)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.KillProcess);
Guid id = MockClientCommandInfo[Command.RestartPartitionMidActionTestFailover];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.RestartPartitionMidActionTestTransient)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.RollbackActionAndRetry);
Guid id = MockClientCommandInfo[Command.RestartPartitionMidActionTestTransient];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.StuckAction)
{
Guid id = MockClientCommandInfo[Command.StuckAction];
task = this.messageProcessor.ProcessStuckCommandAsync(id, null);
}
else if (command == Command.RestartPartitionCancelOuterLoopNoForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.None, // for this case this value does not matter
ServiceInternalFaultType.None,
RollbackState.RollingBackDueToUserCancel,
FASConstants.OuterLoop);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelOuterLoopNoForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackDueToUserCancel);
}
else if (command == Command.RestartPartitionCancelForwardNoForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.None,
ServiceInternalFaultType.None,
RollbackState.RollingBackDueToUserCancel,
FASConstants.ForwardLoop);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelForwardNoForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackDueToUserCancel);
}
else if (command == Command.RestartPartitionCancelForwardExceptionNoForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.MidPerformingActions,
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackDueToUserCancel,
FASConstants.ForwardLoopExceptionBlock);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelForwardExceptionNoForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackDueToUserCancel);
}
else if (command == Command.RestartPartitionCancelOuterCleanupNoForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.PerformingActions,
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackDueToUserCancel,
FASConstants.OuterCleanupLoop);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelOuterCleanupNoForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackDueToUserCancel);
}
else if (command == Command.RestartPartitionCancelCleanupInnerNoForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.PerformingActions,
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackDueToUserCancel,
FASConstants.InnerCleanupLoop);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelCleanupInnerNoForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackDueToUserCancel);
}
else if (command == Command.RestartPartitionCancelOuterLoopForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.PerformingActions, // for this case, this value does not matter
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackForce,
FASConstants.ForwardLoop);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelOuterLoopForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce);
}
else if (command == Command.RestartPartitionCancelForwardForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.PerformingActions,
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackForce,
FASConstants.ForwardLoop);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelForwardForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce);
}
else if (command == Command.RestartPartitionCancelForwardExceptionForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.PerformingActions,
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackForce,
FASConstants.ForwardLoopExceptionBlock);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelForwardExceptionForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce);
}
else if (command == Command.RestartPartitionCancelOuterCleanupForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.PerformingActions,
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackForce,
FASConstants.OuterCleanupLoop);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelOuterCleanupForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce);
}
else if (command == Command.RestartPartitionCancelCleanupInnerForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.PerformingActions,
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackForce,
FASConstants.InnerCleanupLoop);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelCleanupInnerForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce);
}
else if (command == Command.TestRetryStepWithSuccessAfterRetries)
{
// Intentionally fail the step corresponding to StepStateNames.PerformingActions step a few times, then run it normally (pass). It should succeed.
Guid id = MockClientCommandInfo[Command.TestRetryStepWithSuccessAfterRetries];
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.MidPerformingActions,
ServiceInternalFaultType.ThrowThreeTimes);
task = this.messageProcessor.ProcessRetryStepCommandAsync(id, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully, RollbackState.NotRollingBack);
}
else if (command == Command.TestRetryStepWithForceCancel)
{
// Force cancel a command with ActionStateBase.RetryStepWithoutRollingBackOnFailure set to true
Guid id = MockClientCommandInfo[Command.TestRetryStepWithForceCancel];
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.CompletedSuccessfully, // this just has to be a late step so an earlier fault is not used before we reach the situation we want.
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackForce, // note, the graceful one should not cause cancellation since for this type we only allow user cancellation when force is true
FASConstants.InnerForwardLoop,
StepStateNames.PerformingActions);
task = this.messageProcessor.ProcessRetryStepCommandAsync(id, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce);
}
else if (command == Command.StopNodeWithUnknownException)
{
Guid id = MockClientCommandInfo[Command.StopNodeWithUnknownException];
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.MidPerformingActions,
ServiceInternalFaultType.RollbackAction); // In this case, since start and stop node do not rollback like other commands, this exception should cause the step to retry.
Node target = ActionTest.GetNodeWithFASSecondary().Result;
TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, "{0} stopping {1}:{2}", id, target.NodeName, target.NodeInstanceId);
task = this.messageProcessor.ProcessStopNodeCommandAsync(id, target.NodeName, target.NodeInstanceId, 999, FASConstants.DefaultTestTimeout, faultInfo);
// Let the command make progress
Task.Delay(TimeSpan.FromSeconds(30)).Wait();
this.WaitForState(id, StepStateNames.PerformingActions, RollbackState.NotRollingBack);
// This should not result in cancellation, since start and stop node have different rollback policies than the other commands.
TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, "{0} - cancelling with force==false. This should not cancel the command", id);
this.messageProcessor.CancelTestCommandAsync(id, false);
this.WaitForState(id, StepStateNames.PerformingActions, RollbackState.RollingBackDueToUserCancel);
// Now force cancel. This should cancel.
TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, "{0} - cancelling with force==true. This should cancel the command", id);
this.messageProcessor.CancelTestCommandAsync(id, true);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce);
NodeList nodes = ActionTest.GetNodeListAsync().Result;
TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, "{0} - node info:", id);
foreach (Node n in nodes)
{
TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, " OperationId:{0} - NodeName{1}, NodeStatus:{2}, IsStopped:{3}", id, n.NodeName, n.NodeStatus, n.IsStopped);
}
Node targetNodeAfterTest = nodes.Where(n => n.NodeName == target.NodeName).FirstOrDefault();
if (targetNodeAfterTest == null)
{
throw new InvalidOperationException("target node was not found in query after test");
}
if (targetNodeAfterTest.IsStopped == false)
{
throw new InvalidOperationException("target node should have IsStopped true, was false");
}
}
else if (command == Command.StopNodeWithExceptionAndSuccessAfterRetries)
{
Guid id = MockClientCommandInfo[Command.StopNodeWithExceptionAndSuccessAfterRetries];
// Inject a fault during the operation so that step "StepStateNames.MidPerformingActions" has to retry 3 times before succeeding
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.MidPerformingActions,
ServiceInternalFaultType.ThrowThreeTimes);
Node target = ActionTest.GetNodeWithFASSecondary().Result;
TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, "{0} stopping {1}:{2}", id, target.NodeName, target.NodeInstanceId);
task = this.messageProcessor.ProcessStopNodeCommandAsync(id, target.NodeName, target.NodeInstanceId, 999, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully, RollbackState.NotRollingBack);
// Start the stopped node
task = this.messageProcessor.ProcessStartNodeCommandAsync(Guid.NewGuid(), target.NodeName, target.NodeInstanceId, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully, RollbackState.NotRollingBack);
}
else
{
ReleaseAssert.Failfast("Unexpected command");
}
return(task);
}
private async Task ProcessCompletedActionsAsync(CancellationToken cancellationToken)
{
// Look for completed tasks and remove them and move into the historical table. If the task failed, mark it
while (!cancellationToken.IsCancellationRequested)
{
try
{
while (this.pendingTasks.Count != 0)
{
cancellationToken.ThrowIfCancellationRequested();
// find the things in terminal finished state, process accordingly, call release on semaphore
KeyValuePair <Guid, ActionCompletionInfo> kv = this.pendingTasks.Where(p => p.Value.Task.Status == TaskStatus.RanToCompletion).FirstOrDefault();
if (!kv.Equals(default(KeyValuePair <Guid, ActionCompletionInfo>)))
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Fetched a completed action key={0}", kv.Key);
ActionCompletionInfo info;
if (!this.pendingTasks.TryRemove(kv.Key, out info))
{
// This should never happen?
TestabilityTrace.TraceSource.WriteError(TraceType, "Removal of key={0} failed", kv.Key);
ReleaseAssert.Failfast("Removal of key={0} failed", kv.Key);
}
else
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Key={0} removed successfully", kv.Key);
try
{
info.Semaphore.Release();
}
catch (ObjectDisposedException ode)
{
if ((ode.Message == "The semaphore has been disposed.") && (this.isShuttingDown > 0))
{
break;
}
throw;
}
}
}
else
{
TestabilityTrace.TraceSource.WriteNoise(TraceType, "Nothing to remove yet");
}
}
await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken).ConfigureAwait(false);
FaultAnalysisServiceMessageProcessor.CheckForInvalidStatus(this.pendingTasks);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception e)
{
// This shouldn't be reached
TestabilityTrace.TraceSource.WriteError(TraceType, "ProcessCompletedActionsAsync caught {0}", e.ToString());
ReleaseAssert.Failfast("ProcessCompletedActionsAsync caught {0}", e.ToString());
}
}
}
// Launched as a task
private async Task ConsumeAndRunActionsAsync(CancellationToken cancellationToken)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter ConsumeAndRunActions");
using (SemaphoreSlim semaphore = new SemaphoreSlim(this.concurrentRequests, this.concurrentRequests))
{
while (!cancellationToken.IsCancellationRequested)
{
try
{
TestabilityTrace.TraceSource.WriteNoise(TraceType, "DEBUG Waiting for semaphore, max count={0}", this.concurrentRequests);
await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteNoise(TraceType, "DEBUG Done waiting for semaphore");
}
catch (OperationCanceledException e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, e.ToString());
break;
}
ActionStateBase actionState = null;
bool wasSuccessful = false;
try
{
TestabilityTrace.TraceSource.WriteNoise(TraceType, "Trying to Dequeue");
wasSuccessful = this.queue.TryTake(out actionState);
if (!wasSuccessful)
{
TestabilityTrace.TraceSource.WriteNoise(TraceType, "Queue was empty");
semaphore.Release();
await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
continue;
}
TestabilityTrace.TraceSource.WriteNoise(TraceType, "Dequeued Name={0} Key={1}", actionState.ActionType, actionState.OperationId);
}
catch (OperationCanceledException e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, e.ToString());
break;
}
catch (Exception ex)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "TryDequeue failed with={0}", ex.ToString());
semaphore.Release();
continue;
}
Task t = null;
try
{
System.Fabric.FaultAnalysis.Service.Actions.FabricTestAction action = await this.ConstructActionAsync(actionState.ActionType, actionState);
t = this.engine.RunAsync(this.fabricClient, action, actionState, actionState.ServiceInternalFaultInfo, cancellationToken);
}
catch (FabricNotPrimaryException notPrimary)
{
FaultAnalysisServiceUtility.TraceFabricNotPrimary(actionState.OperationId, notPrimary);
break;
}
catch (FabricObjectClosedException objectClosed)
{
FaultAnalysisServiceUtility.TraceFabricObjectClosed(actionState.OperationId, objectClosed);
break;
}
if (!this.pendingTasks.TryAdd(actionState.OperationId, new ActionCompletionInfo(actionState.OperationId, t, semaphore)))
{
TestabilityTrace.TraceSource.WriteError(TraceType, "Add of key={0} failed, was it already added?", actionState.OperationId);
ReleaseAssert.Failfast("Add of key={0} failed, was it already added?", actionState.OperationId);
}
else
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Action type={0}, key={1} started", actionState.ActionType, actionState.OperationId);
}
Interlocked.Increment(ref this.isShuttingDown);
}
TestabilityTrace.TraceSource.WriteWarning(TraceType, "Cancellation was requested");
return;
}
}