public void FabricExceptionListContainsCheck()
{
var errorCode = FabricErrorCode.FabricHealthEntityNotFound;
var list = new List <FabricErrorCode>();
list.Add(errorCode);
var ex = new FabricException("Exception With Non-Zero ErrorCode", errorCode);
Assert.IsTrue(list.Contains(errorCode));
Assert.AreEqual(errorCode, ex.ErrorCode);
Assert.IsTrue(list.Contains(ex.ErrorCode));
}
private static string CreateThreadErrorMessage(Exception ex, bool includeStackTrace = true)
{
const int MaxCharactersOfMessageToCopy = 512;
// there are limits such as 4k for the health report size and 4k for the interop
// use a smaller limit here
const int MaxLength = 4000;
ExceptionStringBuilder sb = new ExceptionStringBuilder(MaxLength);
int HResult = ex.HResult;
string message = ex.Message;
System.Type type = ex.GetType();
sb.Append(GetExceptionInfoString(type, HResult));
sb.AppendLine();
FabricException fabricEx = ex as FabricException;
if (fabricEx != null && (fabricEx.InnerException != null))
{
// Add only 256 bytes of the message to prevent it from overflowing the health report
sb.Append(message, MaxCharactersOfMessageToCopy / 2);
sb.AppendLine();
Exception inner = fabricEx.InnerException;
sb.Append(GetExceptionInfoString(inner.GetType(), inner.HResult));
sb.AppendLine();
// Outer exception + Inner exception message should be upto 512 bytes to prevent it from overflowing the health report
sb.Append(inner.Message, MaxCharactersOfMessageToCopy);
}
else
{
// Add only 512 bytes of the message to prevent it from overflowing the health report
sb.Append(message, MaxCharactersOfMessageToCopy);
}
if (includeStackTrace)
{
// Add the stack trace
sb.AppendLine();
sb.Append(ex.StackTrace);
}
return(sb.ToString());
}
private SuccessRetryOrFail HandleFabricException(FabricException fe, NodeCommandState state)
{
SuccessRetryOrFail status = SuccessRetryOrFail.Invalid;
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - StartNodeUsingNodeNameAsync threw FabricException with ErrorCode={1}", this.State.OperationId, fe.ErrorCode);
if (fe.ErrorCode == FabricErrorCode.InstanceIdMismatch)
{
status = SuccessRetryOrFail.Fail;
}
else if (fe.ErrorCode == FabricErrorCode.NodeIsUp)
{
status = SuccessRetryOrFail.Success;
}
else if (fe.ErrorCode == FabricErrorCode.NodeHasNotStoppedYet)
{
status = SuccessRetryOrFail.RetryStep;
}
else if (fe.ErrorCode == FabricErrorCode.InvalidAddress)
{
if (state.Info.InitialQueriedNodeStatus != NodeStatus.Down && state.Info.NodeWasInitiallyInStoppedState == false)
{
// This is a (probably unlikely) case that may happen if
// 1. The request reaches the target node, which is up and is processed.
// 2. The response is dropped
// 3. The request is retried with enough delay such that the node has now transitioned from up to stopped
// So, we say that if the initial conditions were valid and we get InvalidAddress, then consider it successful.
status = SuccessRetryOrFail.Success;
}
else
{
// The preconditions passed, but the node is down now, so something out of band happened.
status = SuccessRetryOrFail.RetryStep;
}
}
else if (fe.ErrorCode == FabricErrorCode.NodeNotFound)
{
// Always fatal
string nodeNotFoundErrorMessage = string.Format(CultureInfo.InvariantCulture, "{0} - node {1} was not found", state.OperationId, state.Info.NodeName);
TestabilityTrace.TraceSource.WriteError(StepBase.TraceType, nodeNotFoundErrorMessage);
status = SuccessRetryOrFail.Fail;
}
else
{
status = SuccessRetryOrFail.RetryStep;
}
return(status);
}
private async Task <FabricException> ExecuteUpdateRouteCommandAsync(ContainerExecConfig execConfig)
{
FabricException opEx = null;
try
{
await ExecuteUpdateRouteCommandWithRetriesAsync(execConfig);
}
catch (FabricException ex)
{
opEx = ex;
}
return(opEx);
}
private static bool TryHandleFabricException(FabricException fabricException, OperationRetrySettings retrySettings, out ExceptionHandlingResult result)
{
if (fabricException is FabricCannotConnectException || fabricException is FabricEndpointNotFoundException)
{
result = new ExceptionHandlingRetryResult(fabricException, false, retrySettings, int.MaxValue);
return(true);
}
if (fabricException.ErrorCode.Equals(FabricErrorCode.ServiceTooBusy))
{
result = new ExceptionHandlingRetryResult(fabricException, true, retrySettings, int.MaxValue);
return(true);
}
result = null;
return(false);
}
public bool IsRetryable(Exception e)
{
Exception currentEx = e;
for (int i = 0; i < 5 && currentEx != null; i++)
{
if (this.RetryableExceptions.Contains(currentEx.GetType()))
{
return(true);
}
currentEx = currentEx.InnerException;
}
FabricException fabricException = e as FabricException;
if (fabricException != null && this.RetryableFabricErrorCodes.Contains(fabricException.ErrorCode))
{
return(true);
}
if (this.RetrySuccessExceptions.Contains(e.GetType()))
{
return(true);
}
if (fabricException != null && this.RetrySuccessFabricErrorCodes.Contains(fabricException.ErrorCode))
{
return(true);
}
if (e.GetType() == typeof(FabricTransientException))
{
return(true);
}
if (fabricException != null && fabricException.InnerException != null)
{
var ex = fabricException.InnerException;
if (this.InternalRetrySuccessFabricErrorCodes.Contains((uint)ex.HResult))
{
return(true);
}
}
return(false);
}
private static bool HandleException(Exception e, FabricClientRetryErrors errors, out bool retryElseSuccess)
{
FabricException fabricException = e as FabricException;
if (errors.RetryableExceptions.Contains(e.GetType()))
{
retryElseSuccess = true /*retry*/;
return(true);
}
if (fabricException != null && errors.RetryableFabricErrorCodes.Contains(fabricException.ErrorCode))
{
retryElseSuccess = true /*retry*/;
return(true);
}
if (errors.RetrySuccessExceptions.Contains(e.GetType()))
{
retryElseSuccess = false /*success*/;
return(true);
}
if (fabricException != null && errors.RetrySuccessFabricErrorCodes.Contains(fabricException.ErrorCode))
{
retryElseSuccess = false /*success*/;
return(true);
}
if (e.GetType() == typeof(FabricTransientException))
{
retryElseSuccess = true /*retry*/;
return(true);
}
if (fabricException != null && fabricException.InnerException != null)
{
var ex = fabricException.InnerException as COMException;
if (ex != null && errors.InternalRetrySuccessFabricErrorCodes.Contains((uint)ex.ErrorCode))
{
retryElseSuccess = false /*success*/;
return(true);
}
}
retryElseSuccess = false;
return(false);
}
internal static async Task CleanupOnErrorAsync(
string traceType,
bool autoRemove,
bool isContainerRoot,
string cgroupName,
string containerName,
string applicationId,
IContainerActivatorService activatorService,
TimeoutHelper timeoutHelper,
FabricException originalEx)
{
HostingTrace.Source.WriteWarning(
traceType,
"CleanupOnErrorAsync(): Deactivating Container='{0}', AppId={1}.",
containerName,
applicationId);
try
{
var deactivationArg = new ContainerDeactivationArgs()
{
ContainerName = containerName,
ConfiguredForAutoRemove = autoRemove,
IsContainerRoot = isContainerRoot,
CgroupName = cgroupName,
IsGracefulDeactivation = false
};
await Utility.ExecuteWithRetryOnTimeoutAsync(
(operationTimeout) =>
{
return(activatorService.DeactivateContainerAsync(
deactivationArg,
operationTimeout));
},
$"CleanupOnErrorAsync_DeactivateContainerAsync_{containerName}",
traceType,
HostingConfig.Config.DockerRequestTimeout,
timeoutHelper.RemainingTime);
}
catch (FabricException)
{
// Ignore.
}
throw originalEx;
}
private async Task AddOutboundConnectivityToContainerAsync()
{
var netwrokParams = new NetworkConnectConfig()
{
Container = this.containerId
};
FabricException opEx = null;
try
{
await Utility.ExecuteWithRetriesAsync(
(operationTimeout) =>
{
return(this.activator.Client.NetworkOperation.ConnectNetworkAsync(
"nat",
netwrokParams,
operationTimeout));
},
$"AddOutboundConnectivityToContainerAsync_{this.ContainerName}",
TraceType,
HostingConfig.Config.DockerRequestTimeout,
this.timeoutHelper.RemainingTime);
HostingTrace.Source.WriteNoise(
TraceType,
"Adding outbound connectivity to {0} succeeded.",
BuildContainerInfoMessage());
}
catch (Exception ex)
{
var errorMessage = string.Format(
"Adding outbound connectivity failed for {0}",
this.BuildErrorMessage(ex));
HostingTrace.Source.WriteError(TraceType, errorMessage);
opEx = new FabricException(errorMessage, FabricErrorCode.InvalidOperation);
}
if (opEx != null)
{
await this.CleanupOnErrorAsync(opEx);
}
}
private SuccessRetryOrFail HandleFabricException(FabricException fe, NodeCommandState state)
{
SuccessRetryOrFail status = SuccessRetryOrFail.Invalid;
if (fe.ErrorCode == FabricErrorCode.InstanceIdMismatch)
{
TestabilityTrace.TraceSource.WriteWarning(StepBase.TraceType, "{0} - StopNode api threw InstanceIdMismatch", this.State.OperationId);
status = SuccessRetryOrFail.Fail;
}
else if (fe.ErrorCode == FabricErrorCode.InvalidAddress)
{
TestabilityTrace.TraceSource.WriteInfo(
StepBase.TraceType,
"{0} - StopNode HandleFabricException InitialQueriedNodeStatus='{1}', NodeWasInitiallyInStoppedState='{2}'",
this.State.OperationId,
state.Info.InitialQueriedNodeStatus,
state.Info.NodeWasInitiallyInStoppedState);
// If the request was valid (node was up), but the response was dropped, and then the request was retried, we might now be
// sending a stop to a node that is now stopped. The request will return InvalidAddress, but this is really success.
if (state.Info.InitialQueriedNodeStatus != NodeStatus.Down && state.Info.NodeWasInitiallyInStoppedState == false)
{
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - StopNode api threw InvalidAddress, considering stop command successful due to preconditions", this.State.OperationId);
status = SuccessRetryOrFail.Success;
}
else
{
// Invalid address is not expected here. There may have been an out of band stop using the deprecated api. We should retry until we get something else.
TestabilityTrace.TraceSource.WriteWarning(StepBase.TraceType, "{0} - StopNode api threw InvalidAddress", this.State.OperationId);
status = SuccessRetryOrFail.RetryStep;
}
}
else if (fe.ErrorCode == FabricErrorCode.NodeNotFound)
{
status = SuccessRetryOrFail.Fail;
}
else
{
TestabilityTrace.TraceSource.WriteWarning(StepBase.TraceType, "{0} - StopNode api threw {1}", this.State.OperationId, fe);
status = SuccessRetryOrFail.RetryStep;
}
return(status);
}
internal static async Task <string> GetFabricUriFromRequstHeaderForPartitions(string fabricRequestHeader, TimeSpan timeout, CancellationToken cancellationToken)
{
string fabricUri;
try
{
string[] segments = fabricRequestHeader.Split('/');
string type = segments[0];
switch (type)
{
case "Partitions":
if (segments.Length != 2 || string.IsNullOrEmpty(segments[1]))
{
throw new ArgumentException(StringResources.InvalidArguments);
}
string serviceNameUri = await FabricClientHelper.GetFabricServiceUriFromPartitionId(segments[1], timeout, cancellationToken);
string applicationNameUri = await FabricClientHelper.GetFabricApplicationUriFromServiceUri(serviceNameUri, timeout, cancellationToken);
string applicationName = GetNameFromUri(applicationNameUri);
string serviceName = RemoveApplicationNameUriFromServiceNameUri(applicationNameUri, serviceNameUri);
fabricUri = string.Format("{0}/{1}",
CreateUriFromApplicationAndServiceName(applicationName, serviceName), segments[1]);
break;
default:
throw new ArgumentException(StringResources.InvalidArguments);
}
}
catch (Exception exception)
{
LogException(TraceType, exception);
FabricException fabricException = exception as FabricException;
if (fabricException != null)
{
throw;
}
throw new ArgumentException(StringResources.InvalidArguments);
}
return(fabricUri);
}
private static int GetFabricErrorCodeOrEFail(Exception e)
{
int errorCode = 0;
if (!ImageBuilderExe.ManagedToNativeExceptionTranslation.TryGetValue(e.GetType(), out errorCode))
{
FabricException fabricException = e as FabricException;
if (fabricException != null)
{
errorCode = (int)fabricException.ErrorCode;
}
else
{
// TryGetValue() sets the out parameter to default(T) if the entry is not found
//
errorCode = unchecked ((int)NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_IMAGEBUILDER_UNEXPECTED_ERROR);
}
}
return(errorCode);
}
private static void CallAsyncVerifyError(KtlInteropTestComponent target, FabricErrorCode begin = FabricErrorCode.Unknown, FabricErrorCode end = FabricErrorCode.Unknown)
{
FabricException beginException = null;
FabricException endException = null;
Task task = null;
try
{
task = target.OperationAsync((int)begin, (int)end);
}
catch (FabricException e)
{
beginException = e;
}
if (begin != FabricErrorCode.Unknown)
{
Assert.IsNotNull(beginException);
Assert.AreEqual(beginException.ErrorCode, begin);
return;
}
try
{
task.Wait();
}
catch (AggregateException e)
{
endException = e.InnerException as FabricException;
}
if (end != FabricErrorCode.Unknown)
{
Assert.IsNotNull(endException);
Assert.AreEqual(endException.ErrorCode, end);
}
}
private async Task StartContainerAsync()
{
FabricException opEx = null;
try
{
await Utility.ExecuteWithRetriesAsync(
(operationTimeout) =>
{
return(this.activator.Client.ContainerOperation.StartContainerAsync(
this.activationArgs.ContainerDescription.ContainerName,
operationTimeout));
},
$"StartContainerAsync_{this.ContainerName}",
TraceType,
HostingConfig.Config.DockerRequestTimeout,
this.timeoutHelper.RemainingTime);
HostingTrace.Source.WriteInfo(
TraceType,
"Starting {0} succeeded.",
BuildContainerInfoMessage());
}
catch (Exception ex)
{
var errorMessage = string.Format(
"Failed to start Container. {0}",
this.BuildErrorMessage(ex));
HostingTrace.Source.WriteError(TraceType, errorMessage);
opEx = new FabricException(errorMessage, FabricErrorCode.InvalidOperation);
}
if (opEx != null)
{
await this.CleanupOnErrorAsync(opEx);
}
}
private async Task CleanupOnErrorAsync(FabricException originalEx)
{
HostingTrace.Source.WriteWarning(
TraceType,
"CleanupOnErrorAsync(): Deactivating {0}",
BuildContainerInfoMessage());
try
{
var deactivationArg = new ContainerDeactivationArgs()
{
ContainerName = this.activationArgs.ContainerDescription.ContainerName,
ConfiguredForAutoRemove = this.activationArgs.ContainerDescription.AutoRemove,
IsContainerRoot = this.activationArgs.ContainerDescription.IsContainerRoot,
CgroupName = this.activationArgs.ProcessDescription.CgroupName,
IsGracefulDeactivation = false
};
await Utility.ExecuteWithRetriesAsync(
(operationTimeout) =>
{
return(this.activator.DeactivateContainerAsync(
deactivationArg,
operationTimeout));
},
$"CleanupOnErrorAsync_DeactivateContainerAsync_{this.ContainerName}",
TraceType,
HostingConfig.Config.DockerRequestTimeout,
this.timeoutHelper.RemainingTime);
}
catch (FabricException)
{
// Ignore.
}
throw originalEx;
}
public OperationResult <WinFabricPropertyBatchResult> ConvertToWinFabricPropertyBatchResult(uint errorCode, PropertyBatchResult batchResult)
{
var result = new OperationResult <WinFabricPropertyBatchResult>(errorCode);
if (batchResult != null)
{
int id = this.BatchResultHolder.StoreObject(batchResult);
FabricErrorCode indexErrorCode = 0;
if (batchResult.FailedOperationException != null)
{
if (batchResult.FailedOperationException is OperationCanceledException || batchResult.FailedOperationException is TimeoutException)
{
throw batchResult.FailedOperationException;
}
FabricException exception = batchResult.FailedOperationException as FabricException;
if (exception == null)
{
throw new ArgumentException("batchResult.FailedOperationException {0} should be a FabricException", batchResult.FailedOperationException);
}
indexErrorCode = exception.ErrorCode;
}
WinFabricPropertyBatchResult testBatch = new WinFabricPropertyBatchResult(id)
{
Result = batchResult,
FailedOperationErrorCode = indexErrorCode,
};
result = FabricClientState.CreateOperationResultFromNativeErrorCode <WinFabricPropertyBatchResult>(testBatch);
}
return(result);
}
private bool HandleRollback(Guid operationId, Exception exception, List <FabricErrorCode> expectedErrorCodes)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Enter HandleRollback", operationId);
if (exception == null)
{
string error = string.Format(CultureInfo.InvariantCulture, "{0} - exception is null", operationId);
ReleaseAssert.Failfast(error);
}
if (exception is FatalException)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - HandleRollback - FatalException", operationId);
return(false);
}
if (exception is RollbackAndRetryCommandException)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - HandleRollback - RollbackAndRetryCommandException", operationId);
return(true);
}
// Use for internal testing
if (exception is InvalidOperationException)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - HandleRollback - InvalidOperationException", operationId);
return(false);
}
if (exception is OperationCanceledException)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Retrying after OperationCanceledException", operationId);
return(true);
}
if (exception is FabricInvalidAddressException)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Retrying after {1}", operationId, exception);
return(true);
}
// Add to this list
if (exception is FabricException)
{
FabricException fabricException = exception as FabricException;
if ((fabricException.ErrorCode == FabricErrorCode.InvalidReplicaStateForReplicaOperation) || // eg restart of down replica
(fabricException.ErrorCode == FabricErrorCode.InvalidAddress) ||
(fabricException.ErrorCode == FabricErrorCode.NotReady) ||
(fabricException.ErrorCode == FabricErrorCode.OperationTimedOut) ||
(fabricException.ErrorCode == FabricErrorCode.ReplicaDoesNotExist))
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Retrying after {1}", operationId, exception);
return(true);
}
}
FabricException innerException = exception as FabricException;
if (innerException == null)
{
// malformed exception
string error = string.Format(CultureInfo.InvariantCulture, "{0} - Exception type was not expected: '{1}'", operationId, exception);
TestabilityTrace.TraceSource.WriteError(TraceType, "{0}", error);
return(false);
}
if (innerException is FabricTransientException)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Retrying after inner ex {1}", operationId, exception);
return(true);
}
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Bottom of HandleRollback, printing exception info", operationId);
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Exception: {1}", operationId, exception);
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Inner ex {1}", operationId, innerException);
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Inner ex error code {1}", operationId, innerException.ErrorCode);
// nonretryable
return(false);
}
private static bool HandleException(Exception e, FabricClientRetryErrors errors, out bool retryElseSuccess)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "HandleException(): e='{0}', HResult='{0}'", e.ToString(), e.HResult);
FabricException fabricException = e as FabricException;
if (fabricException != null)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "HandleException(): errorCode='{0}'", fabricException.ErrorCode);
}
Exception currentEx = e;
for (int i = 0; i < 5 && currentEx != null; i++)
{
if (errors.RetryableExceptions.Contains(currentEx.GetType()))
{
retryElseSuccess = true /*retry*/;
return(true);
}
currentEx = currentEx.InnerException;
}
if (fabricException != null && errors.RetryableFabricErrorCodes.Contains(fabricException.ErrorCode))
{
retryElseSuccess = true /*retry*/;
return(true);
}
if (errors.RetrySuccessExceptions.Contains(e.GetType()))
{
retryElseSuccess = false /*success*/;
return(true);
}
if (fabricException != null && errors.RetrySuccessFabricErrorCodes.Contains(fabricException.ErrorCode))
{
retryElseSuccess = false /*success*/;
return(true);
}
if (e.GetType() == typeof(FabricTransientException))
{
retryElseSuccess = true /*retry*/;
return(true);
}
if (fabricException != null && fabricException.InnerException != null)
{
var ex = fabricException.InnerException;
if (errors.InternalRetrySuccessFabricErrorCodes.Contains((uint)ex.HResult))
{
retryElseSuccess = false /*success*/;
return(true);
}
}
retryElseSuccess = false;
return(false);
}
private async Task AddOutboundConnectivityToContainerAsync()
{
// If container is using open network only on windows, we need additional set up before and after
// starting the container. This is a mitigation for no support for outbound connectivity from
// open network based containers.
if ((this.activationArgs.ContainerDescription.ContainerNetworkConfig.NetworkType & ContainerNetworkType.Open) == ContainerNetworkType.Open &&
!string.IsNullOrEmpty(this.activationArgs.ContainerDescription.ContainerNetworkConfig.OpenNetworkAssignedIp) &&
(this.activationArgs.ContainerDescription.ContainerNetworkConfig.NetworkType & ContainerNetworkType.Isolated) != ContainerNetworkType.Isolated)
{
var netwrokParams = new NetworkConnectConfig()
{
Container = this.containerId
};
FabricException opEx = null;
try
{
await Utility.ExecuteWithRetryOnTimeoutAsync(
(operationTimeout) =>
{
return(this.activator.Client.NetworkOperation.ConnectNetworkAsync(
"nat",
netwrokParams,
operationTimeout));
},
$"AddOutboundConnectivityToContainerAsync_{this.ContainerName}",
TraceType,
HostingConfig.Config.DockerRequestTimeout,
this.timeoutHelper.RemainingTime);
HostingTrace.Source.WriteNoise(
TraceType,
"Adding outbound connectivity to {0} succeeded.",
BuildContainerInfoMessage());
}
catch (Exception ex)
{
var errorMessage = string.Format(
"Adding outbound connectivity failed for {0}",
Utility.BuildErrorMessage(
activationArgs.ContainerDescription.ContainerName,
activationArgs.ContainerDescription.ApplicationId,
activationArgs.ContainerDescription.ApplicationName,
ex));
HostingTrace.Source.WriteError(TraceType, errorMessage);
opEx = new FabricException(errorMessage, FabricErrorCode.InvalidOperation);
}
if (opEx != null)
{
await Utility.CleanupOnErrorAsync(
TraceType,
activationArgs.ContainerDescription.AutoRemove,
activationArgs.ContainerDescription.IsContainerRoot,
activationArgs.ProcessDescription.CgroupName,
activationArgs.ContainerDescription.ContainerName,
activationArgs.ContainerDescription.ApplicationId,
activator,
timeoutHelper,
opEx);
}
}
}
/// <summary>
///
/// </summary>
/// <param name="fabricException"></param>
public FabricError(FabricException fabricException)
{
this.Code = (NativeTypes.FABRIC_ERROR_CODE)fabricException.ErrorCode;
this.Message = fabricException.Message;
}
internal async Task ExecuteUpdateRouteCommandAsync()
{
var strb = new StringBuilder();
foreach (var gatewayIpAddress in this.gatewayIpAddresses)
{
var command = string.Format(UpdateRoutesCommand, gatewayIpAddress, gatewayIpAddress);
if (strb.Length > 0)
{
strb.AppendFormat("&{0}", command);
}
else
{
strb.Append(command);
}
}
// Route changes on RS3 builds need to be executed with higher privilege.
ContainerExecConfig execConfig = new ContainerExecConfig()
{
AttachStdin = true,
AttachStdout = true,
AttachStderr = true,
Privileged = false,
Tty = true,
Cmd = new List <string>()
{
"cmd.exe", "/c", strb.ToString()
},
User = "******"
};
FabricException opEx = null;
opEx = await ExecuteUpdateRouteCommandAsync(execConfig);
if (opEx != null)
{
// reset opEx so that we can deactivate container if second try fails.
opEx = null;
// Execute without higher privilege.
execConfig = new ContainerExecConfig()
{
AttachStdin = true,
AttachStdout = true,
AttachStderr = true,
Privileged = false,
Tty = true,
Cmd = new List <string>()
{
"cmd.exe", "/c", strb.ToString()
}
};
opEx = await ExecuteUpdateRouteCommandAsync(execConfig);
if (opEx != null)
{
await Utility.CleanupOnErrorAsync(
traceType,
autoRemove,
isContainerRoot,
cgroupName,
containerName,
applicationId,
activator,
timeoutHelper,
opEx);
}
}
}
private async Task <bool> IsFabricEntityExisting(string fabricUri, TimeSpan timeout, CancellationToken cancellationToken)
{
string applicationUri = null;
string serviceUri = null;
string partitionId = null;
FabricBackupResourceType fabricBackupResourceType =
UtilityHelper.GetApplicationAndServicePartitionUri(fabricUri, out applicationUri, out serviceUri,
out partitionId);
switch (fabricBackupResourceType)
{
case FabricBackupResourceType.ApplicationUri:
try
{
return
(await FabricClientHelper.ValidateApplicationUri(applicationUri, timeout, cancellationToken));
}
catch (Exception exception)
{
FabricException fabricException = exception as FabricException;
if (fabricException != null && fabricException.ErrorCode == FabricErrorCode.ApplicationNotFound)
{
return(false);
}
}
break;
case FabricBackupResourceType.ServiceUri:
try
{
return
(await FabricClientHelper.ValidateServiceUri(serviceUri, timeout, cancellationToken));
}
catch (Exception exception)
{
FabricException fabricException = exception as FabricException;
if (fabricException != null &&
(fabricException.ErrorCode == FabricErrorCode.ServiceNotFound || fabricException.ErrorCode == FabricErrorCode.ApplicationNotFound))
{
return(false);
}
}
break;
case FabricBackupResourceType.PartitionUri:
try
{
return
(await FabricClientHelper.ValidatePartition(serviceUri, partitionId, timeout, cancellationToken));
}
catch (Exception exception)
{
FabricException fabricException = exception as FabricException;
if (fabricException != null && fabricException.ErrorCode == FabricErrorCode.PartitionNotFound)
{
return(false);
}
}
break;
}
return(true);
}
public override async Task <ActionStateBase> RunAsync(CancellationToken cancellationToken, ServiceInternalFaultInfo serviceInternalFaultInfo)
{
NodeCommandState state = Convert(this.State);
// The return value is ignored, this is just being used to check if the RemoveNodeState was called.
Node queriedNode = await FaultAnalysisServiceUtility.GetNodeInfoAsync(
this.State.OperationId,
this.FabricClient,
state.Info.NodeName,
this.action.Partition,
this.action.StateManager,
this.action.StoppedNodeTable,
this.RequestTimeout,
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - calling StartNodeUsingNodeNameAsync, ApiInputNodeInstanceId={1}", this.State.OperationId, state.Info.InputNodeInstanceId);
Exception exception = null;
try
{
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.FaultManager.StartNodeUsingNodeNameAsync(
state.Info.NodeName,
state.Info.InputNodeInstanceId,
null,
0,
this.RequestTimeout,
cancellationToken),
FabricClientRetryErrors.StartNodeErrors.Value,
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(StepBase.TraceType, "{0} - StartNodeUsingNodeNameAsync threw {1}", this.State.OperationId, e);
exception = e;
}
cancellationToken.ThrowIfCancellationRequested();
SuccessRetryOrFail status = SuccessRetryOrFail.Invalid;
if (exception != null)
{
FabricException fe = exception as FabricException;
if (fe != null)
{
status = this.HandleFabricException(fe, state);
}
else
{
TestabilityTrace.TraceSource.WriteWarning(StepBase.TraceType, "{0} - StartNodeUsingNodeNameAsync threw non-FabricException with ErrorCode={1}", this.State.OperationId, exception);
status = SuccessRetryOrFail.RetryStep;
}
}
else
{
// success
status = SuccessRetryOrFail.Success;
await FaultAnalysisServiceUtility.SetStoppedNodeStateAsync(
this.action.State.OperationId,
this.action.Partition,
this.action.StateManager,
this.action.StoppedNodeTable,
state.Info.NodeName,
false,
cancellationToken).ConfigureAwait(false);
}
ActionTest.PerformInternalServiceFaultIfRequested(this.State.OperationId, serviceInternalFaultInfo, this.State, cancellationToken, true);
if (status == SuccessRetryOrFail.RetryStep)
{
throw new RetrySameStepException("retrystep", exception);
}
else if (status == SuccessRetryOrFail.Fail)
{
throw new FatalException("fatal", exception);
}
else if (status == SuccessRetryOrFail.Success)
{
// no-op
}
else
{
ReleaseAssert.Failfast(string.Format(CultureInfo.InvariantCulture, "This condition should not have been hit. OperationId: {0}", this.State.OperationId));
}
await this.ValidateAsync(this.FabricClient, state, cancellationToken).ConfigureAwait(false);
state.StateProgress.Push(StepStateNames.CompletedSuccessfully);
return(state);
}
private bool IsFabricElementNotFound(FabricException exp)
{
return(exp.ErrorCode == FabricErrorCode.FabricHealthEntityNotFound);
}
internal void HandleRunAsyncUnexpectedFabricException(IServicePartition partition, FabricException fex)
{
ServiceTrace.Source.WriteErrorWithId(
this.traceType + ApiErrorTraceTypeSuffix,
this.traceId,
"RunAsync failed due to an unhandled FabricException causing replica to fault: {0}",
fex.ToString());
this.ReportRunAsyncUnexpectedExceptionHealth(partition, fex);
partition.ReportFault(FaultType.Transient);
}
private static void ValidateUnknownErrorCodeAndHResult(FabricException ex)
{
Assert.AreEqual(ex.ErrorCode, FabricErrorCode.Unknown);
Assert.AreNotEqual(ex.HResult, 0);
}
