public void FabricExceptionListContainsCheck() { var errorCode = FabricErrorCode.FabricHealthEntityNotFound; var list = new List <FabricErrorCode>(); list.Add(errorCode); var ex = new FabricException("Exception With Non-Zero ErrorCode", errorCode); Assert.IsTrue(list.Contains(errorCode)); Assert.AreEqual(errorCode, ex.ErrorCode); Assert.IsTrue(list.Contains(ex.ErrorCode)); }
private static string CreateThreadErrorMessage(Exception ex, bool includeStackTrace = true) { const int MaxCharactersOfMessageToCopy = 512; // there are limits such as 4k for the health report size and 4k for the interop // use a smaller limit here const int MaxLength = 4000; ExceptionStringBuilder sb = new ExceptionStringBuilder(MaxLength); int HResult = ex.HResult; string message = ex.Message; System.Type type = ex.GetType(); sb.Append(GetExceptionInfoString(type, HResult)); sb.AppendLine(); FabricException fabricEx = ex as FabricException; if (fabricEx != null && (fabricEx.InnerException != null)) { // Add only 256 bytes of the message to prevent it from overflowing the health report sb.Append(message, MaxCharactersOfMessageToCopy / 2); sb.AppendLine(); Exception inner = fabricEx.InnerException; sb.Append(GetExceptionInfoString(inner.GetType(), inner.HResult)); sb.AppendLine(); // Outer exception + Inner exception message should be upto 512 bytes to prevent it from overflowing the health report sb.Append(inner.Message, MaxCharactersOfMessageToCopy); } else { // Add only 512 bytes of the message to prevent it from overflowing the health report sb.Append(message, MaxCharactersOfMessageToCopy); } if (includeStackTrace) { // Add the stack trace sb.AppendLine(); sb.Append(ex.StackTrace); } return(sb.ToString()); }
private SuccessRetryOrFail HandleFabricException(FabricException fe, NodeCommandState state) { SuccessRetryOrFail status = SuccessRetryOrFail.Invalid; TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - StartNodeUsingNodeNameAsync threw FabricException with ErrorCode={1}", this.State.OperationId, fe.ErrorCode); if (fe.ErrorCode == FabricErrorCode.InstanceIdMismatch) { status = SuccessRetryOrFail.Fail; } else if (fe.ErrorCode == FabricErrorCode.NodeIsUp) { status = SuccessRetryOrFail.Success; } else if (fe.ErrorCode == FabricErrorCode.NodeHasNotStoppedYet) { status = SuccessRetryOrFail.RetryStep; } else if (fe.ErrorCode == FabricErrorCode.InvalidAddress) { if (state.Info.InitialQueriedNodeStatus != NodeStatus.Down && state.Info.NodeWasInitiallyInStoppedState == false) { // This is a (probably unlikely) case that may happen if // 1. The request reaches the target node, which is up and is processed. // 2. The response is dropped // 3. The request is retried with enough delay such that the node has now transitioned from up to stopped // So, we say that if the initial conditions were valid and we get InvalidAddress, then consider it successful. status = SuccessRetryOrFail.Success; } else { // The preconditions passed, but the node is down now, so something out of band happened. status = SuccessRetryOrFail.RetryStep; } } else if (fe.ErrorCode == FabricErrorCode.NodeNotFound) { // Always fatal string nodeNotFoundErrorMessage = string.Format(CultureInfo.InvariantCulture, "{0} - node {1} was not found", state.OperationId, state.Info.NodeName); TestabilityTrace.TraceSource.WriteError(StepBase.TraceType, nodeNotFoundErrorMessage); status = SuccessRetryOrFail.Fail; } else { status = SuccessRetryOrFail.RetryStep; } return(status); }
private async Task <FabricException> ExecuteUpdateRouteCommandAsync(ContainerExecConfig execConfig) { FabricException opEx = null; try { await ExecuteUpdateRouteCommandWithRetriesAsync(execConfig); } catch (FabricException ex) { opEx = ex; } return(opEx); }
private static bool TryHandleFabricException(FabricException fabricException, OperationRetrySettings retrySettings, out ExceptionHandlingResult result) { if (fabricException is FabricCannotConnectException || fabricException is FabricEndpointNotFoundException) { result = new ExceptionHandlingRetryResult(fabricException, false, retrySettings, int.MaxValue); return(true); } if (fabricException.ErrorCode.Equals(FabricErrorCode.ServiceTooBusy)) { result = new ExceptionHandlingRetryResult(fabricException, true, retrySettings, int.MaxValue); return(true); } result = null; return(false); }
public bool IsRetryable(Exception e) { Exception currentEx = e; for (int i = 0; i < 5 && currentEx != null; i++) { if (this.RetryableExceptions.Contains(currentEx.GetType())) { return(true); } currentEx = currentEx.InnerException; } FabricException fabricException = e as FabricException; if (fabricException != null && this.RetryableFabricErrorCodes.Contains(fabricException.ErrorCode)) { return(true); } if (this.RetrySuccessExceptions.Contains(e.GetType())) { return(true); } if (fabricException != null && this.RetrySuccessFabricErrorCodes.Contains(fabricException.ErrorCode)) { return(true); } if (e.GetType() == typeof(FabricTransientException)) { return(true); } if (fabricException != null && fabricException.InnerException != null) { var ex = fabricException.InnerException; if (this.InternalRetrySuccessFabricErrorCodes.Contains((uint)ex.HResult)) { return(true); } } return(false); }
private static bool HandleException(Exception e, FabricClientRetryErrors errors, out bool retryElseSuccess) { FabricException fabricException = e as FabricException; if (errors.RetryableExceptions.Contains(e.GetType())) { retryElseSuccess = true /*retry*/; return(true); } if (fabricException != null && errors.RetryableFabricErrorCodes.Contains(fabricException.ErrorCode)) { retryElseSuccess = true /*retry*/; return(true); } if (errors.RetrySuccessExceptions.Contains(e.GetType())) { retryElseSuccess = false /*success*/; return(true); } if (fabricException != null && errors.RetrySuccessFabricErrorCodes.Contains(fabricException.ErrorCode)) { retryElseSuccess = false /*success*/; return(true); } if (e.GetType() == typeof(FabricTransientException)) { retryElseSuccess = true /*retry*/; return(true); } if (fabricException != null && fabricException.InnerException != null) { var ex = fabricException.InnerException as COMException; if (ex != null && errors.InternalRetrySuccessFabricErrorCodes.Contains((uint)ex.ErrorCode)) { retryElseSuccess = false /*success*/; return(true); } } retryElseSuccess = false; return(false); }
internal static async Task CleanupOnErrorAsync( string traceType, bool autoRemove, bool isContainerRoot, string cgroupName, string containerName, string applicationId, IContainerActivatorService activatorService, TimeoutHelper timeoutHelper, FabricException originalEx) { HostingTrace.Source.WriteWarning( traceType, "CleanupOnErrorAsync(): Deactivating Container='{0}', AppId={1}.", containerName, applicationId); try { var deactivationArg = new ContainerDeactivationArgs() { ContainerName = containerName, ConfiguredForAutoRemove = autoRemove, IsContainerRoot = isContainerRoot, CgroupName = cgroupName, IsGracefulDeactivation = false }; await Utility.ExecuteWithRetryOnTimeoutAsync( (operationTimeout) => { return(activatorService.DeactivateContainerAsync( deactivationArg, operationTimeout)); }, $"CleanupOnErrorAsync_DeactivateContainerAsync_{containerName}", traceType, HostingConfig.Config.DockerRequestTimeout, timeoutHelper.RemainingTime); } catch (FabricException) { // Ignore. } throw originalEx; }
private async Task AddOutboundConnectivityToContainerAsync() { var netwrokParams = new NetworkConnectConfig() { Container = this.containerId }; FabricException opEx = null; try { await Utility.ExecuteWithRetriesAsync( (operationTimeout) => { return(this.activator.Client.NetworkOperation.ConnectNetworkAsync( "nat", netwrokParams, operationTimeout)); }, $"AddOutboundConnectivityToContainerAsync_{this.ContainerName}", TraceType, HostingConfig.Config.DockerRequestTimeout, this.timeoutHelper.RemainingTime); HostingTrace.Source.WriteNoise( TraceType, "Adding outbound connectivity to {0} succeeded.", BuildContainerInfoMessage()); } catch (Exception ex) { var errorMessage = string.Format( "Adding outbound connectivity failed for {0}", this.BuildErrorMessage(ex)); HostingTrace.Source.WriteError(TraceType, errorMessage); opEx = new FabricException(errorMessage, FabricErrorCode.InvalidOperation); } if (opEx != null) { await this.CleanupOnErrorAsync(opEx); } }
private SuccessRetryOrFail HandleFabricException(FabricException fe, NodeCommandState state) { SuccessRetryOrFail status = SuccessRetryOrFail.Invalid; if (fe.ErrorCode == FabricErrorCode.InstanceIdMismatch) { TestabilityTrace.TraceSource.WriteWarning(StepBase.TraceType, "{0} - StopNode api threw InstanceIdMismatch", this.State.OperationId); status = SuccessRetryOrFail.Fail; } else if (fe.ErrorCode == FabricErrorCode.InvalidAddress) { TestabilityTrace.TraceSource.WriteInfo( StepBase.TraceType, "{0} - StopNode HandleFabricException InitialQueriedNodeStatus='{1}', NodeWasInitiallyInStoppedState='{2}'", this.State.OperationId, state.Info.InitialQueriedNodeStatus, state.Info.NodeWasInitiallyInStoppedState); // If the request was valid (node was up), but the response was dropped, and then the request was retried, we might now be // sending a stop to a node that is now stopped. The request will return InvalidAddress, but this is really success. if (state.Info.InitialQueriedNodeStatus != NodeStatus.Down && state.Info.NodeWasInitiallyInStoppedState == false) { TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - StopNode api threw InvalidAddress, considering stop command successful due to preconditions", this.State.OperationId); status = SuccessRetryOrFail.Success; } else { // Invalid address is not expected here. There may have been an out of band stop using the deprecated api. We should retry until we get something else. TestabilityTrace.TraceSource.WriteWarning(StepBase.TraceType, "{0} - StopNode api threw InvalidAddress", this.State.OperationId); status = SuccessRetryOrFail.RetryStep; } } else if (fe.ErrorCode == FabricErrorCode.NodeNotFound) { status = SuccessRetryOrFail.Fail; } else { TestabilityTrace.TraceSource.WriteWarning(StepBase.TraceType, "{0} - StopNode api threw {1}", this.State.OperationId, fe); status = SuccessRetryOrFail.RetryStep; } return(status); }
internal static async Task <string> GetFabricUriFromRequstHeaderForPartitions(string fabricRequestHeader, TimeSpan timeout, CancellationToken cancellationToken) { string fabricUri; try { string[] segments = fabricRequestHeader.Split('/'); string type = segments[0]; switch (type) { case "Partitions": if (segments.Length != 2 || string.IsNullOrEmpty(segments[1])) { throw new ArgumentException(StringResources.InvalidArguments); } string serviceNameUri = await FabricClientHelper.GetFabricServiceUriFromPartitionId(segments[1], timeout, cancellationToken); string applicationNameUri = await FabricClientHelper.GetFabricApplicationUriFromServiceUri(serviceNameUri, timeout, cancellationToken); string applicationName = GetNameFromUri(applicationNameUri); string serviceName = RemoveApplicationNameUriFromServiceNameUri(applicationNameUri, serviceNameUri); fabricUri = string.Format("{0}/{1}", CreateUriFromApplicationAndServiceName(applicationName, serviceName), segments[1]); break; default: throw new ArgumentException(StringResources.InvalidArguments); } } catch (Exception exception) { LogException(TraceType, exception); FabricException fabricException = exception as FabricException; if (fabricException != null) { throw; } throw new ArgumentException(StringResources.InvalidArguments); } return(fabricUri); }
private static int GetFabricErrorCodeOrEFail(Exception e) { int errorCode = 0; if (!ImageBuilderExe.ManagedToNativeExceptionTranslation.TryGetValue(e.GetType(), out errorCode)) { FabricException fabricException = e as FabricException; if (fabricException != null) { errorCode = (int)fabricException.ErrorCode; } else { // TryGetValue() sets the out parameter to default(T) if the entry is not found // errorCode = unchecked ((int)NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_IMAGEBUILDER_UNEXPECTED_ERROR); } } return(errorCode); }
private static void CallAsyncVerifyError(KtlInteropTestComponent target, FabricErrorCode begin = FabricErrorCode.Unknown, FabricErrorCode end = FabricErrorCode.Unknown) { FabricException beginException = null; FabricException endException = null; Task task = null; try { task = target.OperationAsync((int)begin, (int)end); } catch (FabricException e) { beginException = e; } if (begin != FabricErrorCode.Unknown) { Assert.IsNotNull(beginException); Assert.AreEqual(beginException.ErrorCode, begin); return; } try { task.Wait(); } catch (AggregateException e) { endException = e.InnerException as FabricException; } if (end != FabricErrorCode.Unknown) { Assert.IsNotNull(endException); Assert.AreEqual(endException.ErrorCode, end); } }
private async Task StartContainerAsync() { FabricException opEx = null; try { await Utility.ExecuteWithRetriesAsync( (operationTimeout) => { return(this.activator.Client.ContainerOperation.StartContainerAsync( this.activationArgs.ContainerDescription.ContainerName, operationTimeout)); }, $"StartContainerAsync_{this.ContainerName}", TraceType, HostingConfig.Config.DockerRequestTimeout, this.timeoutHelper.RemainingTime); HostingTrace.Source.WriteInfo( TraceType, "Starting {0} succeeded.", BuildContainerInfoMessage()); } catch (Exception ex) { var errorMessage = string.Format( "Failed to start Container. {0}", this.BuildErrorMessage(ex)); HostingTrace.Source.WriteError(TraceType, errorMessage); opEx = new FabricException(errorMessage, FabricErrorCode.InvalidOperation); } if (opEx != null) { await this.CleanupOnErrorAsync(opEx); } }
private async Task CleanupOnErrorAsync(FabricException originalEx) { HostingTrace.Source.WriteWarning( TraceType, "CleanupOnErrorAsync(): Deactivating {0}", BuildContainerInfoMessage()); try { var deactivationArg = new ContainerDeactivationArgs() { ContainerName = this.activationArgs.ContainerDescription.ContainerName, ConfiguredForAutoRemove = this.activationArgs.ContainerDescription.AutoRemove, IsContainerRoot = this.activationArgs.ContainerDescription.IsContainerRoot, CgroupName = this.activationArgs.ProcessDescription.CgroupName, IsGracefulDeactivation = false }; await Utility.ExecuteWithRetriesAsync( (operationTimeout) => { return(this.activator.DeactivateContainerAsync( deactivationArg, operationTimeout)); }, $"CleanupOnErrorAsync_DeactivateContainerAsync_{this.ContainerName}", TraceType, HostingConfig.Config.DockerRequestTimeout, this.timeoutHelper.RemainingTime); } catch (FabricException) { // Ignore. } throw originalEx; }
public OperationResult <WinFabricPropertyBatchResult> ConvertToWinFabricPropertyBatchResult(uint errorCode, PropertyBatchResult batchResult) { var result = new OperationResult <WinFabricPropertyBatchResult>(errorCode); if (batchResult != null) { int id = this.BatchResultHolder.StoreObject(batchResult); FabricErrorCode indexErrorCode = 0; if (batchResult.FailedOperationException != null) { if (batchResult.FailedOperationException is OperationCanceledException || batchResult.FailedOperationException is TimeoutException) { throw batchResult.FailedOperationException; } FabricException exception = batchResult.FailedOperationException as FabricException; if (exception == null) { throw new ArgumentException("batchResult.FailedOperationException {0} should be a FabricException", batchResult.FailedOperationException); } indexErrorCode = exception.ErrorCode; } WinFabricPropertyBatchResult testBatch = new WinFabricPropertyBatchResult(id) { Result = batchResult, FailedOperationErrorCode = indexErrorCode, }; result = FabricClientState.CreateOperationResultFromNativeErrorCode <WinFabricPropertyBatchResult>(testBatch); } return(result); }
private bool HandleRollback(Guid operationId, Exception exception, List <FabricErrorCode> expectedErrorCodes) { TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Enter HandleRollback", operationId); if (exception == null) { string error = string.Format(CultureInfo.InvariantCulture, "{0} - exception is null", operationId); ReleaseAssert.Failfast(error); } if (exception is FatalException) { TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - HandleRollback - FatalException", operationId); return(false); } if (exception is RollbackAndRetryCommandException) { TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - HandleRollback - RollbackAndRetryCommandException", operationId); return(true); } // Use for internal testing if (exception is InvalidOperationException) { TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - HandleRollback - InvalidOperationException", operationId); return(false); } if (exception is OperationCanceledException) { TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Retrying after OperationCanceledException", operationId); return(true); } if (exception is FabricInvalidAddressException) { TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Retrying after {1}", operationId, exception); return(true); } // Add to this list if (exception is FabricException) { FabricException fabricException = exception as FabricException; if ((fabricException.ErrorCode == FabricErrorCode.InvalidReplicaStateForReplicaOperation) || // eg restart of down replica (fabricException.ErrorCode == FabricErrorCode.InvalidAddress) || (fabricException.ErrorCode == FabricErrorCode.NotReady) || (fabricException.ErrorCode == FabricErrorCode.OperationTimedOut) || (fabricException.ErrorCode == FabricErrorCode.ReplicaDoesNotExist)) { TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Retrying after {1}", operationId, exception); return(true); } } FabricException innerException = exception as FabricException; if (innerException == null) { // malformed exception string error = string.Format(CultureInfo.InvariantCulture, "{0} - Exception type was not expected: '{1}'", operationId, exception); TestabilityTrace.TraceSource.WriteError(TraceType, "{0}", error); return(false); } if (innerException is FabricTransientException) { TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Retrying after inner ex {1}", operationId, exception); return(true); } TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Bottom of HandleRollback, printing exception info", operationId); TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Exception: {1}", operationId, exception); TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Inner ex {1}", operationId, innerException); TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Inner ex error code {1}", operationId, innerException.ErrorCode); // nonretryable return(false); }
private static bool HandleException(Exception e, FabricClientRetryErrors errors, out bool retryElseSuccess) { TestabilityTrace.TraceSource.WriteInfo(TraceType, "HandleException(): e='{0}', HResult='{0}'", e.ToString(), e.HResult); FabricException fabricException = e as FabricException; if (fabricException != null) { TestabilityTrace.TraceSource.WriteInfo(TraceType, "HandleException(): errorCode='{0}'", fabricException.ErrorCode); } Exception currentEx = e; for (int i = 0; i < 5 && currentEx != null; i++) { if (errors.RetryableExceptions.Contains(currentEx.GetType())) { retryElseSuccess = true /*retry*/; return(true); } currentEx = currentEx.InnerException; } if (fabricException != null && errors.RetryableFabricErrorCodes.Contains(fabricException.ErrorCode)) { retryElseSuccess = true /*retry*/; return(true); } if (errors.RetrySuccessExceptions.Contains(e.GetType())) { retryElseSuccess = false /*success*/; return(true); } if (fabricException != null && errors.RetrySuccessFabricErrorCodes.Contains(fabricException.ErrorCode)) { retryElseSuccess = false /*success*/; return(true); } if (e.GetType() == typeof(FabricTransientException)) { retryElseSuccess = true /*retry*/; return(true); } if (fabricException != null && fabricException.InnerException != null) { var ex = fabricException.InnerException; if (errors.InternalRetrySuccessFabricErrorCodes.Contains((uint)ex.HResult)) { retryElseSuccess = false /*success*/; return(true); } } retryElseSuccess = false; return(false); }
private async Task AddOutboundConnectivityToContainerAsync() { // If container is using open network only on windows, we need additional set up before and after // starting the container. This is a mitigation for no support for outbound connectivity from // open network based containers. if ((this.activationArgs.ContainerDescription.ContainerNetworkConfig.NetworkType & ContainerNetworkType.Open) == ContainerNetworkType.Open && !string.IsNullOrEmpty(this.activationArgs.ContainerDescription.ContainerNetworkConfig.OpenNetworkAssignedIp) && (this.activationArgs.ContainerDescription.ContainerNetworkConfig.NetworkType & ContainerNetworkType.Isolated) != ContainerNetworkType.Isolated) { var netwrokParams = new NetworkConnectConfig() { Container = this.containerId }; FabricException opEx = null; try { await Utility.ExecuteWithRetryOnTimeoutAsync( (operationTimeout) => { return(this.activator.Client.NetworkOperation.ConnectNetworkAsync( "nat", netwrokParams, operationTimeout)); }, $"AddOutboundConnectivityToContainerAsync_{this.ContainerName}", TraceType, HostingConfig.Config.DockerRequestTimeout, this.timeoutHelper.RemainingTime); HostingTrace.Source.WriteNoise( TraceType, "Adding outbound connectivity to {0} succeeded.", BuildContainerInfoMessage()); } catch (Exception ex) { var errorMessage = string.Format( "Adding outbound connectivity failed for {0}", Utility.BuildErrorMessage( activationArgs.ContainerDescription.ContainerName, activationArgs.ContainerDescription.ApplicationId, activationArgs.ContainerDescription.ApplicationName, ex)); HostingTrace.Source.WriteError(TraceType, errorMessage); opEx = new FabricException(errorMessage, FabricErrorCode.InvalidOperation); } if (opEx != null) { await Utility.CleanupOnErrorAsync( TraceType, activationArgs.ContainerDescription.AutoRemove, activationArgs.ContainerDescription.IsContainerRoot, activationArgs.ProcessDescription.CgroupName, activationArgs.ContainerDescription.ContainerName, activationArgs.ContainerDescription.ApplicationId, activator, timeoutHelper, opEx); } } }
/// <summary> /// /// </summary> /// <param name="fabricException"></param> public FabricError(FabricException fabricException) { this.Code = (NativeTypes.FABRIC_ERROR_CODE)fabricException.ErrorCode; this.Message = fabricException.Message; }
internal async Task ExecuteUpdateRouteCommandAsync() { var strb = new StringBuilder(); foreach (var gatewayIpAddress in this.gatewayIpAddresses) { var command = string.Format(UpdateRoutesCommand, gatewayIpAddress, gatewayIpAddress); if (strb.Length > 0) { strb.AppendFormat("&{0}", command); } else { strb.Append(command); } } // Route changes on RS3 builds need to be executed with higher privilege. ContainerExecConfig execConfig = new ContainerExecConfig() { AttachStdin = true, AttachStdout = true, AttachStderr = true, Privileged = false, Tty = true, Cmd = new List <string>() { "cmd.exe", "/c", strb.ToString() }, User = "******" }; FabricException opEx = null; opEx = await ExecuteUpdateRouteCommandAsync(execConfig); if (opEx != null) { // reset opEx so that we can deactivate container if second try fails. opEx = null; // Execute without higher privilege. execConfig = new ContainerExecConfig() { AttachStdin = true, AttachStdout = true, AttachStderr = true, Privileged = false, Tty = true, Cmd = new List <string>() { "cmd.exe", "/c", strb.ToString() } }; opEx = await ExecuteUpdateRouteCommandAsync(execConfig); if (opEx != null) { await Utility.CleanupOnErrorAsync( traceType, autoRemove, isContainerRoot, cgroupName, containerName, applicationId, activator, timeoutHelper, opEx); } } }
private async Task <bool> IsFabricEntityExisting(string fabricUri, TimeSpan timeout, CancellationToken cancellationToken) { string applicationUri = null; string serviceUri = null; string partitionId = null; FabricBackupResourceType fabricBackupResourceType = UtilityHelper.GetApplicationAndServicePartitionUri(fabricUri, out applicationUri, out serviceUri, out partitionId); switch (fabricBackupResourceType) { case FabricBackupResourceType.ApplicationUri: try { return (await FabricClientHelper.ValidateApplicationUri(applicationUri, timeout, cancellationToken)); } catch (Exception exception) { FabricException fabricException = exception as FabricException; if (fabricException != null && fabricException.ErrorCode == FabricErrorCode.ApplicationNotFound) { return(false); } } break; case FabricBackupResourceType.ServiceUri: try { return (await FabricClientHelper.ValidateServiceUri(serviceUri, timeout, cancellationToken)); } catch (Exception exception) { FabricException fabricException = exception as FabricException; if (fabricException != null && (fabricException.ErrorCode == FabricErrorCode.ServiceNotFound || fabricException.ErrorCode == FabricErrorCode.ApplicationNotFound)) { return(false); } } break; case FabricBackupResourceType.PartitionUri: try { return (await FabricClientHelper.ValidatePartition(serviceUri, partitionId, timeout, cancellationToken)); } catch (Exception exception) { FabricException fabricException = exception as FabricException; if (fabricException != null && fabricException.ErrorCode == FabricErrorCode.PartitionNotFound) { return(false); } } break; } return(true); }
public override async Task <ActionStateBase> RunAsync(CancellationToken cancellationToken, ServiceInternalFaultInfo serviceInternalFaultInfo) { NodeCommandState state = Convert(this.State); // The return value is ignored, this is just being used to check if the RemoveNodeState was called. Node queriedNode = await FaultAnalysisServiceUtility.GetNodeInfoAsync( this.State.OperationId, this.FabricClient, state.Info.NodeName, this.action.Partition, this.action.StateManager, this.action.StoppedNodeTable, this.RequestTimeout, this.OperationTimeout, cancellationToken).ConfigureAwait(false); TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - calling StartNodeUsingNodeNameAsync, ApiInputNodeInstanceId={1}", this.State.OperationId, state.Info.InputNodeInstanceId); Exception exception = null; try { await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => this.FabricClient.FaultManager.StartNodeUsingNodeNameAsync( state.Info.NodeName, state.Info.InputNodeInstanceId, null, 0, this.RequestTimeout, cancellationToken), FabricClientRetryErrors.StartNodeErrors.Value, this.OperationTimeout, cancellationToken).ConfigureAwait(false); } catch (Exception e) { TestabilityTrace.TraceSource.WriteWarning(StepBase.TraceType, "{0} - StartNodeUsingNodeNameAsync threw {1}", this.State.OperationId, e); exception = e; } cancellationToken.ThrowIfCancellationRequested(); SuccessRetryOrFail status = SuccessRetryOrFail.Invalid; if (exception != null) { FabricException fe = exception as FabricException; if (fe != null) { status = this.HandleFabricException(fe, state); } else { TestabilityTrace.TraceSource.WriteWarning(StepBase.TraceType, "{0} - StartNodeUsingNodeNameAsync threw non-FabricException with ErrorCode={1}", this.State.OperationId, exception); status = SuccessRetryOrFail.RetryStep; } } else { // success status = SuccessRetryOrFail.Success; await FaultAnalysisServiceUtility.SetStoppedNodeStateAsync( this.action.State.OperationId, this.action.Partition, this.action.StateManager, this.action.StoppedNodeTable, state.Info.NodeName, false, cancellationToken).ConfigureAwait(false); } ActionTest.PerformInternalServiceFaultIfRequested(this.State.OperationId, serviceInternalFaultInfo, this.State, cancellationToken, true); if (status == SuccessRetryOrFail.RetryStep) { throw new RetrySameStepException("retrystep", exception); } else if (status == SuccessRetryOrFail.Fail) { throw new FatalException("fatal", exception); } else if (status == SuccessRetryOrFail.Success) { // no-op } else { ReleaseAssert.Failfast(string.Format(CultureInfo.InvariantCulture, "This condition should not have been hit. OperationId: {0}", this.State.OperationId)); } await this.ValidateAsync(this.FabricClient, state, cancellationToken).ConfigureAwait(false); state.StateProgress.Push(StepStateNames.CompletedSuccessfully); return(state); }
private bool IsFabricElementNotFound(FabricException exp) { return(exp.ErrorCode == FabricErrorCode.FabricHealthEntityNotFound); }
internal void HandleRunAsyncUnexpectedFabricException(IServicePartition partition, FabricException fex) { ServiceTrace.Source.WriteErrorWithId( this.traceType + ApiErrorTraceTypeSuffix, this.traceId, "RunAsync failed due to an unhandled FabricException causing replica to fault: {0}", fex.ToString()); this.ReportRunAsyncUnexpectedExceptionHealth(partition, fex); partition.ReportFault(FaultType.Transient); }
private static void ValidateUnknownErrorCodeAndHResult(FabricException ex) { Assert.AreEqual(ex.ErrorCode, FabricErrorCode.Unknown); Assert.AreNotEqual(ex.HResult, 0); }