private async Task ValidateAsync(FabricClient fc, NodeCommandState state, CancellationToken cancellationToken) { // It takes a few seconds for the node to shutdown + a few seconds for FM to find out. await Task.Delay(TimeSpan.FromSeconds(15), cancellationToken).ConfigureAwait(false); TimeoutHelper timeoutHelper = new TimeoutHelper(this.OperationTimeout); Node queriedNode = null; do { TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - start node validating node '{1}' is not down", this.State.OperationId, state.Info.InputNodeInstanceId); queriedNode = await FaultAnalysisServiceUtility.GetNodeInfoAsync( this.State.OperationId, fc, state.Info.NodeName, this.action.Partition, this.action.StateManager, this.action.StoppedNodeTable, this.RequestTimeout, this.OperationTimeout, cancellationToken).ConfigureAwait(false); if (FaultAnalysisServiceUtility.IsNodeRunning(queriedNode)) { break; } await Task.Delay(TimeSpan.FromSeconds(5.0d), cancellationToken).ConfigureAwait(false); }while (timeoutHelper.GetRemainingTime() > TimeSpan.Zero); if (!FaultAnalysisServiceUtility.IsNodeRunning(queriedNode)) { // something is wrong, retry FaultAnalysisServiceUtility.ThrowEngineRetryableException(string.Format(CultureInfo.InvariantCulture, "{0} - start node validation - node is not running yet. Status={1}", state.OperationId, queriedNode.NodeStatus)); } }
private async Task ValidateAsync(FabricClient fc, NodeCommandState state, CancellationToken cancellationToken) { // It takes a few seconds for the node to shutdown + a few seconds for FM to find out. await Task.Delay(TimeSpan.FromSeconds(15), cancellationToken).ConfigureAwait(false); TimeoutHelper timeoutHelper = new TimeoutHelper(this.OperationTimeout); Node queriedNode = null; do { TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - stop node validating node '{1}' is not up", this.State.OperationId, state.Info.InputNodeInstanceId); queriedNode = await FaultAnalysisServiceUtility.GetNodeInfoAsync( this.State.OperationId, fc, state.Info.NodeName, this.action.Partition, this.action.StateManager, this.action.StoppedNodeTable, this.RequestTimeout, this.OperationTimeout, cancellationToken).ConfigureAwait(false); if (!FaultAnalysisServiceUtility.IsNodeRunning(queriedNode)) { break; } await Task.Delay(TimeSpan.FromSeconds(5.0d), cancellationToken).ConfigureAwait(false); }while (timeoutHelper.GetRemainingTime() > TimeSpan.Zero); if (FaultAnalysisServiceUtility.IsNodeRunning(queriedNode)) { // Something is amiss. The api returned success, but we're not reaching the desired state. It might be something out of band happened. This is best effort, so don't fail. TestabilityTrace.TraceSource.WriteWarning(StepBase.TraceType, "{0} - node '{1}' did not reach desired state in {2}", this.State.OperationId, state.Info.InputNodeInstanceId, this.OperationTimeout); } }
public override async Task <ActionStateBase> RunAsync(CancellationToken cancellationToken, ServiceInternalFaultInfo serviceInternalFaultInfo) { NodeCommandState state = Convert(this.State); Node queriedNode = await FaultAnalysisServiceUtility.GetNodeInfoAsync( this.State.OperationId, this.FabricClient, state.Info.NodeName, this.action.Partition, this.action.StateManager, this.action.StoppedNodeTable, this.RequestTimeout, this.OperationTimeout, cancellationToken).ConfigureAwait(false); TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - StartNode LookingUpState reading RD", this.State.OperationId); bool isStopped = await FaultAnalysisServiceUtility.ReadStoppedNodeStateAsync( this.State.OperationId, this.action.Partition, this.action.StateManager, this.action.StoppedNodeTable, state.Info.NodeName, cancellationToken).ConfigureAwait(false); if (FaultAnalysisServiceUtility.IsNodeRunning(queriedNode)) { if (!isStopped) { // For illustration, if you just called StartNodeUsingNodeNameAsync() in this situation w/o checking first, you'd either get instance mismatch or node has not stopped yet // Note: this is different than the logic in the PerformingActions step (the former does not check instance id, the latter does), which is after the call to StartNodeUsingNodeNameAsync(), because // this is a precondition check. Exception nodeAlreadyUp = FaultAnalysisServiceUtility.CreateException( TraceType, NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_NODE_IS_UP, string.Format(CultureInfo.InvariantCulture, "Node {0} already started", state.Info.NodeName), FabricErrorCode.NodeIsUp); throw new FatalException("fatal", nodeAlreadyUp); } else { // The only way this can happen is OOB start. FAS should fix it's incorrect state then fail the command with // node already up. TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - StartNode LookingUpState setting RD entry for node {1} to not stopped", this.State.OperationId, state.Info.NodeName); await FaultAnalysisServiceUtility.SetStoppedNodeStateAsync( this.action.State.OperationId, this.action.Partition, this.action.StateManager, this.action.StoppedNodeTable, queriedNode.NodeName, false, cancellationToken).ConfigureAwait(false); Exception nodeIsUp = FaultAnalysisServiceUtility.CreateException( TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_NODE_IS_UP, string.Format(CultureInfo.InvariantCulture, "Node {0} is up", state.Info.NodeName)); throw new FatalException("fatal", nodeIsUp); } } else if (queriedNode.NodeStatus == NodeStatus.Down && !isStopped) { // This is a special scenario that can happen if: // 1) There was an OOB stop using the old api // 2) A node went down (not stopped, down) // Don't handle this, return node down. Exception nodeIsDown = FaultAnalysisServiceUtility.CreateException( TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_NODE_IS_DOWN, string.Format(CultureInfo.InvariantCulture, "Node {0} is down", state.Info.NodeName)); throw new FatalException("fatal", nodeIsDown); } state.Info.InitialQueriedNodeStatus = queriedNode.NodeStatus; state.Info.NodeWasInitiallyInStoppedState = isStopped; state.StateProgress.Push(StepStateNames.PerformingActions); return(state); }