private async Task ValidateAsync(FabricClient fc, NodeCommandState state, CancellationToken cancellationToken) { // It takes a few seconds for the node to shutdown + a few seconds for FM to find out. await Task.Delay(TimeSpan.FromSeconds(15), cancellationToken).ConfigureAwait(false); TimeoutHelper timeoutHelper = new TimeoutHelper(this.OperationTimeout); Node queriedNode = null; do { TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - start node validating node '{1}' is not down", this.State.OperationId, state.Info.InputNodeInstanceId); queriedNode = await FaultAnalysisServiceUtility.GetNodeInfoAsync( this.State.OperationId, fc, state.Info.NodeName, this.action.Partition, this.action.StateManager, this.action.StoppedNodeTable, this.RequestTimeout, this.OperationTimeout, cancellationToken).ConfigureAwait(false); if (FaultAnalysisServiceUtility.IsNodeRunning(queriedNode)) { break; } await Task.Delay(TimeSpan.FromSeconds(5.0d), cancellationToken).ConfigureAwait(false); }while (timeoutHelper.GetRemainingTime() > TimeSpan.Zero); if (!FaultAnalysisServiceUtility.IsNodeRunning(queriedNode)) { // something is wrong, retry FaultAnalysisServiceUtility.ThrowEngineRetryableException(string.Format(CultureInfo.InvariantCulture, "{0} - start node validation - node is not running yet. Status={1}", state.OperationId, queriedNode.NodeStatus)); } }