public static async Task <RestartReplicaResult> RestartReplicaAsync(
            FabricClient fabricClient,
            ReplicaSelector replicaSelector,
            CompletionMode completionMode,
            TimeSpan requestTimeout,
            TimeSpan operationTimeout,
            CancellationToken cancellationToken)
        {
            System.Fabric.Common.TimeoutHelper helper = new System.Fabric.Common.TimeoutHelper(operationTimeout);

            string          nodeName              = null;
            Guid            partitionId           = Guid.Empty;
            long            replicaId             = 0;
            SelectedReplica replicaSelectorResult = SelectedReplica.None;

            System.Fabric.Common.ThrowIf.Null(replicaSelector, "ReplicaSelector");

            Tuple <SelectedReplica, Replica> replicaStateActionResult = await FaultAnalysisServiceUtility.GetSelectedReplicaAsync(
                fabricClient,
                replicaSelector,
                requestTimeout,
                operationTimeout,
                cancellationToken).ConfigureAwait(false);

            replicaSelectorResult = replicaStateActionResult.Item1;
            if (replicaSelectorResult == null)
            {
                throw new InvalidOperationException("replicaStateActionResult cannot be null");
            }

            partitionId = replicaStateActionResult.Item1.SelectedPartition.PartitionId;

            Replica replicaStateResult = replicaStateActionResult.Item2;

            if (replicaStateResult == null)
            {
                throw new InvalidOperationException("replicaStateResult cannot be null");
            }

            nodeName  = replicaStateResult.NodeName;
            replicaId = replicaStateResult.Id;

            ThrowIf.IsTrue(partitionId == Guid.Empty, "PartitionID");
            ThrowIf.IsTrue(replicaId == 0, "ReplicaID");

            await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
                () => fabricClient.ServiceManager.RestartReplicaAsync(
                    nodeName,
                    partitionId,
                    replicaId,
                    requestTimeout,
                    cancellationToken),
                FabricClientRetryErrors.RestartReplicaErrors.Value,
                operationTimeout,
                cancellationToken).ConfigureAwait(false);

            return(new RestartReplicaResult(replicaSelectorResult));
        }
        public static async Task <TResult> RetryOnTimeout <TResult>(Guid operationId, Func <Task <TResult> > action, TimeSpan timeout, CancellationToken cancellationToken)
        {
            System.Fabric.Common.TimeoutHelper timeoutHelper = new System.Fabric.Common.TimeoutHelper(timeout);

            TResult result        = default(TResult);
            bool    wasSuccessful = false;

            do
            {
                cancellationToken.ThrowIfCancellationRequested();

                try
                {
                    result = await action().ConfigureAwait(false);

                    wasSuccessful = true;
                }
                catch (TimeoutException)
                {
                    TestabilityTrace.TraceSource.WriteWarning(
                        TraceType,
                        "{0} - ProcessGetProgressAsync - RD access timed out, retrying.  Total remaining time {1}",
                        operationId,
                        timeoutHelper.GetRemainingTime());
                }

                if (!wasSuccessful)
                {
                    await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken).ConfigureAwait(false);
                }
            }while (!wasSuccessful && timeoutHelper.GetRemainingTime() > TimeSpan.Zero);

            if (!wasSuccessful)
            {
                throw new FabricException(FabricErrorCode.OperationTimedOut);
            }

            return(result);
        }