protected override async Task ExecuteActionAsync(FabricTestContext testContext, GetSelectedReplicaStateAction action, CancellationToken cancellationToken)
{
ThrowIf.Null(action.ReplicaSelector, "ReplicaSelector");
TimeoutHelper helper = new TimeoutHelper(action.ActionTimeout);
var getPartitionStateAction = new GetSelectedPartitionStateAction(action.ReplicaSelector.PartitionSelector)
{
RequestTimeout = action.RequestTimeout,
ActionTimeout = helper.GetRemainingTime()
};
await testContext.ActionExecutor.RunAsync(getPartitionStateAction, cancellationToken);
Guid partitionId = getPartitionStateAction.Result.PartitionId;
// TODO: make these actions which store state locally as well.
ServiceReplicaList replicasResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync <ServiceReplicaList>(
() => testContext.FabricClient.QueryManager.GetReplicaListAsync(
partitionId,
0,
action.RequestTimeout,
cancellationToken),
helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
Replica replicaResult = action.ReplicaSelector.GetSelectedReplica(replicasResult.ToArray(), testContext.Random, true /*skip invalid replicas*/);
var replicaSelectorResult = new SelectedReplica(replicaResult.Id, getPartitionStateAction.Result);
action.Result = new Tuple <SelectedReplica, Replica>(
replicaSelectorResult,
replicaResult);
ResultTraceString = StringHelper.Format("ReplicaSelector Selected Replica {0}", replicaResult.Id);
}
internal RestartNodeResult(SelectedReplica selectedReplica, NodeResult nodeResult)
{
ReleaseAssert.AssertIfNull(nodeResult, "Node result cannot be null");
ReleaseAssert.AssertIfNull(selectedReplica, "Selected replica cannot be null");
this.NodeResult = nodeResult;
this.SelectedReplica = selectedReplica;
}
public static async Task <Tuple <SelectedReplica, Replica> > GetSelectedReplicaAsync(
FabricClient fabricClient,
ReplicaSelector replicaSelector,
TimeSpan requestTimeout,
TimeSpan operationTimeout,
CancellationToken cancellationToken)
{
ThrowIf.Null(replicaSelector, "ReplicaSelector");
SelectedPartition selectedPartition = await FaultAnalysisServiceUtility.GetSelectedPartitionStateAsync(
fabricClient,
replicaSelector.PartitionSelector,
requestTimeout,
operationTimeout,
cancellationToken).ConfigureAwait(false);
Guid partitionId = selectedPartition.PartitionId;
ServiceReplicaList replicasResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => fabricClient.QueryManager.GetReplicaListAsync(
partitionId,
0,
requestTimeout,
cancellationToken),
operationTimeout,
cancellationToken).ConfigureAwait(false);
Replica replicaResult = replicaSelector.GetSelectedReplica(replicasResult.ToArray(), new Random(), true /*skip invalid replicas*/);
var replicaSelectorResult = new SelectedReplica(replicaResult.Id, selectedPartition);
return(new Tuple <SelectedReplica, Replica>(replicaSelectorResult, replicaResult));
}
public static async Task <RestartReplicaResult> RestartReplicaAsync(
FabricClient fabricClient,
ReplicaSelector replicaSelector,
CompletionMode completionMode,
TimeSpan requestTimeout,
TimeSpan operationTimeout,
CancellationToken cancellationToken)
{
System.Fabric.Common.TimeoutHelper helper = new System.Fabric.Common.TimeoutHelper(operationTimeout);
string nodeName = null;
Guid partitionId = Guid.Empty;
long replicaId = 0;
SelectedReplica replicaSelectorResult = SelectedReplica.None;
System.Fabric.Common.ThrowIf.Null(replicaSelector, "ReplicaSelector");
Tuple <SelectedReplica, Replica> replicaStateActionResult = await FaultAnalysisServiceUtility.GetSelectedReplicaAsync(
fabricClient,
replicaSelector,
requestTimeout,
operationTimeout,
cancellationToken).ConfigureAwait(false);
replicaSelectorResult = replicaStateActionResult.Item1;
if (replicaSelectorResult == null)
{
throw new InvalidOperationException("replicaStateActionResult cannot be null");
}
partitionId = replicaStateActionResult.Item1.SelectedPartition.PartitionId;
Replica replicaStateResult = replicaStateActionResult.Item2;
if (replicaStateResult == null)
{
throw new InvalidOperationException("replicaStateResult cannot be null");
}
nodeName = replicaStateResult.NodeName;
replicaId = replicaStateResult.Id;
ThrowIf.IsTrue(partitionId == Guid.Empty, "PartitionID");
ThrowIf.IsTrue(replicaId == 0, "ReplicaID");
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => fabricClient.ServiceManager.RestartReplicaAsync(
nodeName,
partitionId,
replicaId,
requestTimeout,
cancellationToken),
FabricClientRetryErrors.RestartReplicaErrors.Value,
operationTimeout,
cancellationToken).ConfigureAwait(false);
return(new RestartReplicaResult(replicaSelectorResult));
}
internal static RestartNodeResult CreateFromNative(NativeClient.IFabricRestartNodeResult nativeResult)
{
if (nativeResult == null)
{
return(null);
}
var selectedReplica = new SelectedReplica(0, SelectedPartition.None);
var nodeResult = NodeResult.CreateFromNative(nativeResult.get_Result());
var restartNodeResult = new RestartNodeResult(selectedReplica, nodeResult);
GC.KeepAlive(nativeResult);
return(restartNodeResult);
}
/// <summary>
/// Restart deployed code package result constructor.
/// </summary>
/// <param name="nodeName">node name</param>
/// <param name="applicationName">application name</param>
/// <param name="serviceManifestName">service manifest name</param>
/// <param name="servicePackageActivationId">service package ActivationId</param>
/// <param name="codePackageName">code package name</param>
/// <param name="codePackageInstanceId">code package instance id</param>
/// <param name="selectedReplica">selected replica</param>
internal RestartDeployedCodePackageResult(
string nodeName,
Uri applicationName,
string serviceManifestName,
string servicePackageActivationId,
string codePackageName,
long codePackageInstanceId,
SelectedReplica selectedReplica)
{
ReleaseAssert.AssertIfNull(applicationName, "Application name cannot be null");
this.ApplicationName = applicationName;
ReleaseAssert.AssertIf(string.IsNullOrEmpty(nodeName), "Node name cannot be null or empty");
this.NodeName = nodeName;
ReleaseAssert.AssertIf(string.IsNullOrEmpty(serviceManifestName), "Service manifestName cannot be null or empty");
this.ServiceManifestName = serviceManifestName;
ReleaseAssert.AssertIf(string.IsNullOrEmpty(codePackageName), "Code package name cannot be null or empty");
this.CodePackageName = codePackageName;
ReleaseAssert.AssertIf(codePackageInstanceId == 0, "Code package instance id cannot be zero");
this.CodePackageInstanceId = codePackageInstanceId;
this.SelectedReplica = selectedReplica;
ReleaseAssert.AssertIfNull(servicePackageActivationId, "ServicePackageActivationId cannot be null");
this.ServicePackageActivationId = servicePackageActivationId;
}
protected override async Task ExecuteActionAsync(FabricTestContext testContext, RestartReplicaAction action, CancellationToken cancellationToken)
{
TimeoutHelper helper = new TimeoutHelper(action.ActionTimeout);
string nodeName = action.NodeName;
Guid? partitionId = action.PartitionId;
long? replicaId = action.ReplicaId;
SelectedReplica replicaSelectorResult = SelectedReplica.None;
if (string.IsNullOrEmpty(nodeName) ||
!partitionId.HasValue ||
!replicaId.HasValue)
{
ThrowIf.Null(action.ReplicaSelector, "ReplicaSelector");
var getReplicaStateAction = new GetSelectedReplicaStateAction(action.ReplicaSelector)
{
RequestTimeout = action.RequestTimeout,
ActionTimeout = helper.GetRemainingTime()
};
await testContext.ActionExecutor.RunAsync(getReplicaStateAction, cancellationToken).ConfigureAwait(false);
var replicaStateActionResult = getReplicaStateAction.Result;
replicaSelectorResult = replicaStateActionResult.Item1;
ReleaseAssert.AssertIf(replicaSelectorResult == null, "replicaSelectorResult cannot be null");
partitionId = replicaStateActionResult.Item1.SelectedPartition.PartitionId;
Replica replicaStateResult = replicaStateActionResult.Item2;
ReleaseAssert.AssertIf(replicaStateResult == null, "replicaStateResult cannot be null");
nodeName = replicaStateResult.NodeName;
replicaId = replicaStateResult.Id;
}
ThrowIf.IsFalse(partitionId.HasValue, "PartitionID");
ThrowIf.IsFalse(replicaId.HasValue, "ReplicaID");
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.ServiceManager.RestartReplicaAsync(
nodeName,
partitionId.Value,
replicaId.Value,
action.RequestTimeout,
cancellationToken),
FabricClientRetryErrors.RestartReplicaErrors.Value,
helper.GetRemainingTime(),
cancellationToken);
if (action.CompletionMode == CompletionMode.Verify)
{
// TODO: Check with failover team to see how to confirm that the replica actually restarted. We do not expose instance id for persisted replicas
}
action.Result = new RestartReplicaResult(replicaSelectorResult);
this.ResultTraceString = StringHelper.Format(
"RestartReplicaOrInstance succeeded by restarting replica {0}:{1} node {2} with CompletionMode {3}",
partitionId.Value,
replicaId.Value,
nodeName,
action.CompletionMode);
}
/// <summary>
/// Remove replica result constructor.
/// </summary>
/// <param name="selectedReplica">Selected replica</param>
internal RemoveReplicaResult(SelectedReplica selectedReplica) : base(selectedReplica)
{
}
/// <summary>
/// Restart replica result constructor.
/// </summary>
/// <param name="selectedReplica">Selected replica</param>
internal RestartReplicaResult(SelectedReplica selectedReplica) : base(selectedReplica)
{
}
protected override async Task ExecuteActionAsync(FabricTestContext testContext, RemoveReplicaAction action, CancellationToken cancellationToken)
{
TimeoutHelper helper = new TimeoutHelper(action.ActionTimeout);
string nodeName = action.NodeName;
Guid? partitionId = action.PartitionId;
long? replicaId = action.ReplicaId;
SelectedReplica replicaSelectorResult = SelectedReplica.None;
if (string.IsNullOrEmpty(nodeName) ||
!partitionId.HasValue ||
!replicaId.HasValue)
{
ThrowIf.Null(action.ReplicaSelector, "ReplicaSelector");
var getReplicaStateAction = new GetSelectedReplicaStateAction(action.ReplicaSelector)
{
RequestTimeout = action.RequestTimeout,
ActionTimeout = helper.GetRemainingTime()
};
await testContext.ActionExecutor.RunAsync(getReplicaStateAction, cancellationToken).ConfigureAwait(false);
var replicaStateActionResult = getReplicaStateAction.Result;
ReleaseAssert.AssertIf(replicaStateActionResult == null, "replicaStateActionResult cannot be null");
replicaSelectorResult = replicaStateActionResult.Item1;
partitionId = replicaStateActionResult.Item1.SelectedPartition.PartitionId;
Replica replicaStateResult = replicaStateActionResult.Item2;
ReleaseAssert.AssertIf(replicaStateResult == null, "replicaStateResult cannot be null");
nodeName = replicaStateResult.NodeName;
replicaId = replicaStateResult.Id;
}
ThrowIf.IsFalse(partitionId.HasValue, "PartitionID");
ThrowIf.IsFalse(replicaId.HasValue, "ReplicaID");
bool forceRemove = action.ForceRemove;
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.ServiceManager.RemoveReplicaAsync(
nodeName,
partitionId.Value,
replicaId.Value,
forceRemove,
action.RequestTimeout,
cancellationToken),
FabricClientRetryErrors.RemoveReplicaErrors.Value,
helper.GetRemainingTime(),
cancellationToken);
if (action.CompletionMode == CompletionMode.Verify)
{
// Check that replica on selected node has been removed i.e. the replica id does not exist anymore.
bool success = false;
while (helper.GetRemainingTime() > TimeSpan.Zero)
{
var replicasResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.QueryManager.GetReplicaListAsync(
partitionId.Value,
replicaId.Value,
action.RequestTimeout,
cancellationToken),
helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
bool dropped = replicasResult.Count == 0;
if (!dropped)
{
// Since we added a replica filter the result should contain the replica or none
ReleaseAssert.AssertIf(replicasResult.Count > 1, "More than 1 replica returned with replica filter {0}:{1}", partitionId.Value, replicaId.Value);
ReleaseAssert.AssertIf(replicasResult[0].Id != replicaId, "Incorrect replica Id {0} returned by query instead of {1}", replicasResult[0].Id, replicaId);
dropped = replicasResult[0].ReplicaStatus == ServiceReplicaStatus.Dropped;
}
if (dropped)
{
success = true;
break;
}
ActionTraceSource.WriteInfo(TraceSource, "Replica = {0}:{1} not yet completely removed. Retrying...", partitionId.Value, replicaId.Value);
await AsyncWaiter.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken);
}
if (!success)
{
throw new TimeoutException(StringHelper.Format(StringResources.Error_TestabilityActionTimeout,
"RemoveReplica",
StringHelper.Format("{0}:{1}", partitionId.Value, replicaId.Value)));
}
}
action.Result = new RemoveReplicaResult(replicaSelectorResult);
ResultTraceString = StringHelper.Format(
"RemoveReplicaOrInstance succeeded by removing replica {0}:{1} on node {2} with CompletionMode {3}",
partitionId.Value,
replicaId.Value,
nodeName,
action.CompletionMode);
}
protected override async Task ExecuteActionAsync(FabricTestContext testContext, RestartDeployedCodePackageAction action, CancellationToken cancellationToken)
{
this.helper = new TimeoutHelper(action.ActionTimeout);
string nodeName = action.NodeName;
Uri applicationName = action.ApplicationName;
string serviceManifestName = action.ServiceManifestName;
string servicePackageActivationId = action.ServicePackageActivationId;
string codePackageName = action.CodePackageName;
SelectedReplica replicaSelectorResult = SelectedReplica.None;
ThrowIf.Null(applicationName, "ApplicationName");
if (string.IsNullOrEmpty(nodeName) ||
string.IsNullOrEmpty(serviceManifestName) ||
string.IsNullOrEmpty(codePackageName))
{
ThrowIf.Null(action.ReplicaSelector, "ReplicaSelector");
var getReplicaStateAction = new GetSelectedReplicaStateAction(action.ReplicaSelector)
{
RequestTimeout = action.RequestTimeout,
ActionTimeout = this.helper.GetRemainingTime()
};
await testContext.ActionExecutor.RunAsync(getReplicaStateAction, cancellationToken).ConfigureAwait(false);
var replicaStateActionResult = getReplicaStateAction.Result;
ReleaseAssert.AssertIf(replicaStateActionResult == null, "replicaStateActionResult cannot be null");
replicaSelectorResult = replicaStateActionResult.Item1;
ReleaseAssert.AssertIf(replicaSelectorResult == null || replicaSelectorResult.SelectedPartition == null,
"replicaSelectorResult cannot be null or for a non-null replicaSelectorResult, the selected partition must be non-null");
Guid partitionId = replicaStateActionResult.Item1.SelectedPartition.PartitionId;
Replica replicaStateResult = replicaStateActionResult.Item2;
ReleaseAssert.AssertIf(replicaStateResult == null, "replicaStateResult cannot be null");
nodeName = replicaStateResult.NodeName;
var deployedReplicaListResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync <DeployedServiceReplicaList>(
() => testContext.FabricClient.QueryManager.GetDeployedReplicaListAsync(
nodeName,
applicationName,
null,
partitionId,
action.RequestTimeout,
cancellationToken),
this.helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
DeployedServiceReplica selectedReplica = deployedReplicaListResult.FirstOrDefault(r => r.Partitionid == partitionId);
if (selectedReplica == null)
{
throw new FabricException(
StringHelper.Format(StringResources.Error_DidNotFindDeployedReplicaOnNode, partitionId, nodeName),
FabricErrorCode.ReplicaDoesNotExist);
}
serviceManifestName = selectedReplica.ServiceManifestName;
servicePackageActivationId = selectedReplica.ServicePackageActivationId;
codePackageName = selectedReplica.CodePackageName;
}
ActionTraceSource.WriteInfo(TraceSource, "SelectedReplica: serviceManifestName: {0}, servicePackageActivationId: {1}, codePackageName: {2}", serviceManifestName, servicePackageActivationId, codePackageName);
DeployedCodePackage deployedCodePackageListResult = await this.GetCodePackageInfoAsync(testContext, nodeName, applicationName, serviceManifestName, servicePackageActivationId, codePackageName, action, cancellationToken).ConfigureAwait(false);
var codepackageEntrypointToRestart = GetCodepackageEntrypointToRestart(action, deployedCodePackageListResult);
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.FaultManager.RestartDeployedCodePackageUsingNodeNameAsync(
nodeName,
applicationName,
serviceManifestName,
servicePackageActivationId,
codePackageName,
codepackageEntrypointToRestart.EntryPoint.CodePackageInstanceId,
action.RequestTimeout,
cancellationToken),
this.helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
if (action.CompletionMode == CompletionMode.Verify)
{
bool success = false;
while (this.helper.GetRemainingTime() > TimeSpan.Zero)
{
var deployedCodePackageListResultAfterRestart = await this.GetCodePackageInfoAsync(testContext, nodeName, applicationName, serviceManifestName, servicePackageActivationId, codePackageName, action, cancellationToken).ConfigureAwait(false);
if (deployedCodePackageListResultAfterRestart != null)
{
var entryPointAfterRestart = codepackageEntrypointToRestart.EntryPointType == EntryPointType.Main ? deployedCodePackageListResultAfterRestart.EntryPoint : deployedCodePackageListResultAfterRestart.SetupEntryPoint;
if (entryPointAfterRestart != null && entryPointAfterRestart.CodePackageInstanceId > codepackageEntrypointToRestart.EntryPoint.CodePackageInstanceId && entryPointAfterRestart.EntryPointStatus == EntryPointStatus.Started)
{
success = true;
break;
}
}
ActionTraceSource.WriteInfo(TraceSource, "CodePackage = {0}:{1}:{2} not yet restarted. Retrying...", nodeName, applicationName, codePackageName);
await AsyncWaiter.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken).ConfigureAwait(false);
}
if (!success)
{
throw new TimeoutException(StringHelper.Format(StringResources.Error_TestabilityActionTimeout,
"RestartDeployedCodePackage",
applicationName));
}
}
action.Result = new RestartDeployedCodePackageResult(
nodeName,
applicationName,
serviceManifestName,
servicePackageActivationId,
codePackageName,
codepackageEntrypointToRestart.EntryPoint.CodePackageInstanceId,
replicaSelectorResult);
ResultTraceString = StringHelper.Format("RestartCodePackageAction succeeded for {0}:{1}:{2} with CompletionMode = {3}", nodeName, applicationName, codePackageName, action.CompletionMode);
}
protected override async Task ExecuteActionAsync(FabricTestContext testContext, RestartNodeAction action, CancellationToken cancellationToken)
{
ActionTraceSource.WriteInfo(TraceSource, "Enter RestartNodeAction/ExecuteActionAsync: operationTimeout='{0}', requestTimeout='{1}'", action.ActionTimeout, action.RequestTimeout);
this.helper = new TimeoutHelper(action.ActionTimeout);
SelectedReplica selectedReplica = SelectedReplica.None;
string nodeName = action.NodeName;
BigInteger nodeInstance = action.NodeInstance;
bool createFabricDump = action.CreateFabricDump;
if (string.IsNullOrEmpty(nodeName))
{
ThrowIf.Null(action.ReplicaSelector, "ReplicaSelector");
var getReplicaStateAction = new GetSelectedReplicaStateAction(action.ReplicaSelector)
{
RequestTimeout = action.RequestTimeout,
ActionTimeout = helper.GetRemainingTime()
};
await testContext.ActionExecutor.RunAsync(getReplicaStateAction, cancellationToken).ConfigureAwait(false);
var replicaStateActionResult = getReplicaStateAction.Result;
ReleaseAssert.AssertIf(replicaStateActionResult == null, "replicaStateActionResult cannot be null");
selectedReplica = replicaStateActionResult.Item1;
Replica replicaStateResult = replicaStateActionResult.Item2;
ReleaseAssert.AssertIf(replicaStateResult == null, "replicaStateResult cannot be null");
nodeName = replicaStateResult.NodeName;
nodeInstance = BigInteger.MinusOne;
}
if (nodeInstance == BigInteger.MinusOne)
{
var nodeInfo = await this.GetCurrentNodeInfoAsync(testContext, nodeName, action, cancellationToken).ConfigureAwait(false);
nodeInstance = nodeInfo.NodeInstanceId;
}
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.FaultManager.RestartNodeUsingNodeNameAsync(
nodeName,
nodeInstance,
createFabricDump,
action.RequestTimeout,
cancellationToken),
this.helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
if (action.CompletionMode == CompletionMode.Verify)
{
bool success = false;
while (this.helper.GetRemainingTime() > TimeSpan.Zero)
{
var nodeInfo = await this.GetCurrentNodeInfoAsync(testContext, nodeName, action, cancellationToken).ConfigureAwait(false);
if (nodeInfo.NodeInstanceId > nodeInstance && nodeInfo.IsNodeUp)
{
success = true;
break;
}
ActionTraceSource.WriteInfo(TraceSource, "NodeName = {0} not yet restarted. '{1}' seconds remain. Retrying...", nodeName, this.helper.GetRemainingTime().TotalSeconds);
await AsyncWaiter.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken);
}
if (!success)
{
throw new TimeoutException(StringHelper.Format(StringResources.Error_TestabilityActionTimeout,
"RestartNode",
nodeName));
}
}
// create result
action.Result = new RestartNodeResult(selectedReplica, new NodeResult(nodeName, nodeInstance));
ResultTraceString = StringHelper.Format("RestartNodeAction succeeded for {0}:{1} with CompletionMode = {2}", nodeName, nodeInstance, action.CompletionMode);
}
/// <summary>
/// Replica result constructor.
/// </summary>
/// <param name="selectedReplica">Selected replica</param>
internal ReplicaResult(SelectedReplica selectedReplica)
{
ReleaseAssert.AssertIfNull(selectedReplica, "Selected replica cannot be null");
this.SelectedReplica = selectedReplica;
}
