public static async Task <SelectedPartition> GetPartitionAsync(this IFabricTestabilityClient client, PartitionSelector partitionSelector)
{
var systemFabricCient = GetSystemFabricClient(client);
GetSelectedPartitionStateAction partitionAction = new GetSelectedPartitionStateAction(partitionSelector);
await systemFabricCient.FabricClient.TestManager.TestContext.ActionExecutor.RunAsync(partitionAction);
return(partitionAction.Result);
}
/// <summary>
/// This API supports the Service Fabric platform and is not meant to be called from your code
/// </summary>
/// <param name="token">This API supports the Service Fabric platform and is not meant to be called from your code</param>
/// <returns></returns>
protected override async Task OnExecuteAsync(CancellationToken token)
{
this.serviceDescription = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.ServiceManager.GetServiceDescriptionAsync(
this.failoverTestScenarioParameters.PartitionSelector.ServiceName,
this.failoverTestScenarioParameters.RequestTimeout,
token),
this.failoverTestScenarioParameters.OperationTimeout,
token).ConfigureAwait(false);
bool hasPersistedState = false;
if (this.serviceDescription.IsStateful())
{
StatefulServiceDescription statefulDescription = this.serviceDescription as StatefulServiceDescription;
ReleaseAssert.AssertIf(statefulDescription == null, "Stateful service description is not WinFabricStatefulServiceDescription");
hasPersistedState = statefulDescription.HasPersistedState;
}
Log.WriteInfo(TraceType, "Validating Service health and availability");
await this.FabricClient.TestManager.ValidateServiceAsync(
this.failoverTestScenarioParameters.PartitionSelector.ServiceName,
this.failoverTestScenarioParameters.MaxServiceStabilizationTimeout,
token);
Log.WriteInfo(TraceType, "Getting Selected Partition");
var getPartitionStateAction = new GetSelectedPartitionStateAction(this.failoverTestScenarioParameters.PartitionSelector)
{
RequestTimeout = this.failoverTestScenarioParameters.RequestTimeout,
ActionTimeout = this.failoverTestScenarioParameters.OperationTimeout
};
await this.TestContext.ActionExecutor.RunAsync(getPartitionStateAction, token);
Guid selectedPartitionId = getPartitionStateAction.Result.PartitionId;
Log.WriteInfo(TraceType, "Running test for partition {0}", selectedPartitionId);
this.ReportProgress("Selected partition {0} for testing failover", selectedPartitionId);
PartitionSelector selectedPartition = PartitionSelector.PartitionIdOf(this.failoverTestScenarioParameters.PartitionSelector.ServiceName, selectedPartitionId);
while (this.failoverTestScenarioParameters.TimeToRun - this.GetElapsedTime() > TimeSpan.Zero && !token.IsCancellationRequested)
{
if (this.serviceDescription.IsStateful())
{
ReplicaSelector primaryReplicaSelector = ReplicaSelector.PrimaryOf(selectedPartition);
ReplicaSelector secondaryReplicaSelector = ReplicaSelector.RandomSecondaryOf(selectedPartition);
// Make Primary go through RemoveReplica, RestartReplica and RestartCodePackage
await this.TestReplicaFaultsAsync(primaryReplicaSelector, "Primary", hasPersistedState, token);
// Make Secondary go through RemoveReplica, RestartReplica and RestartCodePackage
await this.TestReplicaFaultsAsync(secondaryReplicaSelector, "Secondary", hasPersistedState, token);
}
else
{
ReplicaSelector randomInstanceSelector = ReplicaSelector.RandomOf(selectedPartition);
// Make Stateless Instance go through RemoveReplica, RestartReplica and RestartCodePackage
await this.TestReplicaFaultsAsync(randomInstanceSelector, "Stateless Instance", hasPersistedState, token);
}
if (this.serviceDescription.IsStateful())
{
// Restart all secondary replicas and make sure the replica set recovers
await this.InvokeAndValidateFaultAsync(
"Restarting all the secondary replicas",
() =>
{
#pragma warning disable 618
return(this.FabricClient.TestManager.RestartPartitionAsync(
selectedPartition,
RestartPartitionMode.OnlyActiveSecondaries,
this.failoverTestScenarioParameters.OperationTimeout,
token));
#pragma warning restore 618
}, token);
// Restart all replicas if service is persisted
if (hasPersistedState)
{
await this.InvokeAndValidateFaultAsync(
"Restarting all replicas including Primary",
() =>
{
#pragma warning disable 618
return(this.FabricClient.TestManager.RestartPartitionAsync(
selectedPartition,
RestartPartitionMode.AllReplicasOrInstances,
this.failoverTestScenarioParameters.OperationTimeout,
token));
#pragma warning restore 618
}, token);
}
// Induce move and swap primary a few times
await this.InvokeAndValidateFaultAsync(
"Move Primary to a different node",
() =>
{
return(this.FabricClient.FaultManager.MovePrimaryAsync(
string.Empty,
selectedPartition,
true,
this.failoverTestScenarioParameters.OperationTimeout,
token));
}, token);
// Induce move secondary a few times
await this.InvokeAndValidateFaultAsync(
"Move Secondary to a different node",
() =>
{
return(this.FabricClient.FaultManager.MoveSecondaryAsync(
string.Empty,
string.Empty,
selectedPartition,
true,
this.failoverTestScenarioParameters.OperationTimeout,
token));
}, token);
}
else
{
// Restart all stateless instances
await this.InvokeAndValidateFaultAsync(
"Restarting all stateless instances for partition",
() =>
{
#pragma warning disable 618
return(this.FabricClient.TestManager.RestartPartitionAsync(
selectedPartition,
RestartPartitionMode.AllReplicasOrInstances,
this.failoverTestScenarioParameters.OperationTimeout,
token));
#pragma warning restore 618
}, token);
}
}
}
