protected override async Task ExecuteActionAsync(FabricTestContext testContext, GetSelectedReplicaStateAction action, CancellationToken cancellationToken)
{
ThrowIf.Null(action.ReplicaSelector, "ReplicaSelector");
TimeoutHelper helper = new TimeoutHelper(action.ActionTimeout);
var getPartitionStateAction = new GetSelectedPartitionStateAction(action.ReplicaSelector.PartitionSelector)
{
RequestTimeout = action.RequestTimeout,
ActionTimeout = helper.GetRemainingTime()
};
await testContext.ActionExecutor.RunAsync(getPartitionStateAction, cancellationToken);
Guid partitionId = getPartitionStateAction.Result.PartitionId;
// TODO: make these actions which store state locally as well.
ServiceReplicaList replicasResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync <ServiceReplicaList>(
() => testContext.FabricClient.QueryManager.GetReplicaListAsync(
partitionId,
0,
action.RequestTimeout,
cancellationToken),
helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
Replica replicaResult = action.ReplicaSelector.GetSelectedReplica(replicasResult.ToArray(), testContext.Random, true /*skip invalid replicas*/);
var replicaSelectorResult = new SelectedReplica(replicaResult.Id, getPartitionStateAction.Result);
action.Result = new Tuple <SelectedReplica, Replica>(
replicaSelectorResult,
replicaResult);
ResultTraceString = StringHelper.Format("ReplicaSelector Selected Replica {0}", replicaResult.Id);
}
protected override async Task ExecuteActionAsync(FabricTestContext testContext, InvokeQuorumLossAction action, CancellationToken cancellationToken)
{
ThrowIf.Null(action.PartitionSelector, "PartitionSelector");
var helper = new TimeoutHelper(action.ActionTimeout);
// get info about the service so we can check type and trss
ServiceDescription result = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.ServiceManager.GetServiceDescriptionAsync(
action.PartitionSelector.ServiceName,
action.RequestTimeout,
cancellationToken),
helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
if (result.Kind != ServiceDescriptionKind.Stateful)
{
throw new InvalidOperationException(StringHelper.Format(StringResources.Error_InvalidServiceTypeTestability, "QuorumLoss", "Stateful", action.PartitionSelector.ServiceName, "Stateless"));
}
StatefulServiceDescription statefulServiceDescription = result as StatefulServiceDescription;
ReleaseAssert.AssertIf(statefulServiceDescription == null, "Service is not a stateful service");
if (!statefulServiceDescription.HasPersistedState)
{
throw new InvalidOperationException(StringHelper.Format(StringResources.Error_InvalidServiceTypeTestability, "QuorumLoss", "Stateful Persistent", action.PartitionSelector.ServiceName, "Stateful In-Memory Only"));
}
// figure out /which/ partition to select
var getPartitionStateAction = new GetSelectedPartitionStateAction(action.PartitionSelector)
{
RequestTimeout = action.RequestTimeout,
ActionTimeout = helper.GetRemainingTime()
};
await testContext.ActionExecutor.RunAsync(getPartitionStateAction, cancellationToken);
Guid partitionId = getPartitionStateAction.Result.PartitionId;
// get data about replicas in that partition
ServiceReplicaList replicasResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.QueryManager.GetReplicaListAsync(
partitionId,
0,
action.RequestTimeout,
cancellationToken),
helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
var removeUTRequestList = new List <Tuple <string, string> >();
Dictionary <Tuple <string, string>, Task> removeUTTaskDictionary = new Dictionary <Tuple <string, string>, Task>();
try
{
var stableReplicas = replicasResult.Where(r => r.ReplicaStatus == ServiceReplicaStatus.Ready).ToArray();
var stableReplicasToRemove = new List <StatefulServiceReplica>();
long replicasToRestartWithoutPrimary =
action.QuorumLossMode == QuorumLossMode.AllReplicas
? stableReplicas.Length - 1
: FabricCluster.GetWriteQuorumSize(replicasResult.Count);
foreach (var replica in stableReplicas)
{
StatefulServiceReplica statefulReplica = replica as StatefulServiceReplica;
ReleaseAssert.AssertIf(statefulReplica == null, "Service Replica is not of stateful type even though service is stateful");
if (statefulReplica.ReplicaRole != ReplicaRole.Primary)
{
replicasToRestartWithoutPrimary--;
}
if (replicasToRestartWithoutPrimary >= 0 || statefulReplica.ReplicaRole == ReplicaRole.Primary)
{
stableReplicasToRemove.Add(statefulReplica);
}
}
// for selected replicas, block reopen so that when we restart the replica (NOT remove the replica) it doesn't come up
var utTaskList = new List <Task>();
foreach (var statefulReplica in stableReplicasToRemove)
{
string nodeName = statefulReplica.NodeName;
UnreliableTransportBehavior behavior = new UnreliableTransportBehavior("*", "StatefulServiceReopen");
behavior.AddFilterForPartitionId(partitionId);
string behaviorName = "BlockStatefulServiceReopen_" + nodeName;
removeUTRequestList.Add(new Tuple <string, string>(nodeName, behaviorName));
utTaskList.Add(
FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() =>
testContext.FabricClient.TestManager.AddUnreliableTransportBehaviorAsync(
nodeName,
behaviorName,
behavior,
action.RequestTimeout,
cancellationToken),
helper.GetRemainingTime(),
cancellationToken));
}
await Task.WhenAll(utTaskList).ConfigureAwait(false);
// TODO: Wait for some time so that the unreliable transport behavior can be read from the files.
// Bug#2271465 - Unreliable transport through API should return only once the behavior has been successfully applied
await Task.Delay(TimeSpan.FromSeconds(5.0), cancellationToken);
var restartReplicaTaskList = new List <Task>();
foreach (var statefulReplica in stableReplicasToRemove)
{
ReplicaSelector replicaSelector = ReplicaSelector.ReplicaIdOf(PartitionSelector.PartitionIdOf(action.PartitionSelector.ServiceName, partitionId), statefulReplica.Id);
var restartReplicaAction = new RestartReplicaAction(replicaSelector)
{
CompletionMode = CompletionMode.DoNotVerify,
RequestTimeout = action.RequestTimeout,
ActionTimeout = helper.GetRemainingTime()
};
restartReplicaTaskList.Add(testContext.ActionExecutor.RunAsync(restartReplicaAction, cancellationToken));
}
await Task.WhenAll(restartReplicaTaskList).ConfigureAwait(false);
await AsyncWaiter.WaitAsync(action.QuorumLossDuration, cancellationToken).ConfigureAwait(false);
// validate
ServicePartitionList partitionsResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.QueryManager.GetPartitionListAsync(
action.PartitionSelector.ServiceName,
null,
action.RequestTimeout,
cancellationToken),
FabricClientRetryErrors.GetPartitionListFabricErrors.Value,
helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
foreach (StatefulServicePartition partition in partitionsResult)
{
if (partition.PartitionInformation.Id == partitionId)
{
ReleaseAssert.AssertIf(partition.PartitionStatus != ServicePartitionStatus.InQuorumLoss, "Partition failed to be in Quorum Loss.");
break;
}
}
foreach (var removeUTParams in removeUTRequestList)
{
var currentParams = removeUTParams;
Task task = FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.TestManager.RemoveUnreliableTransportBehaviorAsync(
currentParams.Item1, /*nodeName*/
currentParams.Item2, /*behaviorName*/
action.RequestTimeout,
cancellationToken),
FabricClientRetryErrors.RemoveUnreliableTransportBehaviorErrors.Value,
helper.GetRemainingTime(),
cancellationToken);
removeUTTaskDictionary[currentParams] = task;
}
await Task.WhenAll(removeUTTaskDictionary.Values).ConfigureAwait(false);
// TODO: Wait for some time so that the removal of this unreliable transport behavior can be read from the files.
// Bug#2271465 - Unreliable transport through API should return only once the behavior has been successully applied
await Task.Delay(TimeSpan.FromSeconds(5.0), cancellationToken);
}
finally
{
var removeUTTaskList = new List <Task>();
foreach (var removeUTRequest in removeUTTaskDictionary)
{
var currentRemoveUTRequest = removeUTRequest;
if (currentRemoveUTRequest.Value == null || currentRemoveUTRequest.Value.IsFaulted)
{
removeUTTaskList.Add(
FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.TestManager.RemoveUnreliableTransportBehaviorAsync(
currentRemoveUTRequest.Key.Item1, /*nodeName*/
currentRemoveUTRequest.Key.Item2, /*behaviorName*/
action.RequestTimeout,
cancellationToken),
FabricClientRetryErrors.RemoveUnreliableTransportBehaviorErrors.Value,
helper.GetRemainingTime(),
cancellationToken));
}
}
Task.WhenAll(removeUTTaskList).Wait(cancellationToken);
// TODO: Wait for some time so that the removal of this unreliable transport behavior can be read from the files.
// Bug#2271465 - Unreliable transport through API should return only once the behavior has been successully applied
Task.Delay(TimeSpan.FromSeconds(5.0), cancellationToken).GetAwaiter().GetResult();
}
action.Result = new InvokeQuorumLossResult(getPartitionStateAction.Result);
this.ResultTraceString = StringHelper.Format("InvokeQuorumLossAction succeeded for {0} with QuorumLossMode = {1}", partitionId, action.QuorumLossMode);
}
protected override async Task ExecuteActionAsync(FabricTestContext testContext, RestartPartitionAction action, CancellationToken cancellationToken)
{
ThrowIf.Null(action.PartitionSelector, "partitionSelector");
this.helper = new TimeoutHelper(action.ActionTimeout);
// get service info so we can validate if the operation is valid
ServiceDescription result = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.ServiceManager.GetServiceDescriptionAsync(
action.PartitionSelector.ServiceName,
action.RequestTimeout,
cancellationToken),
this.helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
if (result.Kind != ServiceDescriptionKind.Stateful && action.RestartPartitionMode == RestartPartitionMode.OnlyActiveSecondaries)
{
throw new InvalidOperationException(StringHelper.Format(StringResources.Error_InvalidServiceTypeTestability, "RestartPartitionMode.OnlyActiveSecondaries", "Stateful", action.PartitionSelector.ServiceName, "Stateless"));
}
bool hasPersistedState = false;
if (result.Kind == ServiceDescriptionKind.Stateful)
{
StatefulServiceDescription statefulDescription = result as StatefulServiceDescription;
ReleaseAssert.AssertIf(statefulDescription == null, "Stateful service description is not WinFabricStatefulServiceDescription");
hasPersistedState = statefulDescription.HasPersistedState;
}
// now actually select a partition
var getPartitionStateAction = new GetSelectedPartitionStateAction(action.PartitionSelector)
{
RequestTimeout = action.RequestTimeout,
ActionTimeout = helper.GetRemainingTime()
};
await testContext.ActionExecutor.RunAsync(getPartitionStateAction, cancellationToken);
Guid partitionId = getPartitionStateAction.Result.PartitionId;
// get replicas for target
ServiceReplicaList replicasResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.QueryManager.GetReplicaListAsync(
partitionId,
0,
action.RequestTimeout,
cancellationToken),
this.helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
// get replicas for fm in order to get the primary
ServiceReplicaList fmReplicasResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.QueryManager.GetReplicaListAsync(
Constants.FmPartitionId,
0,
action.RequestTimeout,
cancellationToken),
this.helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
string fmPrimaryNodeName = string.Empty;
var readyFMReplicas = fmReplicasResult.Where(r => r.ReplicaStatus == ServiceReplicaStatus.Ready).ToArray();
foreach (var replica in readyFMReplicas)
{
StatefulServiceReplica statefulReplica = replica as StatefulServiceReplica;
ReleaseAssert.AssertIf(statefulReplica == null, "FM Replica is not a stateful replica");
if (statefulReplica.ReplicaRole == ReplicaRole.Primary)
{
fmPrimaryNodeName = replica.NodeName;
}
}
if (string.IsNullOrEmpty(fmPrimaryNodeName))
{
throw new FabricException(StringHelper.Format(StringResources.Error_PartitionPrimaryNotReady, "FailoverManager"), FabricErrorCode.NotReady);
}
////------------------------------------------------------
// target ut at the fm primary only
UnreliableTransportBehavior behavior = new UnreliableTransportBehavior("*", "DoReconfiguration");
behavior.AddFilterForPartitionId(partitionId);
string behaviorName = "BlockDoReconfiguration";
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.TestManager.AddUnreliableTransportBehaviorAsync(
fmPrimaryNodeName,
behaviorName,
behavior,
action.RequestTimeout,
cancellationToken),
this.helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
// TODO: Wait for some time so that the unreliable transport behavior can be read from the files.
// Bug#2271465 - Unreliable transport through API should return only once the behavior has been successfully applied
await Task.Delay(TimeSpan.FromSeconds(5.0), cancellationToken).ConfigureAwait(false);
bool triedToRemovedBehavior = false;
// inspect the actual replicas to restart, only operate on stable ones
try
{
var stableReplicasToRestart = replicasResult.Where(r => r.ReplicaStatus == ServiceReplicaStatus.Ready).ToArray();
foreach (var replica in stableReplicasToRestart)
{
var currentReplica = replica;
if (action.RestartPartitionMode == RestartPartitionMode.OnlyActiveSecondaries)
{
StatefulServiceReplica statefulReplica = currentReplica as StatefulServiceReplica;
ReleaseAssert.AssertIf(statefulReplica == null, "Stateful service replica is not StatefulServiceReplica");
if (statefulReplica.ReplicaRole == ReplicaRole.Primary)
{
continue;
}
}
if (hasPersistedState)
{
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.FaultManager.RestartReplicaAsync(
currentReplica.NodeName,
partitionId,
currentReplica.Id,
CompletionMode.DoNotVerify,
action.RequestTimeout.TotalSeconds,
cancellationToken),
this.helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
}
else
{
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.FaultManager.RemoveReplicaAsync(
currentReplica.NodeName,
partitionId,
currentReplica.Id,
CompletionMode.DoNotVerify,
false, /*force remove*/
action.RequestTimeout.TotalSeconds,
cancellationToken),
this.helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
}
}
triedToRemovedBehavior = true;
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.TestManager.RemoveUnreliableTransportBehaviorAsync(
fmPrimaryNodeName,
behaviorName,
action.RequestTimeout,
cancellationToken),
FabricClientRetryErrors.RemoveUnreliableTransportBehaviorErrors.Value,
this.helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
// TODO: Wait for some time so that the unreliable transport behavior can be read from the files.
// Bug#2271465 - Unreliable transport through API should return only once the behavior has been successfully applied
await Task.Delay(TimeSpan.FromSeconds(5.0)).ConfigureAwait(false);
}
finally
{
// TODO: Provide a way to clear all behaviors just in case.
if (!triedToRemovedBehavior)
{
ActionTraceSource.WriteWarning(TraceType, "Exception after adding behavior to block messages. Removing behavior synchronously");
FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.TestManager.RemoveUnreliableTransportBehaviorAsync(
fmPrimaryNodeName,
behaviorName,
action.RequestTimeout,
cancellationToken),
FabricClientRetryErrors.RemoveUnreliableTransportBehaviorErrors.Value,
this.helper.GetRemainingTime(),
cancellationToken).GetAwaiter().GetResult();
// TODO: Wait for some time so that the unreliable transport behavior can be read from the files.
// Bug#2271465 - Unreliable transport through API should return only once the behavior has been successfully applied
Task.Delay(TimeSpan.FromSeconds(5.0)).GetAwaiter().GetResult();
}
}
// -- note there's no explict validation
// action result
action.Result = new RestartPartitionResult(getPartitionStateAction.Result);
ResultTraceString = StringHelper.Format("RestartPartitionAction succeeded for {0} with RestartPartitionMode = {1}", partitionId, action.RestartPartitionMode);
}
protected override async Task ExecuteActionAsync(FabricTestContext testContext, MoveSecondaryAction action, CancellationToken cancellationToken)
{
ThrowIf.Null(action.PartitionSelector, "PartitionSelector");
this.helper = new TimeoutHelper(action.ActionTimeout);
string newSecondaryNode = action.NewSecondaryNodeName;
string currentSecondaryNode = action.CurrentSecondaryNodeName;
var getPartitionStateAction = new GetSelectedPartitionStateAction(action.PartitionSelector)
{
RequestTimeout = action.RequestTimeout,
ActionTimeout = this.helper.GetRemainingTime()
};
await testContext.ActionExecutor.RunAsync(getPartitionStateAction, cancellationToken).ConfigureAwait(false);
Guid partitionId = getPartitionStateAction.Result.PartitionId;
if (!action.IgnoreConstraints)
{
// get current primary replica node name.
ServiceReplicaList replicasResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.QueryManager.GetReplicaListAsync(
partitionId,
0,
action.RequestTimeout,
cancellationToken),
this.helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
string currentPrimaryNodeInfo = string.Empty;
List <string> currentSecReplicaNodes = new List <string>();
foreach (var replica in replicasResult)
{
StatefulServiceReplica statefulReplica = replica as StatefulServiceReplica;
if (statefulReplica == null)
{
throw new InvalidOperationException(StringHelper.Format(StringResources.Error_InvalidServiceTypeTestability, "MoveSecondary", "Stateful", action.PartitionSelector.ServiceName, "Stateless"));
}
if (statefulReplica.ReplicaRole == ReplicaRole.Primary)
{
currentPrimaryNodeInfo = statefulReplica.NodeName;
if (!string.IsNullOrEmpty(newSecondaryNode) && newSecondaryNode == statefulReplica.NodeName)
{
throw new FabricException(
StringHelper.Format(StringResources.Error_InvalidNodeNameProvided, newSecondaryNode, "MoveSecondary", "Primary exists on node"),
FabricErrorCode.AlreadyPrimaryReplica);
}
}
else if (statefulReplica.ReplicaRole == ReplicaRole.ActiveSecondary)
{
currentSecReplicaNodes.Add(statefulReplica.NodeName);
if (!string.IsNullOrEmpty(newSecondaryNode) && newSecondaryNode == statefulReplica.NodeName)
{
throw new FabricException(
StringHelper.Format(StringResources.Error_InvalidNodeNameProvided, newSecondaryNode, "MoveSecondary", "Secondary exists on node"),
FabricErrorCode.AlreadySecondaryReplica);
}
}
}
if (currentSecReplicaNodes.Count == 0)
{
throw new InvalidOperationException(StringResources.Error_NoSecondariesInReplicaSet);
}
if (string.IsNullOrEmpty(currentSecondaryNode))
{
int num = testContext.Random.Next(currentSecReplicaNodes.Count);
currentSecondaryNode = currentSecReplicaNodes.ElementAt(num);
}
if (!currentSecReplicaNodes.Contains(currentSecondaryNode))
{
throw new FabricException(
StringHelper.Format(StringResources.Error_InvalidNodeNameProvided, newSecondaryNode, "MoveSecondary", "Current node does not have a secondary replica"),
FabricErrorCode.InvalidReplicaStateForReplicaOperation);
}
}
ReleaseAssert.AssertIf(string.IsNullOrEmpty(currentSecondaryNode), "Current node name cannot be null or empty.");
ReleaseAssert.AssertIf(newSecondaryNode == currentSecondaryNode, "Current and New node names are same.");
ActionTraceSource.WriteInfo(TraceSource, "Calling move secondary with current node {0}, new node {1}, partition {2}", currentSecondaryNode, string.IsNullOrEmpty(newSecondaryNode) ? "Random" : newSecondaryNode, partitionId);
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.FaultManager.MoveSecondaryUsingNodeNameAsync(
currentSecondaryNode,
newSecondaryNode,
getPartitionStateAction.Result.ServiceName,
partitionId,
action.IgnoreConstraints,
action.RequestTimeout,
cancellationToken),
FabricClientRetryErrors.MoveSecondaryFabricErrors.Value,
this.helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
action.Result = new MoveSecondaryResult(currentSecondaryNode, newSecondaryNode, getPartitionStateAction.Result);
this.ResultTraceString = StringHelper.Format(
"MoveSecondaryAction succeeded for moving Primary for {0} from {1} to {2}.",
partitionId,
currentSecondaryNode,
newSecondaryNode);
}
protected override async Task ExecuteActionAsync(FabricTestContext testContext, InvokeDataLossAction action, CancellationToken cancellationToken)
{
ThrowIf.Null(action.PartitionSelector, "PartitionSelector");
var helper = new TimeoutHelper(action.ActionTimeout);
ServiceDescription result = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.ServiceManager.GetServiceDescriptionAsync(
action.PartitionSelector.ServiceName,
action.RequestTimeout,
cancellationToken),
helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
if (result.Kind != ServiceDescriptionKind.Stateful)
{
throw new InvalidOperationException(StringHelper.Format(StringResources.Error_InvalidServiceTypeTestability, "DataLoss", "Stateful", action.PartitionSelector.ServiceName, "Stateless"));
}
var getPartitionStateAction = new GetSelectedPartitionStateAction(action.PartitionSelector)
{
RequestTimeout = action.RequestTimeout,
ActionTimeout = helper.GetRemainingTime()
};
await testContext.ActionExecutor.RunAsync(getPartitionStateAction, cancellationToken).ConfigureAwait(false);
Guid partitionId = getPartitionStateAction.Result.PartitionId;
long preDataLossNumber = 0;
ServicePartitionList partitionsResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.QueryManager.GetPartitionListAsync(
action.PartitionSelector.ServiceName,
null,
action.RequestTimeout,
cancellationToken),
helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
bool partitionFound = false;
foreach (StatefulServicePartition partition in partitionsResult)
{
if (partition.PartitionInformation.Id == partitionId)
{
preDataLossNumber = partition.PrimaryEpoch.DataLossNumber;
partitionFound = true;
break;
}
}
if (!partitionFound)
{
throw new FabricException(StringHelper.Format(StringResources.Error_PartitionNotFound), FabricErrorCode.PartitionNotFound);
}
long postDataLossNumber = preDataLossNumber;
do
{
ActionTraceSource.WriteInfo(
TraceType,
"InvokeDataLossAction action pending time:{0}",
helper.GetRemainingTime());
if (helper.GetRemainingTime() <= TimeSpan.Zero)
{
throw new TimeoutException(StringHelper.Format(StringResources.Error_TestabilityActionTimeout, "InvokeDataLoss", partitionId));
}
ServiceReplicaList replicasResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.QueryManager.GetReplicaListAsync(
partitionId,
0,
action.RequestTimeout,
cancellationToken),
helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
ServiceReplicaList fmReplicasResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.QueryManager.GetReplicaListAsync(
Constants.FmPartitionId,
0,
action.RequestTimeout,
cancellationToken),
helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
string fmPrimaryNodeName = string.Empty;
var readyFMReplicas = fmReplicasResult.Where(r => r.ReplicaStatus == ServiceReplicaStatus.Ready).ToArray();
foreach (var replica in readyFMReplicas)
{
StatefulServiceReplica statefulReplica = replica as StatefulServiceReplica;
ReleaseAssert.AssertIf(statefulReplica == null, "FM Replica is not a stateful replica");
if (statefulReplica.ReplicaRole == ReplicaRole.Primary)
{
fmPrimaryNodeName = replica.NodeName;
}
}
if (string.IsNullOrEmpty(fmPrimaryNodeName))
{
throw new FabricException(StringHelper.Format(StringResources.Error_PartitionPrimaryNotReady, "FailoverManager"), FabricErrorCode.NotReady);
}
UnreliableTransportBehavior behavior = new UnreliableTransportBehavior("*", "DoReconfiguration");
behavior.AddFilterForPartitionId(partitionId);
string behaviorName = "BlockDoReconfiguration";
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.TestManager.AddUnreliableTransportBehaviorAsync(
fmPrimaryNodeName,
behaviorName,
behavior,
action.RequestTimeout,
cancellationToken),
helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
// TODO: Wait for some time so that the unreliable transport behavior can be read from the files.
// Bug#2271465 - Unreliable transport through API should return only once the behavior has been successfully applied
await Task.Delay(TimeSpan.FromSeconds(5.0), cancellationToken).ConfigureAwait(false);
bool triedToRemovedBehavior = false;
try
{
var stableReplicasToRemove = replicasResult.Where(r => r.ReplicaStatus == ServiceReplicaStatus.Ready).ToArray();
ActionTraceSource.WriteInfo(TraceType, "Total number of replicas found {0}:{1}", replicasResult.Count(), stableReplicasToRemove.Count());
int replicasToRestartWithoutPrimary =
action.DataLossMode == DataLossMode.FullDataLoss
? stableReplicasToRemove.Length - 1
: (stableReplicasToRemove.Length + 1) / 2 - 1;
foreach (var replica in stableReplicasToRemove)
{
var currentReplica = replica;
StatefulServiceReplica statefulReplica = currentReplica as StatefulServiceReplica;
ReleaseAssert.AssertIf(statefulReplica == null, "Service Replica is not of stateful type even though service is stateful");
ActionTraceSource.WriteInfo(
TraceType,
"Inspecting replica {0}:{1} with role {2} and status {3} to induce data loss",
currentReplica.Id,
partitionId,
statefulReplica.ReplicaRole,
statefulReplica.ReplicaStatus);
if (statefulReplica.ReplicaRole != ReplicaRole.Primary)
{
replicasToRestartWithoutPrimary--;
}
if (replicasToRestartWithoutPrimary >= 0 || statefulReplica.ReplicaRole == ReplicaRole.Primary)
{
ActionTraceSource.WriteInfo(TraceType, "Removing replica {0}:{1} to induce data loss", currentReplica.Id, partitionId);
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.FaultManager.RemoveReplicaAsync(
currentReplica.NodeName,
partitionId,
currentReplica.Id,
CompletionMode.DoNotVerify,
false, /*force remove*/
action.RequestTimeout.TotalSeconds,
cancellationToken),
helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
}
}
triedToRemovedBehavior = true;
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.TestManager.RemoveUnreliableTransportBehaviorAsync(
fmPrimaryNodeName,
behaviorName,
action.RequestTimeout,
cancellationToken),
FabricClientRetryErrors.RemoveUnreliableTransportBehaviorErrors.Value,
helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
// TODO: Wait for some time so that the removal of this unreliable transport behavior can be read from the files.
// Bug#2271465 - Unreliable transport through API should return only once the behavior has been successully applied
await Task.Delay(TimeSpan.FromSeconds(5.0), cancellationToken).ConfigureAwait(false);
// retry check for whether data loss number has increased 5 times else do the entire process again
const int maxRetryCount = 5;
int retryCount = 0;
do
{
partitionsResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.QueryManager.GetPartitionListAsync(
action.PartitionSelector.ServiceName,
null,
action.RequestTimeout,
cancellationToken),
FabricClientRetryErrors.GetPartitionListFabricErrors.Value,
helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
partitionFound = false;
foreach (StatefulServicePartition partition in partitionsResult)
{
if (partition.PartitionInformation.Id == partitionId)
{
postDataLossNumber = partition.PrimaryEpoch.DataLossNumber;
partitionFound = true;
break;
}
}
if (!partitionFound)
{
throw new FabricException(StringHelper.Format(StringResources.Error_PartitionNotFound), FabricErrorCode.PartitionNotFound);
}
ActionTraceSource.WriteInfo(
TraceType,
"Checking data loss numbers for partition {0} with retryCount {1}. Current numbers {2}:{3}",
partitionId,
retryCount,
preDataLossNumber,
postDataLossNumber);
if (postDataLossNumber != preDataLossNumber)
{
break;
}
await AsyncWaiter.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken);
++retryCount;
} while (retryCount < maxRetryCount);
}
finally
{
if (!triedToRemovedBehavior)
{
ActionTraceSource.WriteWarning(TraceType, "Exception after adding behavior to block messages. Removing behavior synchronously");
FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.TestManager.RemoveUnreliableTransportBehaviorAsync(
fmPrimaryNodeName,
behaviorName,
action.RequestTimeout,
cancellationToken),
FabricClientRetryErrors.RemoveUnreliableTransportBehaviorErrors.Value,
helper.GetRemainingTime(),
cancellationToken).GetAwaiter().GetResult();
// TODO: Wait for some time so that the removal of this unreliable transport behavior can be read from the files.
// Bug#2271465 - Unreliable transport through API should return only once the behavior has been successully applied
Task.Delay(TimeSpan.FromSeconds(5.0), cancellationToken).GetAwaiter().GetResult();
}
}
}while (postDataLossNumber == preDataLossNumber);
ActionTraceSource.WriteInfo(
TraceType,
"InvokeDataLossAction action completed postDataLossNumber:{0}, preDataLossNumber:{1}",
postDataLossNumber, preDataLossNumber);
action.Result = new InvokeDataLossResult(getPartitionStateAction.Result);
this.ResultTraceString = StringHelper.Format("InvokeDataLossAction succeeded for {0} with DatalossMode = {1}", partitionId, action.DataLossMode);
}
protected override async Task ExecuteActionAsync(FabricTestContext testContext, MovePrimaryAction action, CancellationToken cancellationToken)
{
ThrowIf.Null(action.PartitionSelector, "PartitionSelector");
this.helper = new TimeoutHelper(action.ActionTimeout);
string newPrimaryNodeName = action.NodeName;
var getPartitionStateAction = new GetSelectedPartitionStateAction(action.PartitionSelector)
{
RequestTimeout = action.RequestTimeout,
ActionTimeout = this.helper.GetRemainingTime()
};
await testContext.ActionExecutor.RunAsync(getPartitionStateAction, cancellationToken);
Guid partitionId = getPartitionStateAction.Result.PartitionId;
if (!action.IgnoreConstraints)
{
// select random node where replica's primary not present
var nodesInfo = await testContext.FabricCluster.GetLatestNodeInfoAsync(action.RequestTimeout, this.helper.GetRemainingTime(), cancellationToken);
if ((nodesInfo == null || nodesInfo.Count() == 0))
{
throw new InvalidOperationException(StringHelper.Format(StringResources.Error_NotEnoughNodesForTestabilityAction, "MovePrimary"));
}
ServiceReplicaList replicasResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.QueryManager.GetReplicaListAsync(
partitionId,
0,
action.RequestTimeout,
cancellationToken),
this.helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
NodeInfo currentPrimaryNodeInfo = null;
string currentPrimaryNodeName = string.Empty;
foreach (var replica in replicasResult)
{
StatefulServiceReplica statefulReplica = replica as StatefulServiceReplica;
if (statefulReplica == null)
{
throw new InvalidOperationException(StringHelper.Format(StringResources.Error_InvalidServiceTypeTestability, "MovePrimary", "Stateful", action.PartitionSelector.ServiceName, "Stateless"));
}
if (statefulReplica.ReplicaRole == ReplicaRole.Primary)
{
currentPrimaryNodeInfo = nodesInfo.FirstOrDefault(n => n.NodeName == statefulReplica.NodeName);
if (!string.IsNullOrEmpty(newPrimaryNodeName) && newPrimaryNodeName == statefulReplica.NodeName)
{
throw new FabricException(
StringHelper.Format(StringResources.Error_InvalidNodeNameProvided, newPrimaryNodeName, "MovePrimary", "Primary already exists on node"),
FabricErrorCode.AlreadyPrimaryReplica);
}
break;
}
}
if (currentPrimaryNodeInfo == null)
{
throw new FabricException(StringHelper.Format(StringResources.Error_PartitionPrimaryNotReady, action.PartitionSelector + ":" + partitionId), FabricErrorCode.NotReady);
}
currentPrimaryNodeName = currentPrimaryNodeInfo.NodeName;
if (newPrimaryNodeName == currentPrimaryNodeName)
{
throw new FabricException(
StringHelper.Format(StringResources.Error_InvalidNodeNameProvided, newPrimaryNodeName, "MovePrimary", "Primary already exists on node"),
FabricErrorCode.AlreadyPrimaryReplica);
}
}
ActionTraceSource.WriteInfo(TraceSource, "Calling move primary with node {0}, partition {1}", string.IsNullOrEmpty(newPrimaryNodeName) ? "Random" : newPrimaryNodeName, partitionId);
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.FaultManager.MovePrimaryUsingNodeNameAsync(
newPrimaryNodeName,
getPartitionStateAction.Result.ServiceName,
partitionId,
action.IgnoreConstraints,
action.RequestTimeout,
cancellationToken),
FabricClientRetryErrors.MovePrimaryFabricErrors.Value,
this.helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
action.Result = new MovePrimaryResult(newPrimaryNodeName, getPartitionStateAction.Result);
ResultTraceString = StringHelper.Format("MovePrimaryAction succeeded for moving Primary for {0} to node {1}.", partitionId, newPrimaryNodeName);
}