private void GenerateAndEnqueueRandomActions(IReadOnlyCollection <NodeEntity> nodeList, bool unsafeModeEnabled, Guid clusterSnapshotId, Guid activityId = default(Guid))
{
var nodeStateHelper = new NodeFaultInformationHelper(
nodeList,
this.testParameters.MinLiveNodesRatio,
this.testParameters.MinLiveSeedNodesRatio,
unsafeModeEnabled,
this.Random);
var action = this.CreateNodeFaultAction(nodeStateHelper, clusterSnapshotId, activityId);
this.EnqueueAction(action);
}
// Returns a nodeInfo of a node whose status can be toggled. Returns null if can't find a valid node.
private NodeStateTransitionAction CreateOneNodeFaultAction(NodeFaultInformationHelper nodeStateHelper, Guid clusterSnapshotId, Guid activityId = default(Guid))
{
NodeStateTransitionAction action = null;
NodeEntity selectedNode = null;
for (int trial = 0; trial < maxTrialPerActionGeneration && action == null; trial++)
{
// Choose category
NodeFaultCategory faultCategory = (NodeFaultCategory)this.nodeFaultCategoryChooser.NextRoll();
switch (faultCategory)
{
case NodeFaultCategory.RestartFault:
{
selectedNode = nodeStateHelper.GetUpNodeToFault(activityId);
if (selectedNode != null)
{
var nodeInfo = nodeStateHelper.ClusterNodes.FirstOrDefault(n => n.CurrentNodeInfo.NodeName == selectedNode.CurrentNodeInfo.NodeName);
ReleaseAssert.AssertIf(nodeInfo == null, "NodeInfo should not be null");
TestabilityTrace.TraceSource.WriteInfo(this.TraceType, "{0}: '{1}' is chosen for RestartNode.", activityId, selectedNode);
action = new RestartNodeStateTransitionAction(nodeInfo.CurrentNodeInfo.NodeName, (long)nodeInfo.CurrentNodeInfo.NodeInstanceId, clusterSnapshotId);
}
}
break;
case NodeFaultCategory.StartStopFault:
{
selectedNode = this.Random.NextDouble() > 0.5
? nodeStateHelper.GetUpNodeToFault(activityId)
: nodeStateHelper.GetDownNodeToFault(activityId);
if (selectedNode != null)
{
var node = selectedNode;
var nodeInfo = nodeStateHelper.ClusterNodes.FirstOrDefault(n => n.CurrentNodeInfo.NodeName == node.CurrentNodeInfo.NodeName);
ReleaseAssert.AssertIf(nodeInfo == null, "NodeInfo should not be null");
if (nodeInfo.CurrentNodeInfo.IsNodeUp)
{
TestabilityTrace.TraceSource.WriteInfo(this.TraceType, "{0}: '{1}' is chosen for StopNode.", activityId, selectedNode);
action = new StopNodeStateTransitionAction(nodeInfo.CurrentNodeInfo.NodeName, (long)nodeInfo.CurrentNodeInfo.NodeInstanceId, clusterSnapshotId);
nodeInfo.CurrentNodeInfo.IsNodeUp = false;
}
else
{
TestabilityTrace.TraceSource.WriteInfo(this.TraceType, "{0}: '{1}' is chosen for StartNode.", activityId, selectedNode);
action = new StartNodeStateTransitionAction(nodeInfo.CurrentNodeInfo.NodeName, (long)nodeInfo.CurrentNodeInfo.NodeInstanceId, clusterSnapshotId);
}
}
}
break;
default:
throw new ArgumentException("Unknown category:" + faultCategory);
}
// Validate this generated action.
if (action != null)
{
ReleaseAssert.AssertIf(selectedNode == null, "selectedNode cannot be null");
selectedNode.FaultNode(activityId);
}
}
return(action);
}
// Creates and returns random nodes whose state can be toggled
private StateTransitionAction CreateNodeFaultAction(NodeFaultInformationHelper nodeStateHelper, Guid clusterSnapshotId, Guid activityId = default(Guid))
{
var action = this.CreateOneNodeFaultAction(nodeStateHelper, clusterSnapshotId, activityId);
return(action);
}
