protected PythonStatementSemantic(PythonStatementSemantic original, Cloner cloner)
: base(original, cloner)
{
TreeNodePrefixPos = original.TreeNodePrefixPos;
before = original.before;
after = original.after;
executedCases = original.executedCases;
}
// copied from AbstractSemanticAnalyzationCrossover
private void CheckDifference(PythonStatementSemantic beforeSemantics, PythonStatementSemantic afterSemantics)
{
SemanticallyDifferentFromRootedParentParameter.ActualValue = new BoolValue(false);
// check all results
var resKeys = beforeSemantics.Before.Keys.Where(x => x.StartsWith("res"));
foreach (var resKey in resKeys)
{
var beforeRes = beforeSemantics.After.Keys.Contains(resKey) ? beforeSemantics.After[resKey] : beforeSemantics.Before[resKey];
var afterRes = afterSemantics.After.Keys.Contains(resKey) ? afterSemantics.After[resKey] : afterSemantics.Before[resKey];
var enumParent = beforeRes.GetEnumerator();
var enumChild = afterRes.GetEnumerator();
var type = ProblemData.Variables.GetTypesOfVariables().First(x => x.Value.Contains(resKey)).Key;
if (type.IsListType())
{
// always move forward both enumerators (do not use short-circuit evaluation!)
while (enumParent.MoveNext() & enumChild.MoveNext())
{
if (!JToken.EqualityComparer.Equals((JArray)enumParent.Current, (JArray)enumChild.Current))
{
SemanticallyDifferentFromRootedParentParameter.ActualValue.Value = true;
break;
}
}
if (enumParent.MoveNext() || enumChild.MoveNext())
{
SemanticallyDifferentFromRootedParentParameter.ActualValue.Value = true;
}
}
else
{
// always move forward both enumerators (do not use short-circuit evaluation!)
while (enumParent.MoveNext() & enumChild.MoveNext())
{
if (!enumParent.Current.Equals(enumChild.Current))
{
SemanticallyDifferentFromRootedParentParameter.ActualValue.Value = true;
break;
}
}
if (enumParent.MoveNext() || enumChild.MoveNext())
{
SemanticallyDifferentFromRootedParentParameter.ActualValue.Value = true;
}
}
// break if a change has already been found
if (SemanticallyDifferentFromRootedParentParameter.ActualValue.Value)
{
break;
}
}
}
public override void ReplaceBranch(IRandom random, ISymbolicExpressionTree symbolicExpressionTree, ICFGPythonProblemData problemData, ItemArray <PythonStatementSemantic> semantics, PythonProcess pythonProcess, double timeout, int maxTreeLength, int maxTreeDepth, int maximumSemanticTries)
{
if (semantics == null || semantics.Length == 0)
{
ReplaceBranchManipulation.ReplaceRandomBranch(random, symbolicExpressionTree, maxTreeLength, maxTreeDepth);
SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(NoSemantics);
MutationTypeParameter.ActualValue = new IntValue(RandomMutation);
return;
}
var statementProductionNames = SemanticOperatorHelper.GetSemanticProductionNames(symbolicExpressionTree.Root.Grammar);
var variables = problemData.Variables.GetVariableNames().ToList();
string variableSettings = problemData.VariableSettings.Count == 0 ? String.Empty : String.Join(Environment.NewLine, problemData.VariableSettings.Select(x => x.Value));
var allowedSymbols = new List <ISymbol>();
// repeat until a fitting parent and child are found (MAX_TRIES times)
int tries = 0;
int semanticTries = 0;
List <JObject> saveOriginalSemantics = null;
List <JObject> saveReplaceSemantics = null;
List <Tuple <ISymbolicExpressionTreeNode, ISymbolicExpressionTreeNode, int> > possibleChildren = null; // Item1 = parent, Item2 = seedNode, Item3 = childIndex
if (UsesAdditionalSemanticMeasure())
{
saveOriginalSemantics = new List <JObject>(semanticTries);
saveReplaceSemantics = new List <JObject>(semanticTries);
possibleChildren = new List <Tuple <ISymbolicExpressionTreeNode, ISymbolicExpressionTreeNode, int> >(semanticTries);
}
bool success = false;
do
{
#region find mutation point
#pragma warning disable 612, 618
ISymbolicExpressionTreeNode parent = symbolicExpressionTree.Root.IterateNodesPrefix().Skip(1).Where(n => n.SubtreeCount > 0).SelectRandom(random);
#pragma warning restore 612, 618
int childIndex = random.Next(parent.SubtreeCount);
var child = parent.GetSubtree(childIndex);
int maxLength = maxTreeLength - symbolicExpressionTree.Length + child.GetLength();
int maxDepth = maxTreeDepth - symbolicExpressionTree.Root.GetBranchLevel(child);
allowedSymbols.Clear();
foreach (var symbol in parent.Grammar.GetAllowedChildSymbols(parent.Symbol, childIndex))
{
// check basic properties that the new symbol must have
if ((symbol.Name != child.Symbol.Name || symbol.MinimumArity > 0) &&
symbol.InitialFrequency > 0 &&
parent.Grammar.GetMinimumExpressionDepth(symbol) <= maxDepth &&
parent.Grammar.GetMinimumExpressionLength(symbol) <= maxLength)
{
allowedSymbols.Add(symbol);
}
}
#endregion
#region check for semantic difference with a new random tree
if (allowedSymbols.Count > 0)
{
if (semanticTries <= maximumSemanticTries)
{
// do semantic mutation
#region calculate original json output
ISymbolicExpressionTreeNode statement = SemanticOperatorHelper.GetStatementNode(child, statementProductionNames);
var statementPos0 = symbolicExpressionTree.IterateNodesPrefix().ToList().IndexOf(statement);
PythonStatementSemantic curSemantics = null;
if (String.IsNullOrEmpty(variableSettings))
{
curSemantics = semantics.First(x => x.TreeNodePrefixPos == statementPos0);
variableSettings = SemanticOperatorHelper.SemanticToPythonVariableSettings(curSemantics.Before, problemData.Variables.GetVariableTypes());
}
var jsonOriginal = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout);
// compare jsonOriginal to semantic after! Maybe avoid additional evaluation.
#endregion
var seedNode = GenerateAndInsertNewSubtree(random, parent, allowedSymbols, childIndex, maxLength, maxDepth);
#region calculate new json output
JObject jsonReplaced;
if (child == statement)
{
// child is executable, so is the new child
jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(seedNode, pythonProcess, random, problemData, variables, variableSettings, timeout);
}
else
{
jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout);
}
var exception = jsonOriginal["exception"] != null || jsonReplaced["exception"] != null;
if (exception)
{
if (jsonOriginal["exception"] != null)
{
MutationExceptionsParameter.ActualValue.Add(new StringValue(jsonOriginal["exception"].ToString()));
}
if (jsonReplaced["exception"] != null)
{
MutationExceptionsParameter.ActualValue.Add(new StringValue(jsonReplaced["exception"].ToString()));
}
}
if (curSemantics != null && !exception)
{
jsonOriginal = PythonSemanticComparer.ReplaceNotExecutedCases(jsonOriginal, curSemantics.Before, curSemantics.ExecutedCases);
jsonReplaced = PythonSemanticComparer.ReplaceNotExecutedCases(jsonReplaced, curSemantics.Before, curSemantics.ExecutedCases);
jsonOriginal = PythonSemanticComparer.ProduceDifference(jsonOriginal, curSemantics.Before);
jsonReplaced = PythonSemanticComparer.ProduceDifference(jsonReplaced, curSemantics.Before);
}
if (!exception && SemanticMeasure(jsonOriginal, jsonReplaced))
{
success = true;
SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(Different);
TypeSelectedForSimilarityParameter.ActualValue = new StringValue("First Semantic");
MutationTypeParameter.ActualValue = new IntValue(SemanticMutation);
}
else
{
// undo mutation
parent.RemoveSubtree(childIndex);
parent.InsertSubtree(childIndex, child);
allowedSymbols.Clear();
if (!exception && UsesAdditionalSemanticMeasure())
{
saveOriginalSemantics.Add(jsonOriginal);
saveReplaceSemantics.Add(jsonReplaced);
possibleChildren.Add(new Tuple <ISymbolicExpressionTreeNode, ISymbolicExpressionTreeNode, int>(parent, seedNode, childIndex));
}
}
if (problemData.VariableSettings.Count == 0)
{
// reset variableSettings
variableSettings = String.Empty;
}
semanticTries++;
#endregion
#region try second semantic comparison
if (!success && semanticTries >= maximumSemanticTries && UsesAdditionalSemanticMeasure())
{
for (int index = 0; index < saveOriginalSemantics.Count; index++)
{
if (AdditionalSemanticMeasure(saveOriginalSemantics[index], saveReplaceSemantics[index]))
{
var mutation = possibleChildren[index];
mutation.Item1.RemoveSubtree(mutation.Item3);
mutation.Item1.InsertSubtree(mutation.Item3, mutation.Item2);
success = true;
SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(Different);
TypeSelectedForSimilarityParameter.ActualValue = new StringValue("Second Semantic");
MutationTypeParameter.ActualValue = new IntValue(SemanticMutation);
break;
}
}
}
#endregion
}
else
{
// do random mutation
#region calculate original json output
ISymbolicExpressionTreeNode statement = SemanticOperatorHelper.GetStatementNode(child, statementProductionNames);
var statementPos0 = symbolicExpressionTree.IterateNodesPrefix().ToList().IndexOf(statement);
if (String.IsNullOrEmpty(variableSettings))
{
variableSettings = SemanticOperatorHelper.SemanticToPythonVariableSettings(semantics.First(x => x.TreeNodePrefixPos == statementPos0).Before, problemData.Variables.GetVariableTypes());
}
var jsonOriginal = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout);
#endregion
var seedNode = GenerateAndInsertNewSubtree(random, parent, allowedSymbols, childIndex, maxLength, maxDepth);
JObject jsonReplaced;
if (child == statement)
{
// child is executable, so is the new child
jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(seedNode, pythonProcess, random, problemData, variables, variableSettings, timeout);
}
else
{
jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout);
}
if (JToken.EqualityComparer.Equals(jsonOriginal, jsonReplaced))
{
SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(Equvivalent);
}
else
{
SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(Different);
}
TypeSelectedForSimilarityParameter.ActualValue = new StringValue("Random Crossover; Reached Max Semantic Tries");
MutationTypeParameter.ActualValue = new IntValue(RandomMutation);
success = true;
}
}
#endregion
tries++;
} while (tries < MAX_TRIES && !success);
NumberOfTriesParameter.ActualValue = new IntValue(semanticTries);
if (SemanticallyEquivalentMutationParameter.ActualValue == null)
{
SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(NoMutation);
TypeSelectedForSimilarityParameter.ActualValue = new StringValue("No mutation");
MutationTypeParameter.ActualValue = new IntValue(NoMutation);
}
}
/// <summary>
/// Swaps the child node of the cutpoint with the selected branch. Semantics of the child branch will be removed from semantics0. Semantics from semantics1 which belonged to the selected branch will be added to semantics0.
/// </summary>
/// <param name="crossoverPoint">Defines parent and child node from the parent0</param>
/// <param name="selectedBranch">Branch to crossover from parent1</param>
/// <param name="parent0">Parent0</param>
/// <param name="parent1">Parent1</param>
/// <param name="semantics0">Semantics of parent0</param>
/// <param name="semantics1">Semantics of parent1</param>
protected static ItemArray <PythonStatementSemantic> SemanticSwap(CutPoint crossoverPoint, ISymbolicExpressionTreeNode selectedBranch, ISymbolicExpressionTree parent0, ISymbolicExpressionTree parent1, ItemArray <PythonStatementSemantic> semantics0, ItemArray <PythonStatementSemantic> semantics1)
{
var allNodes0Prefix = parent0.IterateNodesPrefix().ToList();
Dictionary <ISymbolicExpressionTreeNode, PythonStatementSemantic> sem0ByNode = new Dictionary <ISymbolicExpressionTreeNode, PythonStatementSemantic>();
var sem0ByNodeIndex = semantics0.ToDictionary(x => x.TreeNodePrefixPos, y => y);
for (int i = 0; i < allNodes0Prefix.Count; i++)
{
if (sem0ByNodeIndex.ContainsKey(i))
{
sem0ByNode.Add(allNodes0Prefix[i], sem0ByNodeIndex[i]);
}
}
var allNodes1Prefix = parent1.IterateNodesPrefix().ToList();
Dictionary <ISymbolicExpressionTreeNode, PythonStatementSemantic> sem1ByNode = new Dictionary <ISymbolicExpressionTreeNode, PythonStatementSemantic>();
var sem1ByNodeIndex = semantics1.ToDictionary(x => x.TreeNodePrefixPos, y => y);
for (int i = 0; i < allNodes1Prefix.Count; i++)
{
if (sem1ByNodeIndex.ContainsKey(i))
{
sem1ByNode.Add(allNodes1Prefix[i], sem1ByNodeIndex[i]);
}
}
if (crossoverPoint.Child != null)
{
// manipulate the tree of parent0 in place
// replace the branch in tree0 with the selected branch from tree1
crossoverPoint.Parent.RemoveSubtree(crossoverPoint.ChildIndex);
if (selectedBranch != null)
{
crossoverPoint.Parent.InsertSubtree(crossoverPoint.ChildIndex, selectedBranch);
}
}
else
{
// child is null (additional child should be added under the parent)
if (selectedBranch != null)
{
crossoverPoint.Parent.AddSubtree(selectedBranch);
}
}
List <PythonStatementSemantic> newSemantics = new List <PythonStatementSemantic>();
var newNodes0Prefix = parent0.IterateNodesPrefix().ToList();
for (int i = 0; i < newNodes0Prefix.Count; i++)
{
PythonStatementSemantic sem = null;
if (sem0ByNode.ContainsKey(newNodes0Prefix[i]))
{
sem = sem0ByNode[newNodes0Prefix[i]];
sem.TreeNodePrefixPos = i;
}
else if (sem1ByNode.ContainsKey(newNodes0Prefix[i]))
{
sem = sem1ByNode[newNodes0Prefix[i]];
sem.TreeNodePrefixPos = i;
}
if (sem != null)
{
newSemantics.Add(sem);
}
}
return(new ItemArray <PythonStatementSemantic>(newSemantics));
}
protected PythonStatementSemantic(PythonStatementSemantic original, Cloner cloner)
: base(original, cloner) {
TreeNodePrefixPos = original.TreeNodePrefixPos;
before = original.before;
after = original.after;
}
protected override ISymbolicExpressionTree Cross(IRandom random, ISymbolicExpressionTree parent0, ISymbolicExpressionTree parent1, ItemArray <PythonStatementSemantic> semantic0, ItemArray <PythonStatementSemantic> semantic1, T problemData, int maxTreeLength, int maxTreeDepth, double internalCrossoverPointProbability, out ItemArray <PythonStatementSemantic> newSemantics)
{
if (semantic0 == null || semantic1 == null || semantic0.Length == 0 || semantic1.Length == 0)
{
parent0 = SubtreeCrossover.Cross(random, parent0, parent1, internalCrossoverPointProbability, maxTreeLength, maxTreeDepth);
newSemantics = null;
TypeSelectedForSimilarityParameter.ActualValue = new StringValue("No Semantics; Random Crossover");
AddStatisticsNoCrossover(NoXoNoSemantics);
return(parent0);
}
newSemantics = semantic0;
var statementProductionNames = SemanticOperatorHelper.GetSemanticProductionNames(parent0.Root.Grammar);
var variables = problemData.Variables.GetVariableNames().ToList();
string variableSettings = problemData.VariableSettings.Count == 0 ? String.Empty : String.Join(Environment.NewLine, problemData.VariableSettings.Select(x => x.Value));
int maximumSemanticTries = MaxComparesParameter.Value.Value;
int semanticTries = 0;
var saveOriginalSemantics = new List <JObject>(maximumSemanticTries);
var saveReplaceSemantics = new List <JObject>(maximumSemanticTries);
var possibleChildren = new List <Tuple <CutPoint, ISymbolicExpressionTreeNode> >(maximumSemanticTries);
bool success = false;
do
{
// select a random crossover point in the first parent
CutPoint crossoverPoint0;
SelectCrossoverPoint(random, parent0, internalCrossoverPointProbability, maxTreeLength, maxTreeDepth, out crossoverPoint0);
int childLength = crossoverPoint0.Child != null?crossoverPoint0.Child.GetLength() : 0;
// calculate the max length and depth that the inserted branch can have
int maxInsertedBranchLength = Math.Max(0, maxTreeLength - (parent0.Length - childLength));
int maxInsertedBranchDepth = Math.Max(0, maxTreeDepth - parent0.Root.GetBranchLevel(crossoverPoint0.Parent));
List <ISymbolicExpressionTreeNode> allowedBranches = new List <ISymbolicExpressionTreeNode>();
parent1.Root.ForEachNodePostfix((n) => {
if (n.GetLength() <= maxInsertedBranchLength &&
n.GetDepth() <= maxInsertedBranchDepth && crossoverPoint0.IsMatchingPointType(n))
{
allowedBranches.Add(n);
}
});
// empty branch
if (crossoverPoint0.IsMatchingPointType(null))
{
allowedBranches.Add(null);
}
if (allowedBranches.Count != 0)
{
var selectedBranch = SelectRandomBranch(random, allowedBranches, internalCrossoverPointProbability);
ISymbolicExpressionTreeNode statement = SemanticOperatorHelper.GetStatementNode(crossoverPoint0.Child, statementProductionNames);
var statementPos0 = parent0.IterateNodesPrefix().ToList().IndexOf(statement);
PythonStatementSemantic curSemantics = null;
if (String.IsNullOrEmpty(variableSettings))
{
curSemantics = semantic0.First(x => x.TreeNodePrefixPos == statementPos0);
variableSettings = SemanticOperatorHelper.SemanticToPythonVariableSettings(curSemantics.Before, problemData.Variables.GetVariableTypes());
}
var jsonOriginal = SemanticOperatorHelper.EvaluateStatementNode(statement, PyProcess, random, problemData, variables, variableSettings, Timeout);
JObject jsonReplaced;
if (statement == crossoverPoint0.Child)
{
// selectedBranch is also executable
jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(selectedBranch, PyProcess, random, problemData, variables, variableSettings, Timeout);
}
else
{
crossoverPoint0.Parent.RemoveSubtree(crossoverPoint0.ChildIndex);
var parent = selectedBranch.Parent; // save parent
crossoverPoint0.Parent.InsertSubtree(crossoverPoint0.ChildIndex, selectedBranch); // this will affect node.Parent
jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(statement, PyProcess, random, problemData, variables, variableSettings, Timeout);
crossoverPoint0.Parent.RemoveSubtree(crossoverPoint0.ChildIndex); // removes intermediate parent from node
selectedBranch.Parent = parent; // restore parent
crossoverPoint0.Parent.InsertSubtree(crossoverPoint0.ChildIndex, crossoverPoint0.Child); // restore cutPoint
}
var exception = jsonOriginal["exception"] != null || jsonReplaced["exception"] != null;
if (exception)
{
if (jsonOriginal["exception"] != null)
{
CrossoverExceptionsParameter.ActualValue.Add(new StringValue(jsonOriginal["exception"].ToString()));
}
if (jsonReplaced["exception"] != null)
{
CrossoverExceptionsParameter.ActualValue.Add(new StringValue(jsonReplaced["exception"].ToString()));
}
}
if (curSemantics != null && !exception)
{
jsonOriginal = PythonSemanticComparer.ReplaceNotExecutedCases(jsonOriginal, curSemantics.Before, curSemantics.ExecutedCases);
jsonReplaced = PythonSemanticComparer.ReplaceNotExecutedCases(jsonReplaced, curSemantics.Before, curSemantics.ExecutedCases);
jsonOriginal = PythonSemanticComparer.ProduceDifference(jsonOriginal, curSemantics.Before);
jsonReplaced = PythonSemanticComparer.ProduceDifference(jsonReplaced, curSemantics.Before);
}
if (!exception && SemanticMeasure(jsonOriginal, jsonReplaced))
{
newSemantics = SemanticSwap(crossoverPoint0, selectedBranch, parent0, parent1, semantic0, semantic1);
SemanticallyEquivalentCrossoverParameter.ActualValue = new IntValue(2);
TypeSelectedForSimilarityParameter.ActualValue = new StringValue("First Semantic");
AddStatistics(semantic0, parent0, statement == crossoverPoint0.Child ? selectedBranch : statement, crossoverPoint0, jsonOriginal, selectedBranch, random, problemData, variables, variableSettings); // parent zero has been changed is now considered the child
success = true;
}
else
{
saveOriginalSemantics.Add(jsonOriginal);
saveReplaceSemantics.Add(jsonReplaced);
possibleChildren.Add(new Tuple <CutPoint, ISymbolicExpressionTreeNode>(crossoverPoint0, selectedBranch));
}
}
semanticTries++;
#region try second semantic comparison
if (!success && semanticTries >= maximumSemanticTries)
{
for (int index = 0; index < saveOriginalSemantics.Count; index++)
{
var exception = saveOriginalSemantics[index]["exception"] == null && saveReplaceSemantics[index]["exception"] == null;
if (!exception && AdditionalSemanticMeasure(saveOriginalSemantics[index], saveReplaceSemantics[index]))
{
var crossover = possibleChildren[index];
crossoverPoint0 = crossover.Item1;
// Recreate jsonOriginal as it might have changed due to ReplaceNotExecutedCases and ProduceDifference
var jsonOriginal = GenerateOriginalJson(crossoverPoint0, statementProductionNames, parent0, variableSettings, semantic0, problemData, random, variables);
newSemantics = SemanticSwap(crossoverPoint0, crossover.Item2, parent0, parent1, semantic0, semantic1);
var statement = SemanticOperatorHelper.GetStatementNode(crossover.Item2, statementProductionNames);
SemanticallyEquivalentCrossoverParameter.ActualValue = new IntValue(2);
TypeSelectedForSimilarityParameter.ActualValue = new StringValue("Second Semantic");
AddStatistics(semantic0, parent0, statement, crossoverPoint0, jsonOriginal, crossover.Item2, random, problemData, variables, variableSettings); // parent zero has been changed is now considered the child
success = true;
break;
}
}
}
#endregion
} while (!success && semanticTries < maximumSemanticTries);
if (!success)
{
// Last change. If any possible crossover was found, do a crossover with the first one
if (saveOriginalSemantics.Any())
{
var crossover = possibleChildren.First();
var crossoverPoint0 = crossover.Item1;
// Recreate jsonOriginal as it might have changed due to ReplaceNotExecutedCases and ProduceDifference
var jsonOriginal = GenerateOriginalJson(crossoverPoint0, statementProductionNames, parent0, variableSettings, semantic0, problemData, random, variables);
newSemantics = SemanticSwap(crossoverPoint0, crossover.Item2, parent0, parent1, semantic0, semantic1);
var statement = SemanticOperatorHelper.GetStatementNode(crossover.Item2, statementProductionNames);
TypeSelectedForSimilarityParameter.ActualValue = new StringValue("Random Crossover; Reached Max Semantic Tries");
AddStatistics(semantic0, parent0, statement, crossoverPoint0, jsonOriginal, crossover.Item2, random, problemData, variables, variableSettings); // parent zero has been changed is now considered the child
}
else
{
AddStatisticsNoCrossover(NoXoNoAllowedBranch);
}
}
saveOriginalSemantics.Clear();
saveReplaceSemantics.Clear();
possibleChildren.Clear();
NumberOfCrossoverTries = semanticTries;
return(parent0);
}