public static bool DeleteSubroutine(
IRandom random,
ISymbolicExpressionTree symbolicExpressionTree,
int maxFunctionDefinitions, int maxFunctionArguments) {
var functionDefiningBranches = symbolicExpressionTree.IterateNodesPrefix().OfType<DefunTreeNode>().ToList();
if (!functionDefiningBranches.Any())
// no ADF to delete => abort
return false;
var selectedDefunBranch = functionDefiningBranches.SampleRandom(random);
// remove the selected defun
int defunSubtreeIndex = symbolicExpressionTree.Root.IndexOfSubtree(selectedDefunBranch);
symbolicExpressionTree.Root.RemoveSubtree(defunSubtreeIndex);
// remove references to deleted function
foreach (var subtree in symbolicExpressionTree.Root.Subtrees.OfType<SymbolicExpressionTreeTopLevelNode>()) {
var matchingInvokeSymbol = (from symb in subtree.Grammar.Symbols.OfType<InvokeFunction>()
where symb.FunctionName == selectedDefunBranch.FunctionName
select symb).SingleOrDefault();
if (matchingInvokeSymbol != null) {
subtree.Grammar.RemoveSymbol(matchingInvokeSymbol);
}
}
DeletionByRandomRegeneration(random, symbolicExpressionTree, selectedDefunBranch);
return true;
}
public static Instruction[] Compile(ISymbolicExpressionTree tree, Func<ISymbolicExpressionTreeNode, byte> opCodeMapper, IEnumerable<Func<Instruction, Instruction>> postInstructionCompiledHooks) {
Dictionary<string, ushort> entryPoint = new Dictionary<string, ushort>();
List<Instruction> code = new List<Instruction>();
// compile main body branches
foreach (var branch in tree.Root.GetSubtree(0).Subtrees) {
code.AddRange(Compile(branch, opCodeMapper, postInstructionCompiledHooks));
}
// compile function branches
var functionBranches = from node in tree.IterateNodesPrefix()
where node.Symbol is Defun
select node;
foreach (DefunTreeNode branch in functionBranches) {
if (code.Count > ushort.MaxValue) throw new ArgumentException("Code for the tree is too long (> ushort.MaxValue).");
entryPoint[branch.FunctionName] = (ushort)code.Count;
code.AddRange(Compile(branch.GetSubtree(0), opCodeMapper, postInstructionCompiledHooks));
}
// address of all functions is fixed now
// iterate through code again and fill in the jump locations
for (int i = 0; i < code.Count; i++) {
Instruction instr = code[i];
if (instr.dynamicNode.Symbol is InvokeFunction) {
var invokeNode = (InvokeFunctionTreeNode)instr.dynamicNode;
instr.data = entryPoint[invokeNode.Symbol.FunctionName];
}
}
return code.ToArray();
}
public static bool DeleteArgument(
IRandom random,
ISymbolicExpressionTree symbolicExpressionTree,
int maxFunctionDefinitions, int maxFunctionArguments) {
var functionDefiningBranches = symbolicExpressionTree.IterateNodesPrefix().OfType<DefunTreeNode>().ToList();
if (!functionDefiningBranches.Any())
// no function defining branch => abort
return false;
var selectedDefunBranch = functionDefiningBranches.SampleRandom(random);
if (selectedDefunBranch.NumberOfArguments <= 1)
// argument deletion by consolidation is not possible => abort
return false;
// the argument to be removed is always the one with the largest index
// (otherwise we would have to decrement the index of the larger argument symbols)
var removedArgument = (from sym in selectedDefunBranch.Grammar.Symbols.OfType<Argument>()
select sym.ArgumentIndex).Distinct().OrderBy(x => x).Last();
// find invocations of the manipulated funcion and remove the specified argument tree
var invocationNodes = (from node in symbolicExpressionTree.IterateNodesPrefix().OfType<InvokeFunctionTreeNode>()
where node.Symbol.FunctionName == selectedDefunBranch.FunctionName
select node).ToList();
foreach (var invokeNode in invocationNodes) {
invokeNode.RemoveSubtree(removedArgument);
}
DeleteArgumentByConsolidation(random, selectedDefunBranch, removedArgument);
// delete the dynamic argument symbol that matches the argument to be removed
var matchingSymbol = selectedDefunBranch.Grammar.Symbols.OfType<Argument>().Single(s => s.ArgumentIndex == removedArgument);
selectedDefunBranch.Grammar.RemoveSymbol(matchingSymbol);
selectedDefunBranch.NumberOfArguments--;
// reduce arity in known functions of all root branches
foreach (var subtree in symbolicExpressionTree.Root.Subtrees) {
var matchingInvokeSymbol = subtree.Grammar.Symbols.OfType<InvokeFunction>().SingleOrDefault(s => s.FunctionName == selectedDefunBranch.FunctionName);
if (matchingInvokeSymbol != null) {
subtree.Grammar.SetSubtreeCount(matchingInvokeSymbol, selectedDefunBranch.NumberOfArguments, selectedDefunBranch.NumberOfArguments);
}
}
return true;
}
public static Instruction[] Compile(ISymbolicExpressionTree tree, Func <ISymbolicExpressionTreeNode, byte> opCodeMapper, IEnumerable <Func <Instruction, Instruction> > postInstructionCompiledHooks)
{
Dictionary <string, ushort> entryPoint = new Dictionary <string, ushort>();
List <Instruction> code = new List <Instruction>();
// compile main body branches
foreach (var branch in tree.Root.GetSubtree(0).Subtrees)
{
code.AddRange(Compile(branch, opCodeMapper, postInstructionCompiledHooks));
}
// compile function branches
var functionBranches = from node in tree.IterateNodesPrefix()
where node.Symbol is Defun
select node;
foreach (DefunTreeNode branch in functionBranches)
{
if (code.Count > ushort.MaxValue)
{
throw new ArgumentException("Code for the tree is too long (> ushort.MaxValue).");
}
entryPoint[branch.FunctionName] = (ushort)code.Count;
code.AddRange(Compile(branch.GetSubtree(0), opCodeMapper, postInstructionCompiledHooks));
}
// address of all functions is fixed now
// iterate through code again and fill in the jump locations
for (int i = 0; i < code.Count; i++)
{
Instruction instr = code[i];
if (instr.dynamicNode.Symbol is InvokeFunction)
{
var invokeNode = (InvokeFunctionTreeNode)instr.dynamicNode;
instr.data = entryPoint[invokeNode.Symbol.FunctionName];
}
}
return(code.ToArray());
}
private static Instruction[] PrepareInterpreterState(ISymbolicExpressionTree tree, IDictionary <string, Interval> variableRanges)
{
if (variableRanges == null)
{
throw new ArgumentNullException("No variablew ranges are present!", nameof(variableRanges));
}
//Check if all variables used in the tree are present in the dataset
foreach (var variable in tree.IterateNodesPrefix().OfType <VariableTreeNode>().Select(n => n.VariableName).Distinct())
{
if (!variableRanges.ContainsKey(variable))
{
throw new InvalidOperationException($"No ranges for variable {variable} is present");
}
}
Instruction[] code = SymbolicExpressionTreeCompiler.Compile(tree, OpCodes.MapSymbolToOpCode);
foreach (Instruction instr in code.Where(i => i.opCode == OpCodes.Variable))
{
var variableTreeNode = (VariableTreeNode)instr.dynamicNode;
instr.data = variableRanges[variableTreeNode.VariableName];
}
return(code);
}
private static IEnumerable<string> UsedFunctionNames(ISymbolicExpressionTree symbolicExpressionTree) {
return from node in symbolicExpressionTree.IterateNodesPrefix()
where node.Symbol is Defun
select ((DefunTreeNode)node).FunctionName;
}
public static bool CreateSubroutine(
IRandom random,
ISymbolicExpressionTree symbolicExpressionTree,
int maxTreeLength, int maxTreeDepth,
int maxFunctionDefinitions, int maxFunctionArguments) {
var functionDefiningBranches = symbolicExpressionTree.IterateNodesPrefix().OfType<DefunTreeNode>();
if (functionDefiningBranches.Count() >= maxFunctionDefinitions)
// allowed maximum number of ADF reached => abort
return false;
if (symbolicExpressionTree.Length + 4 > maxTreeLength)
// defining a new function causes an length increase by 4 nodes (max) if the max tree length is reached => abort
return false;
string formatString = new StringBuilder().Append('0', (int)Math.Log10(maxFunctionDefinitions * 10 - 1)).ToString(); // >= 100 functions => ###
var allowedFunctionNames = from index in Enumerable.Range(0, maxFunctionDefinitions)
select "ADF" + index.ToString(formatString);
// select a random body (either the result producing branch or an ADF branch)
var bodies = from node in symbolicExpressionTree.Root.Subtrees
select new { Tree = node, Length = node.GetLength() };
var totalNumberOfBodyNodes = bodies.Select(x => x.Length).Sum();
int r = random.Next(totalNumberOfBodyNodes);
int aggregatedNumberOfBodyNodes = 0;
ISymbolicExpressionTreeNode selectedBody = null;
foreach (var body in bodies) {
aggregatedNumberOfBodyNodes += body.Length;
if (aggregatedNumberOfBodyNodes > r)
selectedBody = body.Tree;
}
// sanity check
if (selectedBody == null) throw new InvalidOperationException();
// select a random cut point in the selected branch
var allCutPoints = (from parent in selectedBody.IterateNodesPrefix()
from subtree in parent.Subtrees
select new CutPoint(parent, subtree)).ToList();
if (!allCutPoints.Any())
// no cut points => abort
return false;
string newFunctionName = allowedFunctionNames.Except(functionDefiningBranches.Select(x => x.FunctionName)).First();
var selectedCutPoint = allCutPoints.SampleRandom(random);
// select random branches as argument cut-off points (replaced by argument terminal nodes in the function)
List<CutPoint> argumentCutPoints = SelectRandomArgumentBranches(selectedCutPoint.Child, random, ARGUMENT_CUTOFF_PROBABILITY, maxFunctionArguments);
ISymbolicExpressionTreeNode functionBody = selectedCutPoint.Child;
// disconnect the function body from the tree
selectedCutPoint.Parent.RemoveSubtree(selectedCutPoint.ChildIndex);
// disconnect the argument branches from the function
functionBody = DisconnectBranches(functionBody, argumentCutPoints);
// insert a function invocation symbol instead
var invokeNode = (InvokeFunctionTreeNode)(new InvokeFunction(newFunctionName)).CreateTreeNode();
selectedCutPoint.Parent.InsertSubtree(selectedCutPoint.ChildIndex, invokeNode);
// add the branches selected as argument as subtrees of the function invocation node
foreach (var argumentCutPoint in argumentCutPoints)
invokeNode.AddSubtree(argumentCutPoint.Child);
// insert a new function defining branch
var defunNode = (DefunTreeNode)(new Defun()).CreateTreeNode();
defunNode.FunctionName = newFunctionName;
defunNode.AddSubtree(functionBody);
symbolicExpressionTree.Root.AddSubtree(defunNode);
// the grammar in the newly defined function is a clone of the grammar of the originating branch
defunNode.SetGrammar((ISymbolicExpressionTreeGrammar)selectedBody.Grammar.Clone());
var allowedChildSymbols = selectedBody.Grammar.GetAllowedChildSymbols(selectedBody.Symbol);
foreach (var allowedChildSymbol in allowedChildSymbols)
defunNode.Grammar.AddAllowedChildSymbol(defunNode.Symbol, allowedChildSymbol);
var maxSubtrees = selectedBody.Grammar.GetMaximumSubtreeCount(selectedBody.Symbol);
for (int i = 0; i < maxSubtrees; i++) {
foreach (var allowedChildSymbol in selectedBody.Grammar.GetAllowedChildSymbols(selectedBody.Symbol, i))
defunNode.Grammar.AddAllowedChildSymbol(defunNode.Symbol, allowedChildSymbol);
}
// remove all argument symbols from grammar except that one contained in cutpoints
var oldArgumentSymbols = selectedBody.Grammar.Symbols.OfType<Argument>().ToList();
foreach (var oldArgSymb in oldArgumentSymbols)
defunNode.Grammar.RemoveSymbol(oldArgSymb);
// find unique argument indexes and matching symbols in the function defining branch
var newArgumentIndexes = (from node in defunNode.IterateNodesPrefix().OfType<ArgumentTreeNode>()
select node.Symbol.ArgumentIndex).Distinct();
// add argument symbols to grammar of function defining branch
GrammarModifier.AddArgumentSymbol(selectedBody.Grammar, defunNode.Grammar, newArgumentIndexes, argumentCutPoints);
defunNode.NumberOfArguments = newArgumentIndexes.Count();
if (defunNode.NumberOfArguments != argumentCutPoints.Count) throw new InvalidOperationException();
// add invoke symbol for newly defined function to the original branch
GrammarModifier.AddInvokeSymbol(selectedBody.Grammar, defunNode.FunctionName, defunNode.NumberOfArguments, selectedCutPoint, argumentCutPoints);
// when the new function body was taken from another function definition
// add invoke symbol for newly defined function to all branches that are allowed to invoke the original branch
if (selectedBody.Symbol is Defun) {
var originalFunctionDefinition = selectedBody as DefunTreeNode;
foreach (var subtree in symbolicExpressionTree.Root.Subtrees) {
var originalBranchInvokeSymbol = (from symb in subtree.Grammar.Symbols.OfType<InvokeFunction>()
where symb.FunctionName == originalFunctionDefinition.FunctionName
select symb).SingleOrDefault();
// when the original branch can be invoked from the subtree then also allow invocation of the function
if (originalBranchInvokeSymbol != null) {
GrammarModifier.AddInvokeSymbol(subtree.Grammar, defunNode.FunctionName, defunNode.NumberOfArguments, selectedCutPoint, argumentCutPoints);
}
}
}
return true;
}
public static bool DuplicateArgument(
IRandom random,
ISymbolicExpressionTree symbolicExpressionTree,
int maxFunctionDefinitions, int maxFunctionArguments) {
var functionDefiningBranches = symbolicExpressionTree.IterateNodesPrefix().OfType<DefunTreeNode>().ToList();
var allowedArgumentIndexes = Enumerable.Range(0, maxFunctionArguments);
if (!functionDefiningBranches.Any())
// no function defining branches => abort
return false;
var selectedDefunBranch = functionDefiningBranches.SampleRandom(random);
var argumentSymbols = selectedDefunBranch.Grammar.Symbols.OfType<Argument>().ToList();
if (!argumentSymbols.Any() || argumentSymbols.Count() >= maxFunctionArguments)
// when no argument or number of arguments is already at max allowed value => abort
return false;
var selectedArgumentSymbol = argumentSymbols.SampleRandom(random);
var takenIndexes = argumentSymbols.Select(s => s.ArgumentIndex);
var newArgumentIndex = allowedArgumentIndexes.Except(takenIndexes).First();
var newArgSymbol = new Argument(newArgumentIndex);
// replace existing references to the original argument with references to the new argument randomly in the selectedBranch
var argumentNodes = selectedDefunBranch.IterateNodesPrefix().OfType<ArgumentTreeNode>();
foreach (var argNode in argumentNodes) {
if (argNode.Symbol == selectedArgumentSymbol) {
if (random.NextDouble() < 0.5) {
argNode.Symbol = newArgSymbol;
}
}
}
// find invocations of the functions and duplicate the matching argument branch
var invocationNodes = (from node in symbolicExpressionTree.IterateNodesPrefix().OfType<InvokeFunctionTreeNode>()
where node.Symbol.FunctionName == selectedDefunBranch.FunctionName
where node.Subtrees.Count() == selectedDefunBranch.NumberOfArguments
select node).ToList();
// do this repeatedly until no matching invocations are found
while (invocationNodes.Count() > 0) {
List<ISymbolicExpressionTreeNode> newlyAddedBranches = new List<ISymbolicExpressionTreeNode>();
foreach (var invokeNode in invocationNodes) {
// check that the invocation node really has the correct number of arguments
if (invokeNode.Subtrees.Count() != selectedDefunBranch.NumberOfArguments) throw new InvalidOperationException();
var argumentBranch = invokeNode.GetSubtree(selectedArgumentSymbol.ArgumentIndex);
var clonedArgumentBranch = (ISymbolicExpressionTreeNode)argumentBranch.Clone();
invokeNode.InsertSubtree(newArgumentIndex, clonedArgumentBranch);
newlyAddedBranches.Add(clonedArgumentBranch);
}
invocationNodes = (from newlyAddedBranch in newlyAddedBranches
from node in newlyAddedBranch.IterateNodesPrefix().OfType<InvokeFunctionTreeNode>()
where node.Symbol.FunctionName == selectedDefunBranch.FunctionName
where node.Subtrees.Count() == selectedDefunBranch.NumberOfArguments
select node).ToList();
}
// register the new argument symbol and increase the number of arguments of the ADF
selectedDefunBranch.Grammar.AddSymbol(newArgSymbol);
selectedDefunBranch.Grammar.SetSubtreeCount(newArgSymbol, 0, 0);
// allow the duplicated argument as child of all other arguments where the orginal argument was allowed
GrammarModifier.SetAllowedParentSymbols(selectedDefunBranch.Grammar, selectedArgumentSymbol, newArgSymbol);
selectedDefunBranch.NumberOfArguments++;
// increase the arity of the changed ADF in all branches that can use this ADF
foreach (var subtree in symbolicExpressionTree.Root.Subtrees) {
var matchingInvokeSymbol = (from symb in subtree.Grammar.Symbols.OfType<InvokeFunction>()
where symb.FunctionName == selectedDefunBranch.FunctionName
select symb).SingleOrDefault();
if (matchingInvokeSymbol != null) {
subtree.Grammar.SetSubtreeCount(matchingInvokeSymbol, selectedDefunBranch.NumberOfArguments, selectedDefunBranch.NumberOfArguments);
foreach (var symb in subtree.Grammar.Symbols) {
if (symb is StartSymbol || symb is ProgramRootSymbol) continue;
if (subtree.Grammar.IsAllowedChildSymbol(matchingInvokeSymbol, symb, selectedArgumentSymbol.ArgumentIndex))
subtree.Grammar.AddAllowedChildSymbol(matchingInvokeSymbol, symb, newArgumentIndex);
}
}
}
return true;
}
protected override ISymbolicExpressionTree Cross(IRandom random, ISymbolicExpressionTree parent0, ISymbolicExpressionTree parent1, ItemArray <PythonStatementSemantic> semantic0, ItemArray <PythonStatementSemantic> semantic1, T problemData, int maxTreeLength, int maxTreeDepth, double internalCrossoverPointProbability, out ItemArray <PythonStatementSemantic> newSemantics)
{
if (semantic0 == null || semantic1 == null || semantic0.Length == 0 || semantic1.Length == 0)
{
parent0 = SubtreeCrossover.Cross(random, parent0, parent1, internalCrossoverPointProbability, maxTreeLength, maxTreeDepth);
newSemantics = null;
AddStatisticsNoCrossover(NoXoNoSemantics);
return(parent0);
}
newSemantics = semantic0;
// select andom crossover points in the first parent
var crossoverPoints0 = SelectCrossoverPointsOfMatchingType(random, parent0, internalCrossoverPointProbability, maxTreeLength, maxTreeDepth, NumberOfCutPointsFirst);
var primaryCrossoverPoint = crossoverPoints0.First();
int childLength = primaryCrossoverPoint.Child != null?primaryCrossoverPoint.Child.GetLength() : 0;
// calculate the max length and depth that the inserted branch can have
int maxInsertedBranchLength = maxTreeLength - (parent0.Length - childLength);
int maxInsertedBranchDepth = maxTreeDepth - parent0.Root.GetBranchLevel(primaryCrossoverPoint.Child);
List <ISymbolicExpressionTreeNode> allowedBranches = new List <ISymbolicExpressionTreeNode>();
parent1.Root.ForEachNodePostfix((n) => {
if (n.GetLength() <= maxInsertedBranchLength &&
n.GetDepth() <= maxInsertedBranchDepth && primaryCrossoverPoint.IsMatchingPointType(n))
{
allowedBranches.Add(n);
}
});
// empty branch
if (primaryCrossoverPoint.IsMatchingPointType(null))
{
allowedBranches.Add(null);
}
// set NumberOfAllowedBranches
NumberOfAllowedBranches = allowedBranches.Count;
if (allowedBranches.Count == 0)
{
AddStatisticsNoCrossover(NoXoNoAllowedBranch);
return(parent0);
}
// select NumberOfCutPointsSecond random crossover points
// Ignore internalCrossoverPointProbability, was already used to select cutpoint and the cutpoint child already reduces possibilities
var compBranches = allowedBranches.SampleRandomWithoutRepetition(random, NumberOfCutPointsSecond).ToList(); // convert to list so that the linq expression is only executed once, to always get the same branches
var allowedBranchesPerCutpoint = new List <IEnumerable <ISymbolicExpressionTreeNode> >()
{
compBranches
};
allowedBranchesPerCutpoint.AddRange(crossoverPoints0.Skip(1).Select(x => FindFittingNodes(x, parent0, compBranches, maxTreeLength, maxTreeDepth)));
// set NumberOfPossibleBranchesSelected
NumberOfPossibleBranchesSelected = compBranches.Count;
var statementProductionNames = SemanticOperatorHelper.GetSemanticProductionNames(primaryCrossoverPoint.Parent.Grammar);
// find first node that can be used for evaluation in parent0
ISymbolicExpressionTreeNode statement = SemanticOperatorHelper.GetStatementNode(primaryCrossoverPoint.Child, statementProductionNames);
if (statement == null)
{
newSemantics = SemanticSwap(primaryCrossoverPoint, compBranches.SampleRandom(random), parent0, parent1, semantic0, semantic1);
AddStatisticsNoCrossover(NoXoNoStatement);
return(parent0);
}
var statementPos0 = parent0.IterateNodesPrefix().ToList().IndexOf(statement);
string variableSettings;
if (problemData.VariableSettings.Count == 0)
{
variableSettings = SemanticOperatorHelper.SemanticToPythonVariableSettings(semantic0.First(x => x.TreeNodePrefixPos == statementPos0).Before, problemData.Variables.GetVariableTypes());
}
else
{
variableSettings = String.Join(Environment.NewLine, problemData.VariableSettings.Select(x => x.Value));
}
var variables = problemData.Variables.GetVariableNames().ToList();
// only used for analyzation, otherwise could be removed
var json0 = SemanticOperatorHelper.EvaluateStatementNode(statement, PyProcess, random, problemData, variables, variableSettings, Timeout);
ISymbolicExpressionTreeNode selectedBranch;
CutPoint selectedCutPoint;
if (statement == primaryCrossoverPoint.Child)
{
SelectBranchFromExecutableNodes(crossoverPoints0.ToList(), allowedBranchesPerCutpoint, random, variables, variableSettings, problemData, out selectedBranch, out selectedCutPoint);
}
else
{
//SelectBranchFromNodes1(statement, crossoverPoints0.ToList(), allowedBranchesPerCutpoint, random, variables, variableSettings, problemData, out selectedBranch, out selectedCutPoint);
SelectBranchFromNodes2(statementProductionNames, crossoverPoints0.ToList(), allowedBranchesPerCutpoint, random, variables, variableSettings, problemData, out selectedBranch, out selectedCutPoint);
}
// perform the actual swap
if (selectedBranch != null)
{
newSemantics = SemanticSwap(selectedCutPoint, selectedBranch, parent0, parent1, semantic0, semantic1);
AddStatistics(semantic0, parent0, statement == primaryCrossoverPoint.Child ? selectedBranch : statement, selectedCutPoint, json0, selectedBranch, random, problemData, variables, variableSettings); // parent zero has been changed is now considered the child
}
else
{
AddStatisticsNoCrossover(NoXoNoSelectedBranch);
}
return(parent0);
}
public static bool CreateSubroutine(
IRandom random,
ISymbolicExpressionTree symbolicExpressionTree,
int maxTreeLength, int maxTreeDepth,
int maxFunctionDefinitions, int maxFunctionArguments)
{
var functionDefiningBranches = symbolicExpressionTree.IterateNodesPrefix().OfType <DefunTreeNode>();
if (functionDefiningBranches.Count() >= maxFunctionDefinitions)
{
// allowed maximum number of ADF reached => abort
return(false);
}
if (symbolicExpressionTree.Length + 4 > maxTreeLength)
{
// defining a new function causes an length increase by 4 nodes (max) if the max tree length is reached => abort
return(false);
}
string formatString = new StringBuilder().Append('0', (int)Math.Log10(maxFunctionDefinitions * 10 - 1)).ToString(); // >= 100 functions => ###
var allowedFunctionNames = from index in Enumerable.Range(0, maxFunctionDefinitions)
select "ADF" + index.ToString(formatString);
// select a random body (either the result producing branch or an ADF branch)
var bodies = from node in symbolicExpressionTree.Root.Subtrees
select new { Tree = node, Length = node.GetLength() };
var totalNumberOfBodyNodes = bodies.Select(x => x.Length).Sum();
int r = random.Next(totalNumberOfBodyNodes);
int aggregatedNumberOfBodyNodes = 0;
ISymbolicExpressionTreeNode selectedBody = null;
foreach (var body in bodies)
{
aggregatedNumberOfBodyNodes += body.Length;
if (aggregatedNumberOfBodyNodes > r)
{
selectedBody = body.Tree;
}
}
// sanity check
if (selectedBody == null)
{
throw new InvalidOperationException();
}
// select a random cut point in the selected branch
var allCutPoints = (from parent in selectedBody.IterateNodesPrefix()
from subtree in parent.Subtrees
select new CutPoint(parent, subtree)).ToList();
if (!allCutPoints.Any())
{
// no cut points => abort
return(false);
}
string newFunctionName = allowedFunctionNames.Except(functionDefiningBranches.Select(x => x.FunctionName)).First();
var selectedCutPoint = allCutPoints.SampleRandom(random);
// select random branches as argument cut-off points (replaced by argument terminal nodes in the function)
List <CutPoint> argumentCutPoints = SelectRandomArgumentBranches(selectedCutPoint.Child, random, ARGUMENT_CUTOFF_PROBABILITY, maxFunctionArguments);
ISymbolicExpressionTreeNode functionBody = selectedCutPoint.Child;
// disconnect the function body from the tree
selectedCutPoint.Parent.RemoveSubtree(selectedCutPoint.ChildIndex);
// disconnect the argument branches from the function
functionBody = DisconnectBranches(functionBody, argumentCutPoints);
// insert a function invocation symbol instead
var invokeNode = (InvokeFunctionTreeNode)(new InvokeFunction(newFunctionName)).CreateTreeNode();
selectedCutPoint.Parent.InsertSubtree(selectedCutPoint.ChildIndex, invokeNode);
// add the branches selected as argument as subtrees of the function invocation node
foreach (var argumentCutPoint in argumentCutPoints)
{
invokeNode.AddSubtree(argumentCutPoint.Child);
}
// insert a new function defining branch
var defunNode = (DefunTreeNode)(new Defun()).CreateTreeNode();
defunNode.FunctionName = newFunctionName;
defunNode.AddSubtree(functionBody);
symbolicExpressionTree.Root.AddSubtree(defunNode);
// the grammar in the newly defined function is a clone of the grammar of the originating branch
defunNode.SetGrammar((ISymbolicExpressionTreeGrammar)selectedBody.Grammar.Clone());
var allowedChildSymbols = selectedBody.Grammar.GetAllowedChildSymbols(selectedBody.Symbol);
foreach (var allowedChildSymbol in allowedChildSymbols)
{
defunNode.Grammar.AddAllowedChildSymbol(defunNode.Symbol, allowedChildSymbol);
}
var maxSubtrees = selectedBody.Grammar.GetMaximumSubtreeCount(selectedBody.Symbol);
for (int i = 0; i < maxSubtrees; i++)
{
foreach (var allowedChildSymbol in selectedBody.Grammar.GetAllowedChildSymbols(selectedBody.Symbol, i))
{
defunNode.Grammar.AddAllowedChildSymbol(defunNode.Symbol, allowedChildSymbol);
}
}
// remove all argument symbols from grammar except that one contained in cutpoints
var oldArgumentSymbols = selectedBody.Grammar.Symbols.OfType <Argument>().ToList();
foreach (var oldArgSymb in oldArgumentSymbols)
{
defunNode.Grammar.RemoveSymbol(oldArgSymb);
}
// find unique argument indexes and matching symbols in the function defining branch
var newArgumentIndexes = (from node in defunNode.IterateNodesPrefix().OfType <ArgumentTreeNode>()
select node.Symbol.ArgumentIndex).Distinct();
// add argument symbols to grammar of function defining branch
GrammarModifier.AddArgumentSymbol(selectedBody.Grammar, defunNode.Grammar, newArgumentIndexes, argumentCutPoints);
defunNode.NumberOfArguments = newArgumentIndexes.Count();
if (defunNode.NumberOfArguments != argumentCutPoints.Count)
{
throw new InvalidOperationException();
}
// add invoke symbol for newly defined function to the original branch
GrammarModifier.AddInvokeSymbol(selectedBody.Grammar, defunNode.FunctionName, defunNode.NumberOfArguments, selectedCutPoint, argumentCutPoints);
// when the new function body was taken from another function definition
// add invoke symbol for newly defined function to all branches that are allowed to invoke the original branch
if (selectedBody.Symbol is Defun)
{
var originalFunctionDefinition = selectedBody as DefunTreeNode;
foreach (var subtree in symbolicExpressionTree.Root.Subtrees)
{
var originalBranchInvokeSymbol = (from symb in subtree.Grammar.Symbols.OfType <InvokeFunction>()
where symb.FunctionName == originalFunctionDefinition.FunctionName
select symb).SingleOrDefault();
// when the original branch can be invoked from the subtree then also allow invocation of the function
if (originalBranchInvokeSymbol != null)
{
GrammarModifier.AddInvokeSymbol(subtree.Grammar, defunNode.FunctionName, defunNode.NumberOfArguments, selectedCutPoint, argumentCutPoints);
}
}
}
return(true);
}
public Tuple<IEnumerable<bool>, IEnumerable<double>, double, string, List<PythonStatementSemantic>> EvaluateAndTraceProgram(string program, string input, string output, IEnumerable<int> indices, string header, ISymbolicExpressionTree tree, double timeout = 1) {
string traceProgram = traceCodeWithVariables
+ program;
traceProgram += traceCodeWithVariables == String.Empty
? String.Empty
: traceTableReduceEntries;
EvaluationScript es = pythonProcessParameter.ActualValue.CreateEvaluationScript(traceProgram, input, output, timeout);
es.Variables.Add("traceTable");
JObject json = pythonProcessParameter.ActualValue.SendAndEvaluateProgram(es);
var baseResult = pythonProcessParameter.ActualValue.GetVariablesFromJson(json, indices.Count());
if (json["traceTable"] == null) {
return new Tuple<IEnumerable<bool>, IEnumerable<double>, double, string, List<PythonStatementSemantic>>(baseResult.Item1, baseResult.Item2, baseResult.Item3, baseResult.Item4, new List<PythonStatementSemantic>());
}
var traceTable = JsonConvert.DeserializeObject<IDictionary<int, IDictionary<string, IList>>>(json["traceTable"].ToString());
IList<int> traceChanges = traceTable.Keys.OrderBy(x => x).ToList();
List<PythonStatementSemantic> semantics = new List<PythonStatementSemantic>();
ISymbolicExpressionTreeNode root = tree.Root;
var statementProductions = ((GroupSymbol)root.Grammar.GetSymbol("Rule: <code>")).Symbols;
var statementProductionNames = statementProductions.Select(x => x.Name);
IList<int> lineTraces = traceTable.Keys.OrderBy(x => x).ToList();
// add one, because the values are set after the statement is executed
// add two, for inval and outval
int curline = traceCode.Count(c => c == '\n') + header.Count(c => c == '\n') + 1 + 2;
var symbolToLineDict = FindStatementSymbolsInTree(root, statementProductionNames, ref curline);
var symbolLines = symbolToLineDict.Values.OrderBy(x => x).ToList();
var prefixTreeNodes = tree.IterateNodesPrefix().ToList();
foreach (var symbolLine in symbolToLineDict) {
Dictionary<string, IList> before = new Dictionary<string, IList>();
var linesBefore = lineTraces.Where(x => x <= symbolLine.Value).OrderByDescending(x => x);
foreach (var l in linesBefore) {
foreach (var change in traceTable[l]) {
if (!before.ContainsKey(change.Key)) {
before.Add(change.Key, change.Value);
}
}
}
int after = -1;
int pos = traceChanges.IndexOf(linesBefore.Max());
if (pos + 1 < traceChanges.Count // has to be in the array
// there cannot be another line which comes after the current one, but before the trace change
// otherwise the current line did not change anything
&& !symbolLines.Any(x => x > symbolLine.Value && x < traceChanges[pos + 1])) {
after = traceChanges[pos + 1];
}
if (after >= 0) {
semantics.Add(new PythonStatementSemantic() {
TreeNodePrefixPos = prefixTreeNodes.IndexOf(symbolLine.Key),
Before = before,
After = traceTable[after],
});
} else {
semantics.Add(new PythonStatementSemantic() {
TreeNodePrefixPos = prefixTreeNodes.IndexOf(symbolLine.Key),
Before = before,
After = new Dictionary<string, IList>(),
});
}
}
return new Tuple<IEnumerable<bool>, IEnumerable<double>, double, string, List<PythonStatementSemantic>>(baseResult.Item1, baseResult.Item2, baseResult.Item3, baseResult.Item4, semantics);
}
private void CalculateVariableMapping(ISymbolicExpressionTree tree, IDataset dataset) {
int columnIndex = 0;
int inputIndex = 0;
var usedVariables = tree.IterateNodesPrefix().OfType<VariableTreeNode>().Select(v => v.VariableName).Distinct();
foreach (var variable in dataset.VariableNames) {
columnIndex++;
if (!usedVariables.Contains(variable)) continue;
inputIndex++;
variableNameMapping[variable] = GetExcelColumnName(inputIndex);
}
}
public static bool DuplicateArgument(
IRandom random,
ISymbolicExpressionTree symbolicExpressionTree,
int maxFunctionDefinitions, int maxFunctionArguments)
{
var functionDefiningBranches = symbolicExpressionTree.IterateNodesPrefix().OfType <DefunTreeNode>().ToList();
var allowedArgumentIndexes = Enumerable.Range(0, maxFunctionArguments);
if (!functionDefiningBranches.Any())
{
// no function defining branches => abort
return(false);
}
var selectedDefunBranch = functionDefiningBranches.SampleRandom(random);
var argumentSymbols = selectedDefunBranch.Grammar.Symbols.OfType <Argument>().ToList();
if (!argumentSymbols.Any() || argumentSymbols.Count() >= maxFunctionArguments)
{
// when no argument or number of arguments is already at max allowed value => abort
return(false);
}
var selectedArgumentSymbol = argumentSymbols.SampleRandom(random);
var takenIndexes = argumentSymbols.Select(s => s.ArgumentIndex);
var newArgumentIndex = allowedArgumentIndexes.Except(takenIndexes).First();
var newArgSymbol = new Argument(newArgumentIndex);
// replace existing references to the original argument with references to the new argument randomly in the selectedBranch
var argumentNodes = selectedDefunBranch.IterateNodesPrefix().OfType <ArgumentTreeNode>();
foreach (var argNode in argumentNodes)
{
if (argNode.Symbol == selectedArgumentSymbol)
{
if (random.NextDouble() < 0.5)
{
argNode.Symbol = newArgSymbol;
}
}
}
// find invocations of the functions and duplicate the matching argument branch
var invocationNodes = (from node in symbolicExpressionTree.IterateNodesPrefix().OfType <InvokeFunctionTreeNode>()
where node.Symbol.FunctionName == selectedDefunBranch.FunctionName
where node.Subtrees.Count() == selectedDefunBranch.NumberOfArguments
select node).ToList();
// do this repeatedly until no matching invocations are found
while (invocationNodes.Count() > 0)
{
List <ISymbolicExpressionTreeNode> newlyAddedBranches = new List <ISymbolicExpressionTreeNode>();
foreach (var invokeNode in invocationNodes)
{
// check that the invocation node really has the correct number of arguments
if (invokeNode.Subtrees.Count() != selectedDefunBranch.NumberOfArguments)
{
throw new InvalidOperationException();
}
var argumentBranch = invokeNode.GetSubtree(selectedArgumentSymbol.ArgumentIndex);
var clonedArgumentBranch = (ISymbolicExpressionTreeNode)argumentBranch.Clone();
invokeNode.InsertSubtree(newArgumentIndex, clonedArgumentBranch);
newlyAddedBranches.Add(clonedArgumentBranch);
}
invocationNodes = (from newlyAddedBranch in newlyAddedBranches
from node in newlyAddedBranch.IterateNodesPrefix().OfType <InvokeFunctionTreeNode>()
where node.Symbol.FunctionName == selectedDefunBranch.FunctionName
where node.Subtrees.Count() == selectedDefunBranch.NumberOfArguments
select node).ToList();
}
// register the new argument symbol and increase the number of arguments of the ADF
selectedDefunBranch.Grammar.AddSymbol(newArgSymbol);
selectedDefunBranch.Grammar.SetSubtreeCount(newArgSymbol, 0, 0);
// allow the duplicated argument as child of all other arguments where the orginal argument was allowed
GrammarModifier.SetAllowedParentSymbols(selectedDefunBranch.Grammar, selectedArgumentSymbol, newArgSymbol);
selectedDefunBranch.NumberOfArguments++;
// increase the arity of the changed ADF in all branches that can use this ADF
foreach (var subtree in symbolicExpressionTree.Root.Subtrees)
{
var matchingInvokeSymbol = (from symb in subtree.Grammar.Symbols.OfType <InvokeFunction>()
where symb.FunctionName == selectedDefunBranch.FunctionName
select symb).SingleOrDefault();
if (matchingInvokeSymbol != null)
{
subtree.Grammar.SetSubtreeCount(matchingInvokeSymbol, selectedDefunBranch.NumberOfArguments, selectedDefunBranch.NumberOfArguments);
foreach (var symb in subtree.Grammar.Symbols)
{
if (symb is StartSymbol || symb is ProgramRootSymbol)
{
continue;
}
if (subtree.Grammar.IsAllowedChildSymbol(matchingInvokeSymbol, symb, selectedArgumentSymbol.ArgumentIndex))
{
subtree.Grammar.AddAllowedChildSymbol(matchingInvokeSymbol, symb, newArgumentIndex);
}
}
}
}
return(true);
}
protected override ISymbolicExpressionTree Cross(IRandom random, ISymbolicExpressionTree parent0, ISymbolicExpressionTree parent1, ItemArray <PythonStatementSemantic> semantic0, ItemArray <PythonStatementSemantic> semantic1, T problemData, int maxTreeLength, int maxTreeDepth, double internalCrossoverPointProbability, out ItemArray <PythonStatementSemantic> newSemantics)
{
if (semantic0 == null || semantic1 == null || semantic0.Length == 0 || semantic1.Length == 0)
{
parent0 = SubtreeCrossover.Cross(random, parent0, parent1, internalCrossoverPointProbability, maxTreeLength, maxTreeDepth);
newSemantics = null;
TypeSelectedForSimilarityParameter.ActualValue = new StringValue("No Semantics; Random Crossover");
AddStatisticsNoCrossover(NoXoNoSemantics);
return(parent0);
}
newSemantics = semantic0;
var statementProductionNames = SemanticOperatorHelper.GetSemanticProductionNames(parent0.Root.Grammar);
var variables = problemData.Variables.GetVariableNames().ToList();
string variableSettings = problemData.VariableSettings.Count == 0 ? String.Empty : String.Join(Environment.NewLine, problemData.VariableSettings.Select(x => x.Value));
int maximumSemanticTries = MaxComparesParameter.Value.Value;
int semanticTries = 0;
var saveOriginalSemantics = new List <JObject>(maximumSemanticTries);
var saveReplaceSemantics = new List <JObject>(maximumSemanticTries);
var possibleChildren = new List <Tuple <CutPoint, ISymbolicExpressionTreeNode> >(maximumSemanticTries);
bool success = false;
do
{
// select a random crossover point in the first parent
CutPoint crossoverPoint0;
SelectCrossoverPoint(random, parent0, internalCrossoverPointProbability, maxTreeLength, maxTreeDepth, out crossoverPoint0);
int childLength = crossoverPoint0.Child != null?crossoverPoint0.Child.GetLength() : 0;
// calculate the max length and depth that the inserted branch can have
int maxInsertedBranchLength = Math.Max(0, maxTreeLength - (parent0.Length - childLength));
int maxInsertedBranchDepth = Math.Max(0, maxTreeDepth - parent0.Root.GetBranchLevel(crossoverPoint0.Parent));
List <ISymbolicExpressionTreeNode> allowedBranches = new List <ISymbolicExpressionTreeNode>();
parent1.Root.ForEachNodePostfix((n) => {
if (n.GetLength() <= maxInsertedBranchLength &&
n.GetDepth() <= maxInsertedBranchDepth && crossoverPoint0.IsMatchingPointType(n))
{
allowedBranches.Add(n);
}
});
// empty branch
if (crossoverPoint0.IsMatchingPointType(null))
{
allowedBranches.Add(null);
}
if (allowedBranches.Count != 0)
{
var selectedBranch = SelectRandomBranch(random, allowedBranches, internalCrossoverPointProbability);
ISymbolicExpressionTreeNode statement = SemanticOperatorHelper.GetStatementNode(crossoverPoint0.Child, statementProductionNames);
var statementPos0 = parent0.IterateNodesPrefix().ToList().IndexOf(statement);
PythonStatementSemantic curSemantics = null;
if (String.IsNullOrEmpty(variableSettings))
{
curSemantics = semantic0.First(x => x.TreeNodePrefixPos == statementPos0);
variableSettings = SemanticOperatorHelper.SemanticToPythonVariableSettings(curSemantics.Before, problemData.Variables.GetVariableTypes());
}
var jsonOriginal = SemanticOperatorHelper.EvaluateStatementNode(statement, PyProcess, random, problemData, variables, variableSettings, Timeout);
JObject jsonReplaced;
if (statement == crossoverPoint0.Child)
{
// selectedBranch is also executable
jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(selectedBranch, PyProcess, random, problemData, variables, variableSettings, Timeout);
}
else
{
crossoverPoint0.Parent.RemoveSubtree(crossoverPoint0.ChildIndex);
var parent = selectedBranch.Parent; // save parent
crossoverPoint0.Parent.InsertSubtree(crossoverPoint0.ChildIndex, selectedBranch); // this will affect node.Parent
jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(statement, PyProcess, random, problemData, variables, variableSettings, Timeout);
crossoverPoint0.Parent.RemoveSubtree(crossoverPoint0.ChildIndex); // removes intermediate parent from node
selectedBranch.Parent = parent; // restore parent
crossoverPoint0.Parent.InsertSubtree(crossoverPoint0.ChildIndex, crossoverPoint0.Child); // restore cutPoint
}
var exception = jsonOriginal["exception"] != null || jsonReplaced["exception"] != null;
if (exception)
{
if (jsonOriginal["exception"] != null)
{
CrossoverExceptionsParameter.ActualValue.Add(new StringValue(jsonOriginal["exception"].ToString()));
}
if (jsonReplaced["exception"] != null)
{
CrossoverExceptionsParameter.ActualValue.Add(new StringValue(jsonReplaced["exception"].ToString()));
}
}
if (curSemantics != null && !exception)
{
jsonOriginal = PythonSemanticComparer.ReplaceNotExecutedCases(jsonOriginal, curSemantics.Before, curSemantics.ExecutedCases);
jsonReplaced = PythonSemanticComparer.ReplaceNotExecutedCases(jsonReplaced, curSemantics.Before, curSemantics.ExecutedCases);
jsonOriginal = PythonSemanticComparer.ProduceDifference(jsonOriginal, curSemantics.Before);
jsonReplaced = PythonSemanticComparer.ProduceDifference(jsonReplaced, curSemantics.Before);
}
if (!exception && SemanticMeasure(jsonOriginal, jsonReplaced))
{
newSemantics = SemanticSwap(crossoverPoint0, selectedBranch, parent0, parent1, semantic0, semantic1);
SemanticallyEquivalentCrossoverParameter.ActualValue = new IntValue(2);
TypeSelectedForSimilarityParameter.ActualValue = new StringValue("First Semantic");
AddStatistics(semantic0, parent0, statement == crossoverPoint0.Child ? selectedBranch : statement, crossoverPoint0, jsonOriginal, selectedBranch, random, problemData, variables, variableSettings); // parent zero has been changed is now considered the child
success = true;
}
else
{
saveOriginalSemantics.Add(jsonOriginal);
saveReplaceSemantics.Add(jsonReplaced);
possibleChildren.Add(new Tuple <CutPoint, ISymbolicExpressionTreeNode>(crossoverPoint0, selectedBranch));
}
}
semanticTries++;
#region try second semantic comparison
if (!success && semanticTries >= maximumSemanticTries)
{
for (int index = 0; index < saveOriginalSemantics.Count; index++)
{
var exception = saveOriginalSemantics[index]["exception"] == null && saveReplaceSemantics[index]["exception"] == null;
if (!exception && AdditionalSemanticMeasure(saveOriginalSemantics[index], saveReplaceSemantics[index]))
{
var crossover = possibleChildren[index];
crossoverPoint0 = crossover.Item1;
// Recreate jsonOriginal as it might have changed due to ReplaceNotExecutedCases and ProduceDifference
var jsonOriginal = GenerateOriginalJson(crossoverPoint0, statementProductionNames, parent0, variableSettings, semantic0, problemData, random, variables);
newSemantics = SemanticSwap(crossoverPoint0, crossover.Item2, parent0, parent1, semantic0, semantic1);
var statement = SemanticOperatorHelper.GetStatementNode(crossover.Item2, statementProductionNames);
SemanticallyEquivalentCrossoverParameter.ActualValue = new IntValue(2);
TypeSelectedForSimilarityParameter.ActualValue = new StringValue("Second Semantic");
AddStatistics(semantic0, parent0, statement, crossoverPoint0, jsonOriginal, crossover.Item2, random, problemData, variables, variableSettings); // parent zero has been changed is now considered the child
success = true;
break;
}
}
}
#endregion
} while (!success && semanticTries < maximumSemanticTries);
if (!success)
{
// Last change. If any possible crossover was found, do a crossover with the first one
if (saveOriginalSemantics.Any())
{
var crossover = possibleChildren.First();
var crossoverPoint0 = crossover.Item1;
// Recreate jsonOriginal as it might have changed due to ReplaceNotExecutedCases and ProduceDifference
var jsonOriginal = GenerateOriginalJson(crossoverPoint0, statementProductionNames, parent0, variableSettings, semantic0, problemData, random, variables);
newSemantics = SemanticSwap(crossoverPoint0, crossover.Item2, parent0, parent1, semantic0, semantic1);
var statement = SemanticOperatorHelper.GetStatementNode(crossover.Item2, statementProductionNames);
TypeSelectedForSimilarityParameter.ActualValue = new StringValue("Random Crossover; Reached Max Semantic Tries");
AddStatistics(semantic0, parent0, statement, crossoverPoint0, jsonOriginal, crossover.Item2, random, problemData, variables, variableSettings); // parent zero has been changed is now considered the child
}
else
{
AddStatisticsNoCrossover(NoXoNoAllowedBranch);
}
}
saveOriginalSemantics.Clear();
saveReplaceSemantics.Clear();
possibleChildren.Clear();
NumberOfCrossoverTries = semanticTries;
return(parent0);
}
public override void ReplaceBranch(IRandom random, ISymbolicExpressionTree symbolicExpressionTree, ICFGPythonProblemData problemData, ItemArray <PythonStatementSemantic> semantics, PythonProcess pythonProcess, double timeout, int maxTreeLength, int maxTreeDepth, int maximumSemanticTries)
{
if (semantics == null || semantics.Length == 0)
{
ReplaceBranchManipulation.ReplaceRandomBranch(random, symbolicExpressionTree, maxTreeLength, maxTreeDepth);
SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(NoSemantics);
MutationTypeParameter.ActualValue = new IntValue(RandomMutation);
return;
}
var allowedSymbols = new List <ISymbol>();
ISymbolicExpressionTreeNode parent;
int childIndex;
int maxLength;
int maxDepth;
// repeat until a fitting parent and child are found (MAX_TRIES times)
int tries = 0;
ISymbolicExpressionTreeNode child;
do
{
#pragma warning disable 612, 618
parent = symbolicExpressionTree.Root.IterateNodesPrefix().Skip(1).Where(n => n.SubtreeCount > 0).SelectRandom(random);
#pragma warning restore 612, 618
childIndex = random.Next(parent.SubtreeCount);
child = parent.GetSubtree(childIndex);
maxLength = maxTreeLength - symbolicExpressionTree.Length + child.GetLength();
maxDepth = maxTreeDepth - symbolicExpressionTree.Root.GetBranchLevel(child);
allowedSymbols.Clear();
foreach (var symbol in parent.Grammar.GetAllowedChildSymbols(parent.Symbol, childIndex))
{
// check basic properties that the new symbol must have
if ((symbol.Name != child.Symbol.Name || symbol.MinimumArity > 0) &&
symbol.InitialFrequency > 0 &&
parent.Grammar.GetMinimumExpressionDepth(symbol) <= maxDepth &&
parent.Grammar.GetMinimumExpressionLength(symbol) <= maxLength)
{
allowedSymbols.Add(symbol);
}
}
tries++;
} while (tries < MAX_TRIES && allowedSymbols.Count == 0);
NumberOfTriesParameter.ActualValue = new IntValue(tries); //normal tries. not semantic tries
if (tries < MAX_TRIES)
{
var statementProductionNames = SemanticOperatorHelper.GetSemanticProductionNames(symbolicExpressionTree.Root.Grammar);
var variables = problemData.Variables.GetVariableNames().ToList();
#region calculate original json output
ISymbolicExpressionTreeNode statement = SemanticOperatorHelper.GetStatementNode(child, statementProductionNames);
string variableSettings = problemData.VariableSettings.Count == 0 ? String.Empty : String.Join(Environment.NewLine, problemData.VariableSettings.Select(x => x.Value));
var statementPos0 = symbolicExpressionTree.IterateNodesPrefix().ToList().IndexOf(statement);
if (String.IsNullOrEmpty(variableSettings))
{
variableSettings = SemanticOperatorHelper.SemanticToPythonVariableSettings(semantics.First(x => x.TreeNodePrefixPos == statementPos0).Before, problemData.Variables.GetVariableTypes());
}
var jsonOriginal = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout);
#endregion
var weights = allowedSymbols.Select(s => s.InitialFrequency).ToList();
#pragma warning disable 612, 618
var seedSymbol = allowedSymbols.SelectRandom(weights, random);
#pragma warning restore 612, 618
// replace the old node with the new node
var seedNode = seedSymbol.CreateTreeNode();
if (seedNode.HasLocalParameters)
{
seedNode.ResetLocalParameters(random);
}
parent.RemoveSubtree(childIndex);
parent.InsertSubtree(childIndex, seedNode);
ProbabilisticTreeCreator.PTC2(random, seedNode, maxLength, maxDepth);
JObject jsonReplaced;
if (child == statement)
{
// child is executable, so is the new child
jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(seedNode, pythonProcess, random, problemData, variables, variableSettings, timeout);
}
else
{
jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout);
}
if (JToken.EqualityComparer.Equals(jsonOriginal, jsonReplaced))
{
SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(Equvivalent);
}
else
{
SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(Different);
}
MutationTypeParameter.ActualValue = new IntValue(RandomMutation);
}
else
{
SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(NoMutation);
MutationTypeParameter.ActualValue = new IntValue(NoMutation);
}
}
private void ComboBoxSelectedIndexChanged(object sender, EventArgs e) {
string freeVariable = (string)comboBox.SelectedItem;
IEnumerable<string> fixedVariables = comboBox.Items.OfType<string>()
.Except(new string[] { freeVariable });
variableNodes.Clear();
clonedTree = (ISymbolicExpressionTree)Content.Model.SymbolicExpressionTree.Clone();
foreach (var varNode in clonedTree.IterateNodesPrefix().OfType<VariableTreeNode>()) {
if (fixedVariables.Contains(varNode.VariableName)) {
if (!variableNodes.ContainsKey(varNode.VariableName))
variableNodes.Add(varNode.VariableName, new List<ISymbolicExpressionTreeNode>());
int childIndex = varNode.Parent.IndexOfSubtree(varNode);
var replacementNode = MakeConstantTreeNode(medianValues[varNode.VariableName]);
var parent = varNode.Parent;
parent.RemoveSubtree(childIndex);
parent.InsertSubtree(childIndex, MakeProduct(replacementNode, varNode.Weight));
variableNodes[varNode.VariableName].Add(replacementNode);
}
}
CreateSliders(fixedVariables);
UpdateScatterPlot();
UpdateResponseSeries();
}
public static bool DuplicateSubroutine(
IRandom random,
ISymbolicExpressionTree symbolicExpressionTree,
int maxFunctionDefinitions, int maxFunctionArguments) {
var functionDefiningBranches = symbolicExpressionTree.IterateNodesPrefix().OfType<DefunTreeNode>().ToList();
if (!functionDefiningBranches.Any() || functionDefiningBranches.Count() == maxFunctionDefinitions)
// no function defining branches to duplicate or already reached the max number of ADFs
return false;
string formatString = new StringBuilder().Append('0', (int)Math.Log10(maxFunctionDefinitions) + 1).ToString(); // >= 100 functions => ###
var allowedFunctionNames = from index in Enumerable.Range(0, maxFunctionDefinitions)
select "ADF" + index.ToString(formatString);
var selectedBranch = functionDefiningBranches.SampleRandom(random);
var duplicatedDefunBranch = (DefunTreeNode)selectedBranch.Clone();
string newFunctionName = allowedFunctionNames.Except(UsedFunctionNames(symbolicExpressionTree)).First();
duplicatedDefunBranch.FunctionName = newFunctionName;
symbolicExpressionTree.Root.AddSubtree(duplicatedDefunBranch);
duplicatedDefunBranch.SetGrammar((ISymbolicExpressionTreeGrammar)selectedBranch.Grammar.Clone());
var allowedChildSymbols = selectedBranch.Grammar.GetAllowedChildSymbols(selectedBranch.Symbol);
foreach (var allowedChildSymbol in allowedChildSymbols)
duplicatedDefunBranch.Grammar.AddAllowedChildSymbol(duplicatedDefunBranch.Symbol, allowedChildSymbol);
var maxSubtrees = selectedBranch.Grammar.GetMaximumSubtreeCount(selectedBranch.Symbol);
for (int i = 0; i < maxSubtrees; i++) {
foreach (var allowedChildSymbol in selectedBranch.Grammar.GetAllowedChildSymbols(selectedBranch.Symbol, i))
duplicatedDefunBranch.Grammar.AddAllowedChildSymbol(duplicatedDefunBranch.Symbol, allowedChildSymbol);
}
// add an invoke symbol for each branch that is allowed to invoke the original function
foreach (var subtree in symbolicExpressionTree.Root.Subtrees.OfType<SymbolicExpressionTreeTopLevelNode>()) {
var matchingInvokeSymbol = (from symb in subtree.Grammar.Symbols.OfType<InvokeFunction>()
where symb.FunctionName == selectedBranch.FunctionName
select symb).SingleOrDefault();
if (matchingInvokeSymbol != null) {
var invokeSymbol = new InvokeFunction(duplicatedDefunBranch.FunctionName);
subtree.Grammar.AddSymbol(invokeSymbol);
subtree.Grammar.SetSubtreeCount(invokeSymbol, duplicatedDefunBranch.NumberOfArguments, duplicatedDefunBranch.NumberOfArguments);
foreach (ISymbol symbol in subtree.Grammar.Symbols) {
if (subtree.Grammar.IsAllowedChildSymbol(symbol, matchingInvokeSymbol))
subtree.Grammar.AddAllowedChildSymbol(symbol, invokeSymbol);
else {
for (int i = 0; i < subtree.Grammar.GetMaximumSubtreeCount(symbol); i++)
if (subtree.Grammar.IsAllowedChildSymbol(symbol, matchingInvokeSymbol, i))
subtree.Grammar.AddAllowedChildSymbol(symbol, invokeSymbol, i);
}
}
foreach (ISymbol symbol in subtree.Grammar.GetAllowedChildSymbols(matchingInvokeSymbol))
if (symbol != invokeSymbol) //avoid duplicate entry invokesymbol / invokesymbol
subtree.Grammar.AddAllowedChildSymbol(invokeSymbol, symbol);
for (int i = 0; i < subtree.Grammar.GetMaximumSubtreeCount(matchingInvokeSymbol); i++) {
foreach (ISymbol symbol in subtree.Grammar.GetAllowedChildSymbols(matchingInvokeSymbol, i).Except(subtree.Grammar.GetAllowedChildSymbols(matchingInvokeSymbol)))
subtree.Grammar.AddAllowedChildSymbol(invokeSymbol, symbol, i);
}
}
// in the current subtree:
// for all invoke nodes of the original function replace the invoke of the original function with an invoke of the new function randomly
var originalFunctionInvocations = from node in subtree.IterateNodesPrefix().OfType<InvokeFunctionTreeNode>()
where node.Symbol.FunctionName == selectedBranch.FunctionName
select node;
foreach (var originalFunctionInvokeNode in originalFunctionInvocations) {
var newInvokeSymbol = (from symb in subtree.Grammar.Symbols.OfType<InvokeFunction>()
where symb.FunctionName == duplicatedDefunBranch.FunctionName
select symb).Single();
// flip coin wether to replace with newly defined function
if (random.NextDouble() < 0.5) {
originalFunctionInvokeNode.Symbol = newInvokeSymbol;
}
}
}
return true;
}
/// <summary>
/// Swaps the child node of the cutpoint with the selected branch. Semantics of the child branch will be removed from semantics0. Semantics from semantics1 which belonged to the selected branch will be added to semantics0.
/// </summary>
/// <param name="crossoverPoint">Defines parent and child node from the parent0</param>
/// <param name="selectedBranch">Branch to crossover from parent1</param>
/// <param name="parent0">Parent0</param>
/// <param name="parent1">Parent1</param>
/// <param name="semantics0">Semantics of parent0</param>
/// <param name="semantics1">Semantics of parent1</param>
protected static ItemArray <PythonStatementSemantic> SemanticSwap(CutPoint crossoverPoint, ISymbolicExpressionTreeNode selectedBranch, ISymbolicExpressionTree parent0, ISymbolicExpressionTree parent1, ItemArray <PythonStatementSemantic> semantics0, ItemArray <PythonStatementSemantic> semantics1)
{
var allNodes0Prefix = parent0.IterateNodesPrefix().ToList();
Dictionary <ISymbolicExpressionTreeNode, PythonStatementSemantic> sem0ByNode = new Dictionary <ISymbolicExpressionTreeNode, PythonStatementSemantic>();
var sem0ByNodeIndex = semantics0.ToDictionary(x => x.TreeNodePrefixPos, y => y);
for (int i = 0; i < allNodes0Prefix.Count; i++)
{
if (sem0ByNodeIndex.ContainsKey(i))
{
sem0ByNode.Add(allNodes0Prefix[i], sem0ByNodeIndex[i]);
}
}
var allNodes1Prefix = parent1.IterateNodesPrefix().ToList();
Dictionary <ISymbolicExpressionTreeNode, PythonStatementSemantic> sem1ByNode = new Dictionary <ISymbolicExpressionTreeNode, PythonStatementSemantic>();
var sem1ByNodeIndex = semantics1.ToDictionary(x => x.TreeNodePrefixPos, y => y);
for (int i = 0; i < allNodes1Prefix.Count; i++)
{
if (sem1ByNodeIndex.ContainsKey(i))
{
sem1ByNode.Add(allNodes1Prefix[i], sem1ByNodeIndex[i]);
}
}
if (crossoverPoint.Child != null)
{
// manipulate the tree of parent0 in place
// replace the branch in tree0 with the selected branch from tree1
crossoverPoint.Parent.RemoveSubtree(crossoverPoint.ChildIndex);
if (selectedBranch != null)
{
crossoverPoint.Parent.InsertSubtree(crossoverPoint.ChildIndex, selectedBranch);
}
}
else
{
// child is null (additional child should be added under the parent)
if (selectedBranch != null)
{
crossoverPoint.Parent.AddSubtree(selectedBranch);
}
}
List <PythonStatementSemantic> newSemantics = new List <PythonStatementSemantic>();
var newNodes0Prefix = parent0.IterateNodesPrefix().ToList();
for (int i = 0; i < newNodes0Prefix.Count; i++)
{
PythonStatementSemantic sem = null;
if (sem0ByNode.ContainsKey(newNodes0Prefix[i]))
{
sem = sem0ByNode[newNodes0Prefix[i]];
sem.TreeNodePrefixPos = i;
}
else if (sem1ByNode.ContainsKey(newNodes0Prefix[i]))
{
sem = sem1ByNode[newNodes0Prefix[i]];
sem.TreeNodePrefixPos = i;
}
if (sem != null)
{
newSemantics.Add(sem);
}
}
return(new ItemArray <PythonStatementSemantic>(newSemantics));
}
private static void DeletionByRandomRegeneration(IRandom random, ISymbolicExpressionTree symbolicExpressionTree, DefunTreeNode selectedDefunBranch) {
// find first invocation and replace it with a randomly generated tree
// can't find all invocations in one step because once we replaced a top level invocation
// the invocations below it are removed already
var invocationCutPoint = (from node in symbolicExpressionTree.IterateNodesPrefix()
from subtree in node.Subtrees.OfType<InvokeFunctionTreeNode>()
where subtree.Symbol.FunctionName == selectedDefunBranch.FunctionName
select new CutPoint(node, subtree)).FirstOrDefault();
while (invocationCutPoint != null) {
// deletion by random regeneration
ISymbolicExpressionTreeNode replacementTree = null;
var allowedSymbolsList = invocationCutPoint.Parent.Grammar.GetAllowedChildSymbols(invocationCutPoint.Parent.Symbol, invocationCutPoint.ChildIndex).ToList();
var weights = allowedSymbolsList.Select(s => s.InitialFrequency);
#pragma warning disable 612, 618
var selectedSymbol = allowedSymbolsList.SelectRandom(weights, random);
#pragma warning restore 612, 618
int minPossibleLength = invocationCutPoint.Parent.Grammar.GetMinimumExpressionLength(selectedSymbol);
int maxLength = Math.Max(minPossibleLength, invocationCutPoint.Child.GetLength());
int minPossibleDepth = invocationCutPoint.Parent.Grammar.GetMinimumExpressionDepth(selectedSymbol);
int maxDepth = Math.Max(minPossibleDepth, invocationCutPoint.Child.GetDepth());
replacementTree = selectedSymbol.CreateTreeNode();
if (replacementTree.HasLocalParameters)
replacementTree.ResetLocalParameters(random);
invocationCutPoint.Parent.RemoveSubtree(invocationCutPoint.ChildIndex);
invocationCutPoint.Parent.InsertSubtree(invocationCutPoint.ChildIndex, replacementTree);
ProbabilisticTreeCreator.PTC2(random, replacementTree, maxLength, maxDepth);
invocationCutPoint = (from node in symbolicExpressionTree.IterateNodesPrefix()
from subtree in node.Subtrees.OfType<InvokeFunctionTreeNode>()
where subtree.Symbol.FunctionName == selectedDefunBranch.FunctionName
select new CutPoint(node, subtree)).FirstOrDefault();
}
}
// ToDo: Remove workaround; use executed lines in PythonStatementSemantic
public Tuple <IEnumerable <bool>, IEnumerable <double>, double, string, List <PythonStatementSemantic> > EvaluateAndTraceProgram(PythonProcess pythonProcess, string program, string input, string output, IEnumerable <int> indices, string header, string footer, ISymbolicExpressionTree tree, double timeout = 1)
{
string traceProgram = traceCodeWithVariables
+ program;
traceProgram += traceCodeWithVariables == String.Empty
? String.Empty
: traceTableReduceEntries;
EvaluationScript es = pythonProcess.CreateEvaluationScript(traceProgram, input, output, timeout);
es.Variables.Add("traceTable");
es.Variables.Add("traceTableBefore");
es.Variables.Add("executedLines");
JObject json = pythonProcess.SendAndEvaluateProgram(es);
var baseResult = pythonProcess.GetVariablesFromJson(json, indices.Count());
if (json["traceTable"] == null)
{
return(new Tuple <IEnumerable <bool>, IEnumerable <double>, double, string, List <PythonStatementSemantic> >(baseResult.Item1, baseResult.Item2, baseResult.Item3, baseResult.Item4, new List <PythonStatementSemantic>()));
}
var traceTable = json["traceTable"].ToObject <IDictionary <int, IDictionary <string, IList> > >();
var traceTableBefore = json["traceTableBefore"].ToObject <IDictionary <int, IDictionary <string, IList> > >();
var executedLines = json["executedLines"].ToObject <IDictionary <int, List <int> > >();
List <PythonStatementSemantic> semantics = new List <PythonStatementSemantic>();
ISymbolicExpressionTreeNode root = tree.Root;
var statementProductionNames = SemanticOperatorHelper.GetSemanticProductionNames(root.Grammar);
// calculate the correct line the semantic evaluation starts from
var code = CFGSymbolicExpressionTreeStringFormatter.StaticFormat(tree);
int curline = es.Script.Count(c => c == '\n') - code.Count(c => c == '\n') - footer.Count(c => c == '\n') - traceTableReduceEntries.Count(c => c == '\n');
var symbolToLineDict = FindStatementSymbolsInTree(root, statementProductionNames, ref curline);
#region workaround: empty line problem with while, can't fix, otherwise FindStatementSymbolsInTree won't work
string[] sciptLines = es.Script.Split('\n');
var beginLineNumbers = symbolToLineDict.Values.Select(x => x[0]).Distinct().ToList();
var moveLines = new Dictionary <int, int>(beginLineNumbers.Count);
foreach (var l in beginLineNumbers)
{
// decrease by one, as sciptLines is an array and start from zero, where lineNumbers started counting from 1
if (String.IsNullOrWhiteSpace(sciptLines[l - 1]) || sciptLines[l - 1].TrimStart().StartsWith("#"))
{
// empty line or comment
var i = l + 1;
while (i - 1 < sciptLines.Length && (String.IsNullOrWhiteSpace(sciptLines[i - 1]) || sciptLines[i - 1].TrimStart().StartsWith("#")))
{
i++;
}
moveLines.Add(l, i);
}
else
{
moveLines.Add(l, l);
}
}
foreach (var symbolLine in symbolToLineDict)
{
symbolLine.Value[0] = moveLines[symbolLine.Value[0]];
}
#endregion
#region fix Before line for <predefined> to have all variables initialised
// not a great way to do it, but the python interpreter does not stop at e.g. 'while False:'
int newMinBefore = traceTableBefore.OrderBy(x => x.Key).First(x => x.Value.ContainsKey("res0") && !x.Value["res0"].Contains(null)).Key; // first line that changes res0
foreach (var symbolToLine in symbolToLineDict)
{
symbolToLine.Value.Add(symbolToLine.Value[0]); // original beginning at [2], which is needed for executed lines
if (symbolToLine.Value[0] < newMinBefore)
{
symbolToLine.Value[0] = newMinBefore;
}
}
#endregion
var prefixTreeNodes = tree.IterateNodesPrefix().ToList();
foreach (var symbolLine in symbolToLineDict)
{
// Before
Dictionary <string, IList> before = new Dictionary <string, IList>();
foreach (var traceChange in traceTableBefore.Where(x => x.Key <= symbolLine.Value[0]).OrderByDescending(x => x.Key))
{
foreach (var variableChange in traceChange.Value)
{
if (!before.ContainsKey(variableChange.Key))
{
before.Add(variableChange.Key, variableChange.Value);
}
}
}
// After
Dictionary <string, IList> after = new Dictionary <string, IList>();
foreach (var traceChange in traceTable.Where(x => x.Key >= symbolLine.Value[0] && x.Key <= symbolLine.Value[1]).OrderByDescending(x => x.Key))
{
foreach (var variableChange in traceChange.Value)
{
if (!after.ContainsKey(variableChange.Key))
{
after.Add(variableChange.Key, variableChange.Value);
}
}
}
// clean after with before
foreach (var key in after.Keys.ToList())
{
if (PythonSemanticComparer.CompareSequence(after[key], before[key]))
{
after.Remove(key);
}
}
// add semantics
var executedLinesWithinStatement = executedLines.Where(x => x.Key >= symbolLine.Value[2] && x.Key <= symbolLine.Value[1]);
semantics.Add(new PythonStatementSemantic()
{
TreeNodePrefixPos = prefixTreeNodes.IndexOf(symbolLine.Key),
ExecutedCases = executedLinesWithinStatement.Any() ? executedLinesWithinStatement.OrderByDescending(x => x.Value.Count).First().Value : new List <int>(),
Before = before,
After = after,
});
}
return(new Tuple <IEnumerable <bool>, IEnumerable <double>, double, string, List <PythonStatementSemantic> >(baseResult.Item1, baseResult.Item2, baseResult.Item3, baseResult.Item4, semantics));
}
protected virtual ISymbolicExpressionTree Cross(IRandom random, ISymbolicExpressionTree parent0, ISymbolicExpressionTree parent1, ItemArray <PythonStatementSemantic> semantic0, ItemArray <PythonStatementSemantic> semantic1, ICFGPythonProblemData problemData, int maxTreeLength, int maxTreeDepth, double internalCrossoverPointProbability, out ItemArray <PythonStatementSemantic> newSemantics)
{
if (semantic0 == null || semantic1 == null || semantic0.Length == 0 || semantic1.Length == 0)
{
parent0 = SubtreeCrossover.Cross(random, parent0, parent1, internalCrossoverPointProbability, maxTreeLength, maxTreeDepth);
newSemantics = null;
return(parent0);
}
newSemantics = semantic0;
// select a random crossover point in the first parent
CutPoint crossoverPoint0;
SelectCrossoverPoint(random, parent0, internalCrossoverPointProbability, maxTreeLength, maxTreeDepth, out crossoverPoint0);
int childLength = crossoverPoint0.Child != null?crossoverPoint0.Child.GetLength() : 0;
// calculate the max length and depth that the inserted branch can have
int maxInsertedBranchLength = maxTreeLength - (parent0.Length - childLength);
int maxInsertedBranchDepth = maxTreeDepth - parent0.Root.GetBranchLevel(crossoverPoint0.Child);
List <ISymbolicExpressionTreeNode> allowedBranches = new List <ISymbolicExpressionTreeNode>();
parent1.Root.ForEachNodePostfix((n) => {
if (n.GetLength() <= maxInsertedBranchLength &&
n.GetDepth() <= maxInsertedBranchDepth && crossoverPoint0.IsMatchingPointType(n))
{
allowedBranches.Add(n);
}
});
// empty branch
if (crossoverPoint0.IsMatchingPointType(null))
{
allowedBranches.Add(null);
}
if (allowedBranches.Count == 0)
{
return(parent0);
}
// select MaxCompares random crossover points
// Use set to avoid having the same node multiple times
HashSet <ISymbolicExpressionTreeNode> compBranches;
if (allowedBranches.Count < MaxComparesParameter.Value.Value)
{
compBranches = new HashSet <ISymbolicExpressionTreeNode>(allowedBranches);
}
else
{
compBranches = new HashSet <ISymbolicExpressionTreeNode>();
for (int i = 0; i < MaxComparesParameter.Value.Value; i++)
{
var possibleBranch = SelectRandomBranch(random, allowedBranches, internalCrossoverPointProbability);
allowedBranches.Remove(possibleBranch);
compBranches.Add(possibleBranch);
}
}
var statementProductionNames = SemanticOperatorHelper.GetSemanticProductionNames(crossoverPoint0.Parent.Grammar);
// find first node that can be used for evaluation in parent0
ISymbolicExpressionTreeNode statement = SemanticOperatorHelper.GetStatementNode(crossoverPoint0.Child, statementProductionNames);
if (statement == null)
{
newSemantics = SemanticSwap(crossoverPoint0, compBranches.SampleRandom(random), parent0, parent1, semantic0, semantic1);
return(parent0);
}
var statementPos0 = parent0.IterateNodesPrefix().ToList().IndexOf(statement);
string variableSettings;
if (problemData.VariableSettings.Count == 0)
{
variableSettings = SemanticOperatorHelper.SemanticToPythonVariableSettings(semantic0.First(x => x.TreeNodePrefixPos == statementPos0).Before, problemData.Variables.GetVariableTypes());
}
else
{
variableSettings = String.Join(Environment.NewLine, problemData.VariableSettings.Select(x => x.Value));
}
var variables = problemData.Variables.GetVariableNames().ToList();
var json0 = SemanticOperatorHelper.EvaluateStatementNode(statement, PyProcess, random, problemData, variables, variableSettings, Timeout);
ISymbolicExpressionTreeNode selectedBranch;
if (!String.IsNullOrWhiteSpace((string)json0["exception"]))
{
selectedBranch = compBranches.SampleRandom(random);
}
else
{
selectedBranch = SelectBranch(statement, crossoverPoint0, compBranches, random, variables, variableSettings, json0, problemData);
}
// perform the actual swap
newSemantics = SemanticSwap(crossoverPoint0, selectedBranch, parent0, parent1, semantic0, semantic1);
return(parent0);
}
private static IEnumerable <string> UsedFunctionNames(ISymbolicExpressionTree symbolicExpressionTree)
{
return(from node in symbolicExpressionTree.IterateNodesPrefix()
where node.Symbol is Defun
select((DefunTreeNode)node).FunctionName);
}
public override void ReplaceBranch(IRandom random, ISymbolicExpressionTree symbolicExpressionTree, ICFGPythonProblemData problemData, ItemArray <PythonStatementSemantic> semantics, PythonProcess pythonProcess, double timeout, int maxTreeLength, int maxTreeDepth, int maximumSemanticTries)
{
if (semantics == null || semantics.Length == 0)
{
ReplaceBranchManipulation.ReplaceRandomBranch(random, symbolicExpressionTree, maxTreeLength, maxTreeDepth);
SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(NoSemantics);
MutationTypeParameter.ActualValue = new IntValue(RandomMutation);
return;
}
var statementProductionNames = SemanticOperatorHelper.GetSemanticProductionNames(symbolicExpressionTree.Root.Grammar);
var variables = problemData.Variables.GetVariableNames().ToList();
string variableSettings = problemData.VariableSettings.Count == 0 ? String.Empty : String.Join(Environment.NewLine, problemData.VariableSettings.Select(x => x.Value));
var allowedSymbols = new List <ISymbol>();
// repeat until a fitting parent and child are found (MAX_TRIES times)
int tries = 0;
int semanticTries = 0;
List <JObject> saveOriginalSemantics = null;
List <JObject> saveReplaceSemantics = null;
List <Tuple <ISymbolicExpressionTreeNode, ISymbolicExpressionTreeNode, int> > possibleChildren = null; // Item1 = parent, Item2 = seedNode, Item3 = childIndex
if (UsesAdditionalSemanticMeasure())
{
saveOriginalSemantics = new List <JObject>(semanticTries);
saveReplaceSemantics = new List <JObject>(semanticTries);
possibleChildren = new List <Tuple <ISymbolicExpressionTreeNode, ISymbolicExpressionTreeNode, int> >(semanticTries);
}
bool success = false;
do
{
#region find mutation point
#pragma warning disable 612, 618
ISymbolicExpressionTreeNode parent = symbolicExpressionTree.Root.IterateNodesPrefix().Skip(1).Where(n => n.SubtreeCount > 0).SelectRandom(random);
#pragma warning restore 612, 618
int childIndex = random.Next(parent.SubtreeCount);
var child = parent.GetSubtree(childIndex);
int maxLength = maxTreeLength - symbolicExpressionTree.Length + child.GetLength();
int maxDepth = maxTreeDepth - symbolicExpressionTree.Root.GetBranchLevel(child);
allowedSymbols.Clear();
foreach (var symbol in parent.Grammar.GetAllowedChildSymbols(parent.Symbol, childIndex))
{
// check basic properties that the new symbol must have
if ((symbol.Name != child.Symbol.Name || symbol.MinimumArity > 0) &&
symbol.InitialFrequency > 0 &&
parent.Grammar.GetMinimumExpressionDepth(symbol) <= maxDepth &&
parent.Grammar.GetMinimumExpressionLength(symbol) <= maxLength)
{
allowedSymbols.Add(symbol);
}
}
#endregion
#region check for semantic difference with a new random tree
if (allowedSymbols.Count > 0)
{
if (semanticTries <= maximumSemanticTries)
{
// do semantic mutation
#region calculate original json output
ISymbolicExpressionTreeNode statement = SemanticOperatorHelper.GetStatementNode(child, statementProductionNames);
var statementPos0 = symbolicExpressionTree.IterateNodesPrefix().ToList().IndexOf(statement);
PythonStatementSemantic curSemantics = null;
if (String.IsNullOrEmpty(variableSettings))
{
curSemantics = semantics.First(x => x.TreeNodePrefixPos == statementPos0);
variableSettings = SemanticOperatorHelper.SemanticToPythonVariableSettings(curSemantics.Before, problemData.Variables.GetVariableTypes());
}
var jsonOriginal = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout);
// compare jsonOriginal to semantic after! Maybe avoid additional evaluation.
#endregion
var seedNode = GenerateAndInsertNewSubtree(random, parent, allowedSymbols, childIndex, maxLength, maxDepth);
#region calculate new json output
JObject jsonReplaced;
if (child == statement)
{
// child is executable, so is the new child
jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(seedNode, pythonProcess, random, problemData, variables, variableSettings, timeout);
}
else
{
jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout);
}
var exception = jsonOriginal["exception"] != null || jsonReplaced["exception"] != null;
if (exception)
{
if (jsonOriginal["exception"] != null)
{
MutationExceptionsParameter.ActualValue.Add(new StringValue(jsonOriginal["exception"].ToString()));
}
if (jsonReplaced["exception"] != null)
{
MutationExceptionsParameter.ActualValue.Add(new StringValue(jsonReplaced["exception"].ToString()));
}
}
if (curSemantics != null && !exception)
{
jsonOriginal = PythonSemanticComparer.ReplaceNotExecutedCases(jsonOriginal, curSemantics.Before, curSemantics.ExecutedCases);
jsonReplaced = PythonSemanticComparer.ReplaceNotExecutedCases(jsonReplaced, curSemantics.Before, curSemantics.ExecutedCases);
jsonOriginal = PythonSemanticComparer.ProduceDifference(jsonOriginal, curSemantics.Before);
jsonReplaced = PythonSemanticComparer.ProduceDifference(jsonReplaced, curSemantics.Before);
}
if (!exception && SemanticMeasure(jsonOriginal, jsonReplaced))
{
success = true;
SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(Different);
TypeSelectedForSimilarityParameter.ActualValue = new StringValue("First Semantic");
MutationTypeParameter.ActualValue = new IntValue(SemanticMutation);
}
else
{
// undo mutation
parent.RemoveSubtree(childIndex);
parent.InsertSubtree(childIndex, child);
allowedSymbols.Clear();
if (!exception && UsesAdditionalSemanticMeasure())
{
saveOriginalSemantics.Add(jsonOriginal);
saveReplaceSemantics.Add(jsonReplaced);
possibleChildren.Add(new Tuple <ISymbolicExpressionTreeNode, ISymbolicExpressionTreeNode, int>(parent, seedNode, childIndex));
}
}
if (problemData.VariableSettings.Count == 0)
{
// reset variableSettings
variableSettings = String.Empty;
}
semanticTries++;
#endregion
#region try second semantic comparison
if (!success && semanticTries >= maximumSemanticTries && UsesAdditionalSemanticMeasure())
{
for (int index = 0; index < saveOriginalSemantics.Count; index++)
{
if (AdditionalSemanticMeasure(saveOriginalSemantics[index], saveReplaceSemantics[index]))
{
var mutation = possibleChildren[index];
mutation.Item1.RemoveSubtree(mutation.Item3);
mutation.Item1.InsertSubtree(mutation.Item3, mutation.Item2);
success = true;
SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(Different);
TypeSelectedForSimilarityParameter.ActualValue = new StringValue("Second Semantic");
MutationTypeParameter.ActualValue = new IntValue(SemanticMutation);
break;
}
}
}
#endregion
}
else
{
// do random mutation
#region calculate original json output
ISymbolicExpressionTreeNode statement = SemanticOperatorHelper.GetStatementNode(child, statementProductionNames);
var statementPos0 = symbolicExpressionTree.IterateNodesPrefix().ToList().IndexOf(statement);
if (String.IsNullOrEmpty(variableSettings))
{
variableSettings = SemanticOperatorHelper.SemanticToPythonVariableSettings(semantics.First(x => x.TreeNodePrefixPos == statementPos0).Before, problemData.Variables.GetVariableTypes());
}
var jsonOriginal = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout);
#endregion
var seedNode = GenerateAndInsertNewSubtree(random, parent, allowedSymbols, childIndex, maxLength, maxDepth);
JObject jsonReplaced;
if (child == statement)
{
// child is executable, so is the new child
jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(seedNode, pythonProcess, random, problemData, variables, variableSettings, timeout);
}
else
{
jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout);
}
if (JToken.EqualityComparer.Equals(jsonOriginal, jsonReplaced))
{
SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(Equvivalent);
}
else
{
SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(Different);
}
TypeSelectedForSimilarityParameter.ActualValue = new StringValue("Random Crossover; Reached Max Semantic Tries");
MutationTypeParameter.ActualValue = new IntValue(RandomMutation);
success = true;
}
}
#endregion
tries++;
} while (tries < MAX_TRIES && !success);
NumberOfTriesParameter.ActualValue = new IntValue(semanticTries);
if (SemanticallyEquivalentMutationParameter.ActualValue == null)
{
SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(NoMutation);
TypeSelectedForSimilarityParameter.ActualValue = new StringValue("No mutation");
MutationTypeParameter.ActualValue = new IntValue(NoMutation);
}
}
public static bool DuplicateSubroutine(
IRandom random,
ISymbolicExpressionTree symbolicExpressionTree,
int maxFunctionDefinitions, int maxFunctionArguments)
{
var functionDefiningBranches = symbolicExpressionTree.IterateNodesPrefix().OfType <DefunTreeNode>().ToList();
if (!functionDefiningBranches.Any() || functionDefiningBranches.Count() == maxFunctionDefinitions)
{
// no function defining branches to duplicate or already reached the max number of ADFs
return(false);
}
string formatString = new StringBuilder().Append('0', (int)Math.Log10(maxFunctionDefinitions) + 1).ToString(); // >= 100 functions => ###
var allowedFunctionNames = from index in Enumerable.Range(0, maxFunctionDefinitions)
select "ADF" + index.ToString(formatString);
var selectedBranch = functionDefiningBranches.SampleRandom(random);
var duplicatedDefunBranch = (DefunTreeNode)selectedBranch.Clone();
string newFunctionName = allowedFunctionNames.Except(UsedFunctionNames(symbolicExpressionTree)).First();
duplicatedDefunBranch.FunctionName = newFunctionName;
symbolicExpressionTree.Root.AddSubtree(duplicatedDefunBranch);
duplicatedDefunBranch.SetGrammar((ISymbolicExpressionTreeGrammar)selectedBranch.Grammar.Clone());
var allowedChildSymbols = selectedBranch.Grammar.GetAllowedChildSymbols(selectedBranch.Symbol);
foreach (var allowedChildSymbol in allowedChildSymbols)
{
duplicatedDefunBranch.Grammar.AddAllowedChildSymbol(duplicatedDefunBranch.Symbol, allowedChildSymbol);
}
var maxSubtrees = selectedBranch.Grammar.GetMaximumSubtreeCount(selectedBranch.Symbol);
for (int i = 0; i < maxSubtrees; i++)
{
foreach (var allowedChildSymbol in selectedBranch.Grammar.GetAllowedChildSymbols(selectedBranch.Symbol, i))
{
duplicatedDefunBranch.Grammar.AddAllowedChildSymbol(duplicatedDefunBranch.Symbol, allowedChildSymbol);
}
}
// add an invoke symbol for each branch that is allowed to invoke the original function
foreach (var subtree in symbolicExpressionTree.Root.Subtrees.OfType <SymbolicExpressionTreeTopLevelNode>())
{
var matchingInvokeSymbol = (from symb in subtree.Grammar.Symbols.OfType <InvokeFunction>()
where symb.FunctionName == selectedBranch.FunctionName
select symb).SingleOrDefault();
if (matchingInvokeSymbol != null)
{
var invokeSymbol = new InvokeFunction(duplicatedDefunBranch.FunctionName);
subtree.Grammar.AddSymbol(invokeSymbol);
subtree.Grammar.SetSubtreeCount(invokeSymbol, duplicatedDefunBranch.NumberOfArguments, duplicatedDefunBranch.NumberOfArguments);
foreach (ISymbol symbol in subtree.Grammar.Symbols)
{
if (subtree.Grammar.IsAllowedChildSymbol(symbol, matchingInvokeSymbol))
{
subtree.Grammar.AddAllowedChildSymbol(symbol, invokeSymbol);
}
else
{
for (int i = 0; i < subtree.Grammar.GetMaximumSubtreeCount(symbol); i++)
{
if (subtree.Grammar.IsAllowedChildSymbol(symbol, matchingInvokeSymbol, i))
{
subtree.Grammar.AddAllowedChildSymbol(symbol, invokeSymbol, i);
}
}
}
}
foreach (ISymbol symbol in subtree.Grammar.GetAllowedChildSymbols(matchingInvokeSymbol))
{
if (symbol != invokeSymbol) //avoid duplicate entry invokesymbol / invokesymbol
{
subtree.Grammar.AddAllowedChildSymbol(invokeSymbol, symbol);
}
}
for (int i = 0; i < subtree.Grammar.GetMaximumSubtreeCount(matchingInvokeSymbol); i++)
{
foreach (ISymbol symbol in subtree.Grammar.GetAllowedChildSymbols(matchingInvokeSymbol, i).Except(subtree.Grammar.GetAllowedChildSymbols(matchingInvokeSymbol)))
{
subtree.Grammar.AddAllowedChildSymbol(invokeSymbol, symbol, i);
}
}
}
// in the current subtree:
// for all invoke nodes of the original function replace the invoke of the original function with an invoke of the new function randomly
var originalFunctionInvocations = from node in subtree.IterateNodesPrefix().OfType <InvokeFunctionTreeNode>()
where node.Symbol.FunctionName == selectedBranch.FunctionName
select node;
foreach (var originalFunctionInvokeNode in originalFunctionInvocations)
{
var newInvokeSymbol = (from symb in subtree.Grammar.Symbols.OfType <InvokeFunction>()
where symb.FunctionName == duplicatedDefunBranch.FunctionName
select symb).Single();
// flip coin wether to replace with newly defined function
if (random.NextDouble() < 0.5)
{
originalFunctionInvokeNode.Symbol = newInvokeSymbol;
}
}
}
return(true);
}
/// <summary>
/// 1. find a mutation point
/// 2. generate new random tree
/// 3. calculate semantic of old and new subtree (or of the closest parent)
/// 4. do mutation if semantically different
/// 5. retry until a certain number of tries is reached
///
/// if no mutation has happened, do random mutation
/// </summary>
public static void ReplaceSemanticallyDifferentBranch(IRandom random, ISymbolicExpressionTree symbolicExpressionTree, ICFGPythonProblemData problemData, ItemArray <PythonStatementSemantic> semantics, PythonProcess pythonProcess, double timeout, int maxTreeLength, int maxTreeDepth, int maximumSemanticTries)
{
if (semantics == null || semantics.Length == 0)
{
ReplaceBranchManipulation.ReplaceRandomBranch(random, symbolicExpressionTree, maxTreeLength, maxTreeDepth);
return;
}
var statementProductionNames = SemanticOperatorHelper.GetSemanticProductionNames(symbolicExpressionTree.Root.Grammar);
var variables = problemData.Variables.GetVariableNames().ToList();
string variableSettings = problemData.VariableSettings.Count == 0 ? String.Empty : String.Join(Environment.NewLine, problemData.VariableSettings.Select(x => x.Value));
var allowedSymbols = new List <ISymbol>();
// repeat until a fitting parent and child are found (MAX_TRIES times)
int tries = 0;
int semanticTries = 0;
do
{
#region find mutation point
#pragma warning disable 612, 618
ISymbolicExpressionTreeNode parent = symbolicExpressionTree.Root.IterateNodesPrefix().Skip(1).Where(n => n.SubtreeCount > 0).SelectRandom(random);
#pragma warning restore 612, 618
int childIndex = random.Next(parent.SubtreeCount);
var child = parent.GetSubtree(childIndex);
int maxLength = maxTreeLength - symbolicExpressionTree.Length + child.GetLength();
int maxDepth = maxTreeDepth - symbolicExpressionTree.Root.GetBranchLevel(child);
allowedSymbols.Clear();
foreach (var symbol in parent.Grammar.GetAllowedChildSymbols(parent.Symbol, childIndex))
{
// check basic properties that the new symbol must have
if ((symbol.Name != child.Symbol.Name || symbol.MinimumArity > 0) &&
symbol.InitialFrequency > 0 &&
parent.Grammar.GetMinimumExpressionDepth(symbol) <= maxDepth &&
parent.Grammar.GetMinimumExpressionLength(symbol) <= maxLength)
{
allowedSymbols.Add(symbol);
}
}
#endregion
#region check for semantic difference with a new random tree
if (allowedSymbols.Count > 0)
{
if (semanticTries <= maximumSemanticTries)
{
// do semantic mutation
#region calculate original json output
ISymbolicExpressionTreeNode statement = SemanticOperatorHelper.GetStatementNode(child, statementProductionNames);
var statementPos0 = symbolicExpressionTree.IterateNodesPrefix().ToList().IndexOf(statement);
if (String.IsNullOrEmpty(variableSettings))
{
variableSettings = SemanticOperatorHelper.SemanticToPythonVariableSettings(semantics.First(x => x.TreeNodePrefixPos == statementPos0).Before, problemData.Variables.GetVariableTypes());
}
var jsonOriginal = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout);
#endregion
var seedNode = GenerateAndInsertNewSubtree(random, parent, allowedSymbols, childIndex, maxLength, maxDepth);
#region calculate new json output
JObject jsonReplaced;
if (child == statement)
{
// child is executable, so is the new child
jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(seedNode, pythonProcess, random, problemData, variables, variableSettings, timeout);
}
else
{
jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout);
}
if (JToken.EqualityComparer.Equals(jsonOriginal, jsonReplaced))
{
// semantically equivalent. undo mutation
parent.RemoveSubtree(childIndex);
parent.InsertSubtree(childIndex, child);
allowedSymbols.Clear();
}
else
{
Console.WriteLine("never happens?");
}
if (problemData.VariableSettings.Count == 0)
{
// reset variableSettings
variableSettings = String.Empty;
}
semanticTries++;
#endregion
}
else
{
// do random mutation
GenerateAndInsertNewSubtree(random, parent, allowedSymbols, childIndex, maxLength, maxDepth);
}
}
#endregion
tries++;
} while (tries < MAX_TRIES && allowedSymbols.Count == 0 && semanticTries <= maximumSemanticTries);
}
public Tuple <IEnumerable <bool>, IEnumerable <double>, double, string, List <PythonStatementSemantic> > EvaluateAndTraceProgram(PythonProcess pythonProcess, string program, string input, string output, IEnumerable <int> indices, string header, string footer, ISymbolicExpressionTree tree, double timeout = 1)
{
string traceProgram = traceCodeWithVariables
+ program;
traceProgram += traceCodeWithVariables == String.Empty
? String.Empty
: traceTableReduceEntries;
EvaluationScript es = pythonProcess.CreateEvaluationScript(traceProgram, input, output, timeout);
es.Variables.Add("traceTable");
es.Variables.Add("traceTableBefore");
es.Variables.Add("executedLines");
JObject json = pythonProcess.SendAndEvaluateProgram(es);
var baseResult = pythonProcess.GetVariablesFromJson(json, indices.Count());
if (json["traceTable"] == null || json["traceTableBefore"] == null)
{
return(new Tuple <IEnumerable <bool>, IEnumerable <double>, double, string, List <PythonStatementSemantic> >(baseResult.Item1, baseResult.Item2, baseResult.Item3, baseResult.Item4, new List <PythonStatementSemantic>()));
}
var traceTable = JsonConvert.DeserializeObject <IDictionary <int, IDictionary <string, IList> > >(json["traceTable"].ToString());
var traceTableBefore = JsonConvert.DeserializeObject <IDictionary <int, IDictionary <string, IList> > >(json["traceTableBefore"].ToString());
var executedLinesNew = JsonConvert.DeserializeObject <IDictionary <int, List <int> > >(json["executedLines"].ToString());
var executedLines = executedLinesNew.Keys.ToList();
executedLines.Sort();
IList <int> traceChanges = traceTable.Keys.OrderBy(x => x).ToList();
IList <int> traceBeforeChanges = traceTableBefore.Keys.OrderBy(x => x).ToList();
List <PythonStatementSemantic> semantics = new List <PythonStatementSemantic>();
ISymbolicExpressionTreeNode root = tree.Root;
var statementProductionNames = SemanticOperatorHelper.GetSemanticProductionNames(root.Grammar);
// calculate the correct line the semantic evaluation starts from
var code = CFGSymbolicExpressionTreeStringFormatter.StaticFormat(tree);
int curline = es.Script.Count(c => c == '\n') - code.Count(c => c == '\n') - footer.Count(c => c == '\n') - traceTableReduceEntries.Count(c => c == '\n');
var symbolToLineDict = FindStatementSymbolsInTree(root, statementProductionNames, ref curline);
var prefixTreeNodes = tree.IterateNodesPrefix().ToList();
foreach (var symbolLine in symbolToLineDict)
{
// Before
Dictionary <string, IList> before = new Dictionary <string, IList>();
foreach (var traceChange in traceTableBefore.Where(x => x.Key <= symbolLine.Value[0]).OrderByDescending(x => x.Key))
{
foreach (var variableChange in traceChange.Value)
{
if (!before.ContainsKey(variableChange.Key))
{
before.Add(variableChange.Key, variableChange.Value);
}
}
}
// After
Dictionary <string, IList> after = new Dictionary <string, IList>();
foreach (var traceChange in traceTable.Where(x => x.Key >= symbolLine.Value[0] && x.Key <= symbolLine.Value[1]).OrderByDescending(x => x.Key))
{
foreach (var variableChange in traceChange.Value)
{
if (!after.ContainsKey(variableChange.Key))
{
after.Add(variableChange.Key, variableChange.Value);
}
}
}
// clean after with before
foreach (var key in after.Keys.ToList())
{
if (CompareSequence(after[key], before[key]))
{
after.Remove(key);
}
}
semantics.Add(new PythonStatementSemantic()
{
TreeNodePrefixPos = prefixTreeNodes.IndexOf(symbolLine.Key),
Before = before,
After = after,
});
}
return(new Tuple <IEnumerable <bool>, IEnumerable <double>, double, string, List <PythonStatementSemantic> >(baseResult.Item1, baseResult.Item2, baseResult.Item3, baseResult.Item4, semantics));
}