public static bool DeleteSubroutine( IRandom random, ISymbolicExpressionTree symbolicExpressionTree, int maxFunctionDefinitions, int maxFunctionArguments) { var functionDefiningBranches = symbolicExpressionTree.IterateNodesPrefix().OfType<DefunTreeNode>().ToList(); if (!functionDefiningBranches.Any()) // no ADF to delete => abort return false; var selectedDefunBranch = functionDefiningBranches.SampleRandom(random); // remove the selected defun int defunSubtreeIndex = symbolicExpressionTree.Root.IndexOfSubtree(selectedDefunBranch); symbolicExpressionTree.Root.RemoveSubtree(defunSubtreeIndex); // remove references to deleted function foreach (var subtree in symbolicExpressionTree.Root.Subtrees.OfType<SymbolicExpressionTreeTopLevelNode>()) { var matchingInvokeSymbol = (from symb in subtree.Grammar.Symbols.OfType<InvokeFunction>() where symb.FunctionName == selectedDefunBranch.FunctionName select symb).SingleOrDefault(); if (matchingInvokeSymbol != null) { subtree.Grammar.RemoveSymbol(matchingInvokeSymbol); } } DeletionByRandomRegeneration(random, symbolicExpressionTree, selectedDefunBranch); return true; }
public static Instruction[] Compile(ISymbolicExpressionTree tree, Func<ISymbolicExpressionTreeNode, byte> opCodeMapper, IEnumerable<Func<Instruction, Instruction>> postInstructionCompiledHooks) { Dictionary<string, ushort> entryPoint = new Dictionary<string, ushort>(); List<Instruction> code = new List<Instruction>(); // compile main body branches foreach (var branch in tree.Root.GetSubtree(0).Subtrees) { code.AddRange(Compile(branch, opCodeMapper, postInstructionCompiledHooks)); } // compile function branches var functionBranches = from node in tree.IterateNodesPrefix() where node.Symbol is Defun select node; foreach (DefunTreeNode branch in functionBranches) { if (code.Count > ushort.MaxValue) throw new ArgumentException("Code for the tree is too long (> ushort.MaxValue)."); entryPoint[branch.FunctionName] = (ushort)code.Count; code.AddRange(Compile(branch.GetSubtree(0), opCodeMapper, postInstructionCompiledHooks)); } // address of all functions is fixed now // iterate through code again and fill in the jump locations for (int i = 0; i < code.Count; i++) { Instruction instr = code[i]; if (instr.dynamicNode.Symbol is InvokeFunction) { var invokeNode = (InvokeFunctionTreeNode)instr.dynamicNode; instr.data = entryPoint[invokeNode.Symbol.FunctionName]; } } return code.ToArray(); }
public static bool DeleteArgument( IRandom random, ISymbolicExpressionTree symbolicExpressionTree, int maxFunctionDefinitions, int maxFunctionArguments) { var functionDefiningBranches = symbolicExpressionTree.IterateNodesPrefix().OfType<DefunTreeNode>().ToList(); if (!functionDefiningBranches.Any()) // no function defining branch => abort return false; var selectedDefunBranch = functionDefiningBranches.SampleRandom(random); if (selectedDefunBranch.NumberOfArguments <= 1) // argument deletion by consolidation is not possible => abort return false; // the argument to be removed is always the one with the largest index // (otherwise we would have to decrement the index of the larger argument symbols) var removedArgument = (from sym in selectedDefunBranch.Grammar.Symbols.OfType<Argument>() select sym.ArgumentIndex).Distinct().OrderBy(x => x).Last(); // find invocations of the manipulated funcion and remove the specified argument tree var invocationNodes = (from node in symbolicExpressionTree.IterateNodesPrefix().OfType<InvokeFunctionTreeNode>() where node.Symbol.FunctionName == selectedDefunBranch.FunctionName select node).ToList(); foreach (var invokeNode in invocationNodes) { invokeNode.RemoveSubtree(removedArgument); } DeleteArgumentByConsolidation(random, selectedDefunBranch, removedArgument); // delete the dynamic argument symbol that matches the argument to be removed var matchingSymbol = selectedDefunBranch.Grammar.Symbols.OfType<Argument>().Single(s => s.ArgumentIndex == removedArgument); selectedDefunBranch.Grammar.RemoveSymbol(matchingSymbol); selectedDefunBranch.NumberOfArguments--; // reduce arity in known functions of all root branches foreach (var subtree in symbolicExpressionTree.Root.Subtrees) { var matchingInvokeSymbol = subtree.Grammar.Symbols.OfType<InvokeFunction>().SingleOrDefault(s => s.FunctionName == selectedDefunBranch.FunctionName); if (matchingInvokeSymbol != null) { subtree.Grammar.SetSubtreeCount(matchingInvokeSymbol, selectedDefunBranch.NumberOfArguments, selectedDefunBranch.NumberOfArguments); } } return true; }
public static Instruction[] Compile(ISymbolicExpressionTree tree, Func <ISymbolicExpressionTreeNode, byte> opCodeMapper, IEnumerable <Func <Instruction, Instruction> > postInstructionCompiledHooks) { Dictionary <string, ushort> entryPoint = new Dictionary <string, ushort>(); List <Instruction> code = new List <Instruction>(); // compile main body branches foreach (var branch in tree.Root.GetSubtree(0).Subtrees) { code.AddRange(Compile(branch, opCodeMapper, postInstructionCompiledHooks)); } // compile function branches var functionBranches = from node in tree.IterateNodesPrefix() where node.Symbol is Defun select node; foreach (DefunTreeNode branch in functionBranches) { if (code.Count > ushort.MaxValue) { throw new ArgumentException("Code for the tree is too long (> ushort.MaxValue)."); } entryPoint[branch.FunctionName] = (ushort)code.Count; code.AddRange(Compile(branch.GetSubtree(0), opCodeMapper, postInstructionCompiledHooks)); } // address of all functions is fixed now // iterate through code again and fill in the jump locations for (int i = 0; i < code.Count; i++) { Instruction instr = code[i]; if (instr.dynamicNode.Symbol is InvokeFunction) { var invokeNode = (InvokeFunctionTreeNode)instr.dynamicNode; instr.data = entryPoint[invokeNode.Symbol.FunctionName]; } } return(code.ToArray()); }
private static Instruction[] PrepareInterpreterState(ISymbolicExpressionTree tree, IDictionary <string, Interval> variableRanges) { if (variableRanges == null) { throw new ArgumentNullException("No variablew ranges are present!", nameof(variableRanges)); } //Check if all variables used in the tree are present in the dataset foreach (var variable in tree.IterateNodesPrefix().OfType <VariableTreeNode>().Select(n => n.VariableName).Distinct()) { if (!variableRanges.ContainsKey(variable)) { throw new InvalidOperationException($"No ranges for variable {variable} is present"); } } Instruction[] code = SymbolicExpressionTreeCompiler.Compile(tree, OpCodes.MapSymbolToOpCode); foreach (Instruction instr in code.Where(i => i.opCode == OpCodes.Variable)) { var variableTreeNode = (VariableTreeNode)instr.dynamicNode; instr.data = variableRanges[variableTreeNode.VariableName]; } return(code); }
private static IEnumerable<string> UsedFunctionNames(ISymbolicExpressionTree symbolicExpressionTree) { return from node in symbolicExpressionTree.IterateNodesPrefix() where node.Symbol is Defun select ((DefunTreeNode)node).FunctionName; }
public static bool CreateSubroutine( IRandom random, ISymbolicExpressionTree symbolicExpressionTree, int maxTreeLength, int maxTreeDepth, int maxFunctionDefinitions, int maxFunctionArguments) { var functionDefiningBranches = symbolicExpressionTree.IterateNodesPrefix().OfType<DefunTreeNode>(); if (functionDefiningBranches.Count() >= maxFunctionDefinitions) // allowed maximum number of ADF reached => abort return false; if (symbolicExpressionTree.Length + 4 > maxTreeLength) // defining a new function causes an length increase by 4 nodes (max) if the max tree length is reached => abort return false; string formatString = new StringBuilder().Append('0', (int)Math.Log10(maxFunctionDefinitions * 10 - 1)).ToString(); // >= 100 functions => ### var allowedFunctionNames = from index in Enumerable.Range(0, maxFunctionDefinitions) select "ADF" + index.ToString(formatString); // select a random body (either the result producing branch or an ADF branch) var bodies = from node in symbolicExpressionTree.Root.Subtrees select new { Tree = node, Length = node.GetLength() }; var totalNumberOfBodyNodes = bodies.Select(x => x.Length).Sum(); int r = random.Next(totalNumberOfBodyNodes); int aggregatedNumberOfBodyNodes = 0; ISymbolicExpressionTreeNode selectedBody = null; foreach (var body in bodies) { aggregatedNumberOfBodyNodes += body.Length; if (aggregatedNumberOfBodyNodes > r) selectedBody = body.Tree; } // sanity check if (selectedBody == null) throw new InvalidOperationException(); // select a random cut point in the selected branch var allCutPoints = (from parent in selectedBody.IterateNodesPrefix() from subtree in parent.Subtrees select new CutPoint(parent, subtree)).ToList(); if (!allCutPoints.Any()) // no cut points => abort return false; string newFunctionName = allowedFunctionNames.Except(functionDefiningBranches.Select(x => x.FunctionName)).First(); var selectedCutPoint = allCutPoints.SampleRandom(random); // select random branches as argument cut-off points (replaced by argument terminal nodes in the function) List<CutPoint> argumentCutPoints = SelectRandomArgumentBranches(selectedCutPoint.Child, random, ARGUMENT_CUTOFF_PROBABILITY, maxFunctionArguments); ISymbolicExpressionTreeNode functionBody = selectedCutPoint.Child; // disconnect the function body from the tree selectedCutPoint.Parent.RemoveSubtree(selectedCutPoint.ChildIndex); // disconnect the argument branches from the function functionBody = DisconnectBranches(functionBody, argumentCutPoints); // insert a function invocation symbol instead var invokeNode = (InvokeFunctionTreeNode)(new InvokeFunction(newFunctionName)).CreateTreeNode(); selectedCutPoint.Parent.InsertSubtree(selectedCutPoint.ChildIndex, invokeNode); // add the branches selected as argument as subtrees of the function invocation node foreach (var argumentCutPoint in argumentCutPoints) invokeNode.AddSubtree(argumentCutPoint.Child); // insert a new function defining branch var defunNode = (DefunTreeNode)(new Defun()).CreateTreeNode(); defunNode.FunctionName = newFunctionName; defunNode.AddSubtree(functionBody); symbolicExpressionTree.Root.AddSubtree(defunNode); // the grammar in the newly defined function is a clone of the grammar of the originating branch defunNode.SetGrammar((ISymbolicExpressionTreeGrammar)selectedBody.Grammar.Clone()); var allowedChildSymbols = selectedBody.Grammar.GetAllowedChildSymbols(selectedBody.Symbol); foreach (var allowedChildSymbol in allowedChildSymbols) defunNode.Grammar.AddAllowedChildSymbol(defunNode.Symbol, allowedChildSymbol); var maxSubtrees = selectedBody.Grammar.GetMaximumSubtreeCount(selectedBody.Symbol); for (int i = 0; i < maxSubtrees; i++) { foreach (var allowedChildSymbol in selectedBody.Grammar.GetAllowedChildSymbols(selectedBody.Symbol, i)) defunNode.Grammar.AddAllowedChildSymbol(defunNode.Symbol, allowedChildSymbol); } // remove all argument symbols from grammar except that one contained in cutpoints var oldArgumentSymbols = selectedBody.Grammar.Symbols.OfType<Argument>().ToList(); foreach (var oldArgSymb in oldArgumentSymbols) defunNode.Grammar.RemoveSymbol(oldArgSymb); // find unique argument indexes and matching symbols in the function defining branch var newArgumentIndexes = (from node in defunNode.IterateNodesPrefix().OfType<ArgumentTreeNode>() select node.Symbol.ArgumentIndex).Distinct(); // add argument symbols to grammar of function defining branch GrammarModifier.AddArgumentSymbol(selectedBody.Grammar, defunNode.Grammar, newArgumentIndexes, argumentCutPoints); defunNode.NumberOfArguments = newArgumentIndexes.Count(); if (defunNode.NumberOfArguments != argumentCutPoints.Count) throw new InvalidOperationException(); // add invoke symbol for newly defined function to the original branch GrammarModifier.AddInvokeSymbol(selectedBody.Grammar, defunNode.FunctionName, defunNode.NumberOfArguments, selectedCutPoint, argumentCutPoints); // when the new function body was taken from another function definition // add invoke symbol for newly defined function to all branches that are allowed to invoke the original branch if (selectedBody.Symbol is Defun) { var originalFunctionDefinition = selectedBody as DefunTreeNode; foreach (var subtree in symbolicExpressionTree.Root.Subtrees) { var originalBranchInvokeSymbol = (from symb in subtree.Grammar.Symbols.OfType<InvokeFunction>() where symb.FunctionName == originalFunctionDefinition.FunctionName select symb).SingleOrDefault(); // when the original branch can be invoked from the subtree then also allow invocation of the function if (originalBranchInvokeSymbol != null) { GrammarModifier.AddInvokeSymbol(subtree.Grammar, defunNode.FunctionName, defunNode.NumberOfArguments, selectedCutPoint, argumentCutPoints); } } } return true; }
public static bool DuplicateArgument( IRandom random, ISymbolicExpressionTree symbolicExpressionTree, int maxFunctionDefinitions, int maxFunctionArguments) { var functionDefiningBranches = symbolicExpressionTree.IterateNodesPrefix().OfType<DefunTreeNode>().ToList(); var allowedArgumentIndexes = Enumerable.Range(0, maxFunctionArguments); if (!functionDefiningBranches.Any()) // no function defining branches => abort return false; var selectedDefunBranch = functionDefiningBranches.SampleRandom(random); var argumentSymbols = selectedDefunBranch.Grammar.Symbols.OfType<Argument>().ToList(); if (!argumentSymbols.Any() || argumentSymbols.Count() >= maxFunctionArguments) // when no argument or number of arguments is already at max allowed value => abort return false; var selectedArgumentSymbol = argumentSymbols.SampleRandom(random); var takenIndexes = argumentSymbols.Select(s => s.ArgumentIndex); var newArgumentIndex = allowedArgumentIndexes.Except(takenIndexes).First(); var newArgSymbol = new Argument(newArgumentIndex); // replace existing references to the original argument with references to the new argument randomly in the selectedBranch var argumentNodes = selectedDefunBranch.IterateNodesPrefix().OfType<ArgumentTreeNode>(); foreach (var argNode in argumentNodes) { if (argNode.Symbol == selectedArgumentSymbol) { if (random.NextDouble() < 0.5) { argNode.Symbol = newArgSymbol; } } } // find invocations of the functions and duplicate the matching argument branch var invocationNodes = (from node in symbolicExpressionTree.IterateNodesPrefix().OfType<InvokeFunctionTreeNode>() where node.Symbol.FunctionName == selectedDefunBranch.FunctionName where node.Subtrees.Count() == selectedDefunBranch.NumberOfArguments select node).ToList(); // do this repeatedly until no matching invocations are found while (invocationNodes.Count() > 0) { List<ISymbolicExpressionTreeNode> newlyAddedBranches = new List<ISymbolicExpressionTreeNode>(); foreach (var invokeNode in invocationNodes) { // check that the invocation node really has the correct number of arguments if (invokeNode.Subtrees.Count() != selectedDefunBranch.NumberOfArguments) throw new InvalidOperationException(); var argumentBranch = invokeNode.GetSubtree(selectedArgumentSymbol.ArgumentIndex); var clonedArgumentBranch = (ISymbolicExpressionTreeNode)argumentBranch.Clone(); invokeNode.InsertSubtree(newArgumentIndex, clonedArgumentBranch); newlyAddedBranches.Add(clonedArgumentBranch); } invocationNodes = (from newlyAddedBranch in newlyAddedBranches from node in newlyAddedBranch.IterateNodesPrefix().OfType<InvokeFunctionTreeNode>() where node.Symbol.FunctionName == selectedDefunBranch.FunctionName where node.Subtrees.Count() == selectedDefunBranch.NumberOfArguments select node).ToList(); } // register the new argument symbol and increase the number of arguments of the ADF selectedDefunBranch.Grammar.AddSymbol(newArgSymbol); selectedDefunBranch.Grammar.SetSubtreeCount(newArgSymbol, 0, 0); // allow the duplicated argument as child of all other arguments where the orginal argument was allowed GrammarModifier.SetAllowedParentSymbols(selectedDefunBranch.Grammar, selectedArgumentSymbol, newArgSymbol); selectedDefunBranch.NumberOfArguments++; // increase the arity of the changed ADF in all branches that can use this ADF foreach (var subtree in symbolicExpressionTree.Root.Subtrees) { var matchingInvokeSymbol = (from symb in subtree.Grammar.Symbols.OfType<InvokeFunction>() where symb.FunctionName == selectedDefunBranch.FunctionName select symb).SingleOrDefault(); if (matchingInvokeSymbol != null) { subtree.Grammar.SetSubtreeCount(matchingInvokeSymbol, selectedDefunBranch.NumberOfArguments, selectedDefunBranch.NumberOfArguments); foreach (var symb in subtree.Grammar.Symbols) { if (symb is StartSymbol || symb is ProgramRootSymbol) continue; if (subtree.Grammar.IsAllowedChildSymbol(matchingInvokeSymbol, symb, selectedArgumentSymbol.ArgumentIndex)) subtree.Grammar.AddAllowedChildSymbol(matchingInvokeSymbol, symb, newArgumentIndex); } } } return true; }
protected override ISymbolicExpressionTree Cross(IRandom random, ISymbolicExpressionTree parent0, ISymbolicExpressionTree parent1, ItemArray <PythonStatementSemantic> semantic0, ItemArray <PythonStatementSemantic> semantic1, T problemData, int maxTreeLength, int maxTreeDepth, double internalCrossoverPointProbability, out ItemArray <PythonStatementSemantic> newSemantics) { if (semantic0 == null || semantic1 == null || semantic0.Length == 0 || semantic1.Length == 0) { parent0 = SubtreeCrossover.Cross(random, parent0, parent1, internalCrossoverPointProbability, maxTreeLength, maxTreeDepth); newSemantics = null; AddStatisticsNoCrossover(NoXoNoSemantics); return(parent0); } newSemantics = semantic0; // select andom crossover points in the first parent var crossoverPoints0 = SelectCrossoverPointsOfMatchingType(random, parent0, internalCrossoverPointProbability, maxTreeLength, maxTreeDepth, NumberOfCutPointsFirst); var primaryCrossoverPoint = crossoverPoints0.First(); int childLength = primaryCrossoverPoint.Child != null?primaryCrossoverPoint.Child.GetLength() : 0; // calculate the max length and depth that the inserted branch can have int maxInsertedBranchLength = maxTreeLength - (parent0.Length - childLength); int maxInsertedBranchDepth = maxTreeDepth - parent0.Root.GetBranchLevel(primaryCrossoverPoint.Child); List <ISymbolicExpressionTreeNode> allowedBranches = new List <ISymbolicExpressionTreeNode>(); parent1.Root.ForEachNodePostfix((n) => { if (n.GetLength() <= maxInsertedBranchLength && n.GetDepth() <= maxInsertedBranchDepth && primaryCrossoverPoint.IsMatchingPointType(n)) { allowedBranches.Add(n); } }); // empty branch if (primaryCrossoverPoint.IsMatchingPointType(null)) { allowedBranches.Add(null); } // set NumberOfAllowedBranches NumberOfAllowedBranches = allowedBranches.Count; if (allowedBranches.Count == 0) { AddStatisticsNoCrossover(NoXoNoAllowedBranch); return(parent0); } // select NumberOfCutPointsSecond random crossover points // Ignore internalCrossoverPointProbability, was already used to select cutpoint and the cutpoint child already reduces possibilities var compBranches = allowedBranches.SampleRandomWithoutRepetition(random, NumberOfCutPointsSecond).ToList(); // convert to list so that the linq expression is only executed once, to always get the same branches var allowedBranchesPerCutpoint = new List <IEnumerable <ISymbolicExpressionTreeNode> >() { compBranches }; allowedBranchesPerCutpoint.AddRange(crossoverPoints0.Skip(1).Select(x => FindFittingNodes(x, parent0, compBranches, maxTreeLength, maxTreeDepth))); // set NumberOfPossibleBranchesSelected NumberOfPossibleBranchesSelected = compBranches.Count; var statementProductionNames = SemanticOperatorHelper.GetSemanticProductionNames(primaryCrossoverPoint.Parent.Grammar); // find first node that can be used for evaluation in parent0 ISymbolicExpressionTreeNode statement = SemanticOperatorHelper.GetStatementNode(primaryCrossoverPoint.Child, statementProductionNames); if (statement == null) { newSemantics = SemanticSwap(primaryCrossoverPoint, compBranches.SampleRandom(random), parent0, parent1, semantic0, semantic1); AddStatisticsNoCrossover(NoXoNoStatement); return(parent0); } var statementPos0 = parent0.IterateNodesPrefix().ToList().IndexOf(statement); string variableSettings; if (problemData.VariableSettings.Count == 0) { variableSettings = SemanticOperatorHelper.SemanticToPythonVariableSettings(semantic0.First(x => x.TreeNodePrefixPos == statementPos0).Before, problemData.Variables.GetVariableTypes()); } else { variableSettings = String.Join(Environment.NewLine, problemData.VariableSettings.Select(x => x.Value)); } var variables = problemData.Variables.GetVariableNames().ToList(); // only used for analyzation, otherwise could be removed var json0 = SemanticOperatorHelper.EvaluateStatementNode(statement, PyProcess, random, problemData, variables, variableSettings, Timeout); ISymbolicExpressionTreeNode selectedBranch; CutPoint selectedCutPoint; if (statement == primaryCrossoverPoint.Child) { SelectBranchFromExecutableNodes(crossoverPoints0.ToList(), allowedBranchesPerCutpoint, random, variables, variableSettings, problemData, out selectedBranch, out selectedCutPoint); } else { //SelectBranchFromNodes1(statement, crossoverPoints0.ToList(), allowedBranchesPerCutpoint, random, variables, variableSettings, problemData, out selectedBranch, out selectedCutPoint); SelectBranchFromNodes2(statementProductionNames, crossoverPoints0.ToList(), allowedBranchesPerCutpoint, random, variables, variableSettings, problemData, out selectedBranch, out selectedCutPoint); } // perform the actual swap if (selectedBranch != null) { newSemantics = SemanticSwap(selectedCutPoint, selectedBranch, parent0, parent1, semantic0, semantic1); AddStatistics(semantic0, parent0, statement == primaryCrossoverPoint.Child ? selectedBranch : statement, selectedCutPoint, json0, selectedBranch, random, problemData, variables, variableSettings); // parent zero has been changed is now considered the child } else { AddStatisticsNoCrossover(NoXoNoSelectedBranch); } return(parent0); }
public static bool CreateSubroutine( IRandom random, ISymbolicExpressionTree symbolicExpressionTree, int maxTreeLength, int maxTreeDepth, int maxFunctionDefinitions, int maxFunctionArguments) { var functionDefiningBranches = symbolicExpressionTree.IterateNodesPrefix().OfType <DefunTreeNode>(); if (functionDefiningBranches.Count() >= maxFunctionDefinitions) { // allowed maximum number of ADF reached => abort return(false); } if (symbolicExpressionTree.Length + 4 > maxTreeLength) { // defining a new function causes an length increase by 4 nodes (max) if the max tree length is reached => abort return(false); } string formatString = new StringBuilder().Append('0', (int)Math.Log10(maxFunctionDefinitions * 10 - 1)).ToString(); // >= 100 functions => ### var allowedFunctionNames = from index in Enumerable.Range(0, maxFunctionDefinitions) select "ADF" + index.ToString(formatString); // select a random body (either the result producing branch or an ADF branch) var bodies = from node in symbolicExpressionTree.Root.Subtrees select new { Tree = node, Length = node.GetLength() }; var totalNumberOfBodyNodes = bodies.Select(x => x.Length).Sum(); int r = random.Next(totalNumberOfBodyNodes); int aggregatedNumberOfBodyNodes = 0; ISymbolicExpressionTreeNode selectedBody = null; foreach (var body in bodies) { aggregatedNumberOfBodyNodes += body.Length; if (aggregatedNumberOfBodyNodes > r) { selectedBody = body.Tree; } } // sanity check if (selectedBody == null) { throw new InvalidOperationException(); } // select a random cut point in the selected branch var allCutPoints = (from parent in selectedBody.IterateNodesPrefix() from subtree in parent.Subtrees select new CutPoint(parent, subtree)).ToList(); if (!allCutPoints.Any()) { // no cut points => abort return(false); } string newFunctionName = allowedFunctionNames.Except(functionDefiningBranches.Select(x => x.FunctionName)).First(); var selectedCutPoint = allCutPoints.SampleRandom(random); // select random branches as argument cut-off points (replaced by argument terminal nodes in the function) List <CutPoint> argumentCutPoints = SelectRandomArgumentBranches(selectedCutPoint.Child, random, ARGUMENT_CUTOFF_PROBABILITY, maxFunctionArguments); ISymbolicExpressionTreeNode functionBody = selectedCutPoint.Child; // disconnect the function body from the tree selectedCutPoint.Parent.RemoveSubtree(selectedCutPoint.ChildIndex); // disconnect the argument branches from the function functionBody = DisconnectBranches(functionBody, argumentCutPoints); // insert a function invocation symbol instead var invokeNode = (InvokeFunctionTreeNode)(new InvokeFunction(newFunctionName)).CreateTreeNode(); selectedCutPoint.Parent.InsertSubtree(selectedCutPoint.ChildIndex, invokeNode); // add the branches selected as argument as subtrees of the function invocation node foreach (var argumentCutPoint in argumentCutPoints) { invokeNode.AddSubtree(argumentCutPoint.Child); } // insert a new function defining branch var defunNode = (DefunTreeNode)(new Defun()).CreateTreeNode(); defunNode.FunctionName = newFunctionName; defunNode.AddSubtree(functionBody); symbolicExpressionTree.Root.AddSubtree(defunNode); // the grammar in the newly defined function is a clone of the grammar of the originating branch defunNode.SetGrammar((ISymbolicExpressionTreeGrammar)selectedBody.Grammar.Clone()); var allowedChildSymbols = selectedBody.Grammar.GetAllowedChildSymbols(selectedBody.Symbol); foreach (var allowedChildSymbol in allowedChildSymbols) { defunNode.Grammar.AddAllowedChildSymbol(defunNode.Symbol, allowedChildSymbol); } var maxSubtrees = selectedBody.Grammar.GetMaximumSubtreeCount(selectedBody.Symbol); for (int i = 0; i < maxSubtrees; i++) { foreach (var allowedChildSymbol in selectedBody.Grammar.GetAllowedChildSymbols(selectedBody.Symbol, i)) { defunNode.Grammar.AddAllowedChildSymbol(defunNode.Symbol, allowedChildSymbol); } } // remove all argument symbols from grammar except that one contained in cutpoints var oldArgumentSymbols = selectedBody.Grammar.Symbols.OfType <Argument>().ToList(); foreach (var oldArgSymb in oldArgumentSymbols) { defunNode.Grammar.RemoveSymbol(oldArgSymb); } // find unique argument indexes and matching symbols in the function defining branch var newArgumentIndexes = (from node in defunNode.IterateNodesPrefix().OfType <ArgumentTreeNode>() select node.Symbol.ArgumentIndex).Distinct(); // add argument symbols to grammar of function defining branch GrammarModifier.AddArgumentSymbol(selectedBody.Grammar, defunNode.Grammar, newArgumentIndexes, argumentCutPoints); defunNode.NumberOfArguments = newArgumentIndexes.Count(); if (defunNode.NumberOfArguments != argumentCutPoints.Count) { throw new InvalidOperationException(); } // add invoke symbol for newly defined function to the original branch GrammarModifier.AddInvokeSymbol(selectedBody.Grammar, defunNode.FunctionName, defunNode.NumberOfArguments, selectedCutPoint, argumentCutPoints); // when the new function body was taken from another function definition // add invoke symbol for newly defined function to all branches that are allowed to invoke the original branch if (selectedBody.Symbol is Defun) { var originalFunctionDefinition = selectedBody as DefunTreeNode; foreach (var subtree in symbolicExpressionTree.Root.Subtrees) { var originalBranchInvokeSymbol = (from symb in subtree.Grammar.Symbols.OfType <InvokeFunction>() where symb.FunctionName == originalFunctionDefinition.FunctionName select symb).SingleOrDefault(); // when the original branch can be invoked from the subtree then also allow invocation of the function if (originalBranchInvokeSymbol != null) { GrammarModifier.AddInvokeSymbol(subtree.Grammar, defunNode.FunctionName, defunNode.NumberOfArguments, selectedCutPoint, argumentCutPoints); } } } return(true); }
public Tuple<IEnumerable<bool>, IEnumerable<double>, double, string, List<PythonStatementSemantic>> EvaluateAndTraceProgram(string program, string input, string output, IEnumerable<int> indices, string header, ISymbolicExpressionTree tree, double timeout = 1) { string traceProgram = traceCodeWithVariables + program; traceProgram += traceCodeWithVariables == String.Empty ? String.Empty : traceTableReduceEntries; EvaluationScript es = pythonProcessParameter.ActualValue.CreateEvaluationScript(traceProgram, input, output, timeout); es.Variables.Add("traceTable"); JObject json = pythonProcessParameter.ActualValue.SendAndEvaluateProgram(es); var baseResult = pythonProcessParameter.ActualValue.GetVariablesFromJson(json, indices.Count()); if (json["traceTable"] == null) { return new Tuple<IEnumerable<bool>, IEnumerable<double>, double, string, List<PythonStatementSemantic>>(baseResult.Item1, baseResult.Item2, baseResult.Item3, baseResult.Item4, new List<PythonStatementSemantic>()); } var traceTable = JsonConvert.DeserializeObject<IDictionary<int, IDictionary<string, IList>>>(json["traceTable"].ToString()); IList<int> traceChanges = traceTable.Keys.OrderBy(x => x).ToList(); List<PythonStatementSemantic> semantics = new List<PythonStatementSemantic>(); ISymbolicExpressionTreeNode root = tree.Root; var statementProductions = ((GroupSymbol)root.Grammar.GetSymbol("Rule: <code>")).Symbols; var statementProductionNames = statementProductions.Select(x => x.Name); IList<int> lineTraces = traceTable.Keys.OrderBy(x => x).ToList(); // add one, because the values are set after the statement is executed // add two, for inval and outval int curline = traceCode.Count(c => c == '\n') + header.Count(c => c == '\n') + 1 + 2; var symbolToLineDict = FindStatementSymbolsInTree(root, statementProductionNames, ref curline); var symbolLines = symbolToLineDict.Values.OrderBy(x => x).ToList(); var prefixTreeNodes = tree.IterateNodesPrefix().ToList(); foreach (var symbolLine in symbolToLineDict) { Dictionary<string, IList> before = new Dictionary<string, IList>(); var linesBefore = lineTraces.Where(x => x <= symbolLine.Value).OrderByDescending(x => x); foreach (var l in linesBefore) { foreach (var change in traceTable[l]) { if (!before.ContainsKey(change.Key)) { before.Add(change.Key, change.Value); } } } int after = -1; int pos = traceChanges.IndexOf(linesBefore.Max()); if (pos + 1 < traceChanges.Count // has to be in the array // there cannot be another line which comes after the current one, but before the trace change // otherwise the current line did not change anything && !symbolLines.Any(x => x > symbolLine.Value && x < traceChanges[pos + 1])) { after = traceChanges[pos + 1]; } if (after >= 0) { semantics.Add(new PythonStatementSemantic() { TreeNodePrefixPos = prefixTreeNodes.IndexOf(symbolLine.Key), Before = before, After = traceTable[after], }); } else { semantics.Add(new PythonStatementSemantic() { TreeNodePrefixPos = prefixTreeNodes.IndexOf(symbolLine.Key), Before = before, After = new Dictionary<string, IList>(), }); } } return new Tuple<IEnumerable<bool>, IEnumerable<double>, double, string, List<PythonStatementSemantic>>(baseResult.Item1, baseResult.Item2, baseResult.Item3, baseResult.Item4, semantics); }
private void CalculateVariableMapping(ISymbolicExpressionTree tree, IDataset dataset) { int columnIndex = 0; int inputIndex = 0; var usedVariables = tree.IterateNodesPrefix().OfType<VariableTreeNode>().Select(v => v.VariableName).Distinct(); foreach (var variable in dataset.VariableNames) { columnIndex++; if (!usedVariables.Contains(variable)) continue; inputIndex++; variableNameMapping[variable] = GetExcelColumnName(inputIndex); } }
public static bool DuplicateArgument( IRandom random, ISymbolicExpressionTree symbolicExpressionTree, int maxFunctionDefinitions, int maxFunctionArguments) { var functionDefiningBranches = symbolicExpressionTree.IterateNodesPrefix().OfType <DefunTreeNode>().ToList(); var allowedArgumentIndexes = Enumerable.Range(0, maxFunctionArguments); if (!functionDefiningBranches.Any()) { // no function defining branches => abort return(false); } var selectedDefunBranch = functionDefiningBranches.SampleRandom(random); var argumentSymbols = selectedDefunBranch.Grammar.Symbols.OfType <Argument>().ToList(); if (!argumentSymbols.Any() || argumentSymbols.Count() >= maxFunctionArguments) { // when no argument or number of arguments is already at max allowed value => abort return(false); } var selectedArgumentSymbol = argumentSymbols.SampleRandom(random); var takenIndexes = argumentSymbols.Select(s => s.ArgumentIndex); var newArgumentIndex = allowedArgumentIndexes.Except(takenIndexes).First(); var newArgSymbol = new Argument(newArgumentIndex); // replace existing references to the original argument with references to the new argument randomly in the selectedBranch var argumentNodes = selectedDefunBranch.IterateNodesPrefix().OfType <ArgumentTreeNode>(); foreach (var argNode in argumentNodes) { if (argNode.Symbol == selectedArgumentSymbol) { if (random.NextDouble() < 0.5) { argNode.Symbol = newArgSymbol; } } } // find invocations of the functions and duplicate the matching argument branch var invocationNodes = (from node in symbolicExpressionTree.IterateNodesPrefix().OfType <InvokeFunctionTreeNode>() where node.Symbol.FunctionName == selectedDefunBranch.FunctionName where node.Subtrees.Count() == selectedDefunBranch.NumberOfArguments select node).ToList(); // do this repeatedly until no matching invocations are found while (invocationNodes.Count() > 0) { List <ISymbolicExpressionTreeNode> newlyAddedBranches = new List <ISymbolicExpressionTreeNode>(); foreach (var invokeNode in invocationNodes) { // check that the invocation node really has the correct number of arguments if (invokeNode.Subtrees.Count() != selectedDefunBranch.NumberOfArguments) { throw new InvalidOperationException(); } var argumentBranch = invokeNode.GetSubtree(selectedArgumentSymbol.ArgumentIndex); var clonedArgumentBranch = (ISymbolicExpressionTreeNode)argumentBranch.Clone(); invokeNode.InsertSubtree(newArgumentIndex, clonedArgumentBranch); newlyAddedBranches.Add(clonedArgumentBranch); } invocationNodes = (from newlyAddedBranch in newlyAddedBranches from node in newlyAddedBranch.IterateNodesPrefix().OfType <InvokeFunctionTreeNode>() where node.Symbol.FunctionName == selectedDefunBranch.FunctionName where node.Subtrees.Count() == selectedDefunBranch.NumberOfArguments select node).ToList(); } // register the new argument symbol and increase the number of arguments of the ADF selectedDefunBranch.Grammar.AddSymbol(newArgSymbol); selectedDefunBranch.Grammar.SetSubtreeCount(newArgSymbol, 0, 0); // allow the duplicated argument as child of all other arguments where the orginal argument was allowed GrammarModifier.SetAllowedParentSymbols(selectedDefunBranch.Grammar, selectedArgumentSymbol, newArgSymbol); selectedDefunBranch.NumberOfArguments++; // increase the arity of the changed ADF in all branches that can use this ADF foreach (var subtree in symbolicExpressionTree.Root.Subtrees) { var matchingInvokeSymbol = (from symb in subtree.Grammar.Symbols.OfType <InvokeFunction>() where symb.FunctionName == selectedDefunBranch.FunctionName select symb).SingleOrDefault(); if (matchingInvokeSymbol != null) { subtree.Grammar.SetSubtreeCount(matchingInvokeSymbol, selectedDefunBranch.NumberOfArguments, selectedDefunBranch.NumberOfArguments); foreach (var symb in subtree.Grammar.Symbols) { if (symb is StartSymbol || symb is ProgramRootSymbol) { continue; } if (subtree.Grammar.IsAllowedChildSymbol(matchingInvokeSymbol, symb, selectedArgumentSymbol.ArgumentIndex)) { subtree.Grammar.AddAllowedChildSymbol(matchingInvokeSymbol, symb, newArgumentIndex); } } } } return(true); }
protected override ISymbolicExpressionTree Cross(IRandom random, ISymbolicExpressionTree parent0, ISymbolicExpressionTree parent1, ItemArray <PythonStatementSemantic> semantic0, ItemArray <PythonStatementSemantic> semantic1, T problemData, int maxTreeLength, int maxTreeDepth, double internalCrossoverPointProbability, out ItemArray <PythonStatementSemantic> newSemantics) { if (semantic0 == null || semantic1 == null || semantic0.Length == 0 || semantic1.Length == 0) { parent0 = SubtreeCrossover.Cross(random, parent0, parent1, internalCrossoverPointProbability, maxTreeLength, maxTreeDepth); newSemantics = null; TypeSelectedForSimilarityParameter.ActualValue = new StringValue("No Semantics; Random Crossover"); AddStatisticsNoCrossover(NoXoNoSemantics); return(parent0); } newSemantics = semantic0; var statementProductionNames = SemanticOperatorHelper.GetSemanticProductionNames(parent0.Root.Grammar); var variables = problemData.Variables.GetVariableNames().ToList(); string variableSettings = problemData.VariableSettings.Count == 0 ? String.Empty : String.Join(Environment.NewLine, problemData.VariableSettings.Select(x => x.Value)); int maximumSemanticTries = MaxComparesParameter.Value.Value; int semanticTries = 0; var saveOriginalSemantics = new List <JObject>(maximumSemanticTries); var saveReplaceSemantics = new List <JObject>(maximumSemanticTries); var possibleChildren = new List <Tuple <CutPoint, ISymbolicExpressionTreeNode> >(maximumSemanticTries); bool success = false; do { // select a random crossover point in the first parent CutPoint crossoverPoint0; SelectCrossoverPoint(random, parent0, internalCrossoverPointProbability, maxTreeLength, maxTreeDepth, out crossoverPoint0); int childLength = crossoverPoint0.Child != null?crossoverPoint0.Child.GetLength() : 0; // calculate the max length and depth that the inserted branch can have int maxInsertedBranchLength = Math.Max(0, maxTreeLength - (parent0.Length - childLength)); int maxInsertedBranchDepth = Math.Max(0, maxTreeDepth - parent0.Root.GetBranchLevel(crossoverPoint0.Parent)); List <ISymbolicExpressionTreeNode> allowedBranches = new List <ISymbolicExpressionTreeNode>(); parent1.Root.ForEachNodePostfix((n) => { if (n.GetLength() <= maxInsertedBranchLength && n.GetDepth() <= maxInsertedBranchDepth && crossoverPoint0.IsMatchingPointType(n)) { allowedBranches.Add(n); } }); // empty branch if (crossoverPoint0.IsMatchingPointType(null)) { allowedBranches.Add(null); } if (allowedBranches.Count != 0) { var selectedBranch = SelectRandomBranch(random, allowedBranches, internalCrossoverPointProbability); ISymbolicExpressionTreeNode statement = SemanticOperatorHelper.GetStatementNode(crossoverPoint0.Child, statementProductionNames); var statementPos0 = parent0.IterateNodesPrefix().ToList().IndexOf(statement); PythonStatementSemantic curSemantics = null; if (String.IsNullOrEmpty(variableSettings)) { curSemantics = semantic0.First(x => x.TreeNodePrefixPos == statementPos0); variableSettings = SemanticOperatorHelper.SemanticToPythonVariableSettings(curSemantics.Before, problemData.Variables.GetVariableTypes()); } var jsonOriginal = SemanticOperatorHelper.EvaluateStatementNode(statement, PyProcess, random, problemData, variables, variableSettings, Timeout); JObject jsonReplaced; if (statement == crossoverPoint0.Child) { // selectedBranch is also executable jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(selectedBranch, PyProcess, random, problemData, variables, variableSettings, Timeout); } else { crossoverPoint0.Parent.RemoveSubtree(crossoverPoint0.ChildIndex); var parent = selectedBranch.Parent; // save parent crossoverPoint0.Parent.InsertSubtree(crossoverPoint0.ChildIndex, selectedBranch); // this will affect node.Parent jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(statement, PyProcess, random, problemData, variables, variableSettings, Timeout); crossoverPoint0.Parent.RemoveSubtree(crossoverPoint0.ChildIndex); // removes intermediate parent from node selectedBranch.Parent = parent; // restore parent crossoverPoint0.Parent.InsertSubtree(crossoverPoint0.ChildIndex, crossoverPoint0.Child); // restore cutPoint } var exception = jsonOriginal["exception"] != null || jsonReplaced["exception"] != null; if (exception) { if (jsonOriginal["exception"] != null) { CrossoverExceptionsParameter.ActualValue.Add(new StringValue(jsonOriginal["exception"].ToString())); } if (jsonReplaced["exception"] != null) { CrossoverExceptionsParameter.ActualValue.Add(new StringValue(jsonReplaced["exception"].ToString())); } } if (curSemantics != null && !exception) { jsonOriginal = PythonSemanticComparer.ReplaceNotExecutedCases(jsonOriginal, curSemantics.Before, curSemantics.ExecutedCases); jsonReplaced = PythonSemanticComparer.ReplaceNotExecutedCases(jsonReplaced, curSemantics.Before, curSemantics.ExecutedCases); jsonOriginal = PythonSemanticComparer.ProduceDifference(jsonOriginal, curSemantics.Before); jsonReplaced = PythonSemanticComparer.ProduceDifference(jsonReplaced, curSemantics.Before); } if (!exception && SemanticMeasure(jsonOriginal, jsonReplaced)) { newSemantics = SemanticSwap(crossoverPoint0, selectedBranch, parent0, parent1, semantic0, semantic1); SemanticallyEquivalentCrossoverParameter.ActualValue = new IntValue(2); TypeSelectedForSimilarityParameter.ActualValue = new StringValue("First Semantic"); AddStatistics(semantic0, parent0, statement == crossoverPoint0.Child ? selectedBranch : statement, crossoverPoint0, jsonOriginal, selectedBranch, random, problemData, variables, variableSettings); // parent zero has been changed is now considered the child success = true; } else { saveOriginalSemantics.Add(jsonOriginal); saveReplaceSemantics.Add(jsonReplaced); possibleChildren.Add(new Tuple <CutPoint, ISymbolicExpressionTreeNode>(crossoverPoint0, selectedBranch)); } } semanticTries++; #region try second semantic comparison if (!success && semanticTries >= maximumSemanticTries) { for (int index = 0; index < saveOriginalSemantics.Count; index++) { var exception = saveOriginalSemantics[index]["exception"] == null && saveReplaceSemantics[index]["exception"] == null; if (!exception && AdditionalSemanticMeasure(saveOriginalSemantics[index], saveReplaceSemantics[index])) { var crossover = possibleChildren[index]; crossoverPoint0 = crossover.Item1; // Recreate jsonOriginal as it might have changed due to ReplaceNotExecutedCases and ProduceDifference var jsonOriginal = GenerateOriginalJson(crossoverPoint0, statementProductionNames, parent0, variableSettings, semantic0, problemData, random, variables); newSemantics = SemanticSwap(crossoverPoint0, crossover.Item2, parent0, parent1, semantic0, semantic1); var statement = SemanticOperatorHelper.GetStatementNode(crossover.Item2, statementProductionNames); SemanticallyEquivalentCrossoverParameter.ActualValue = new IntValue(2); TypeSelectedForSimilarityParameter.ActualValue = new StringValue("Second Semantic"); AddStatistics(semantic0, parent0, statement, crossoverPoint0, jsonOriginal, crossover.Item2, random, problemData, variables, variableSettings); // parent zero has been changed is now considered the child success = true; break; } } } #endregion } while (!success && semanticTries < maximumSemanticTries); if (!success) { // Last change. If any possible crossover was found, do a crossover with the first one if (saveOriginalSemantics.Any()) { var crossover = possibleChildren.First(); var crossoverPoint0 = crossover.Item1; // Recreate jsonOriginal as it might have changed due to ReplaceNotExecutedCases and ProduceDifference var jsonOriginal = GenerateOriginalJson(crossoverPoint0, statementProductionNames, parent0, variableSettings, semantic0, problemData, random, variables); newSemantics = SemanticSwap(crossoverPoint0, crossover.Item2, parent0, parent1, semantic0, semantic1); var statement = SemanticOperatorHelper.GetStatementNode(crossover.Item2, statementProductionNames); TypeSelectedForSimilarityParameter.ActualValue = new StringValue("Random Crossover; Reached Max Semantic Tries"); AddStatistics(semantic0, parent0, statement, crossoverPoint0, jsonOriginal, crossover.Item2, random, problemData, variables, variableSettings); // parent zero has been changed is now considered the child } else { AddStatisticsNoCrossover(NoXoNoAllowedBranch); } } saveOriginalSemantics.Clear(); saveReplaceSemantics.Clear(); possibleChildren.Clear(); NumberOfCrossoverTries = semanticTries; return(parent0); }
public override void ReplaceBranch(IRandom random, ISymbolicExpressionTree symbolicExpressionTree, ICFGPythonProblemData problemData, ItemArray <PythonStatementSemantic> semantics, PythonProcess pythonProcess, double timeout, int maxTreeLength, int maxTreeDepth, int maximumSemanticTries) { if (semantics == null || semantics.Length == 0) { ReplaceBranchManipulation.ReplaceRandomBranch(random, symbolicExpressionTree, maxTreeLength, maxTreeDepth); SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(NoSemantics); MutationTypeParameter.ActualValue = new IntValue(RandomMutation); return; } var allowedSymbols = new List <ISymbol>(); ISymbolicExpressionTreeNode parent; int childIndex; int maxLength; int maxDepth; // repeat until a fitting parent and child are found (MAX_TRIES times) int tries = 0; ISymbolicExpressionTreeNode child; do { #pragma warning disable 612, 618 parent = symbolicExpressionTree.Root.IterateNodesPrefix().Skip(1).Where(n => n.SubtreeCount > 0).SelectRandom(random); #pragma warning restore 612, 618 childIndex = random.Next(parent.SubtreeCount); child = parent.GetSubtree(childIndex); maxLength = maxTreeLength - symbolicExpressionTree.Length + child.GetLength(); maxDepth = maxTreeDepth - symbolicExpressionTree.Root.GetBranchLevel(child); allowedSymbols.Clear(); foreach (var symbol in parent.Grammar.GetAllowedChildSymbols(parent.Symbol, childIndex)) { // check basic properties that the new symbol must have if ((symbol.Name != child.Symbol.Name || symbol.MinimumArity > 0) && symbol.InitialFrequency > 0 && parent.Grammar.GetMinimumExpressionDepth(symbol) <= maxDepth && parent.Grammar.GetMinimumExpressionLength(symbol) <= maxLength) { allowedSymbols.Add(symbol); } } tries++; } while (tries < MAX_TRIES && allowedSymbols.Count == 0); NumberOfTriesParameter.ActualValue = new IntValue(tries); //normal tries. not semantic tries if (tries < MAX_TRIES) { var statementProductionNames = SemanticOperatorHelper.GetSemanticProductionNames(symbolicExpressionTree.Root.Grammar); var variables = problemData.Variables.GetVariableNames().ToList(); #region calculate original json output ISymbolicExpressionTreeNode statement = SemanticOperatorHelper.GetStatementNode(child, statementProductionNames); string variableSettings = problemData.VariableSettings.Count == 0 ? String.Empty : String.Join(Environment.NewLine, problemData.VariableSettings.Select(x => x.Value)); var statementPos0 = symbolicExpressionTree.IterateNodesPrefix().ToList().IndexOf(statement); if (String.IsNullOrEmpty(variableSettings)) { variableSettings = SemanticOperatorHelper.SemanticToPythonVariableSettings(semantics.First(x => x.TreeNodePrefixPos == statementPos0).Before, problemData.Variables.GetVariableTypes()); } var jsonOriginal = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout); #endregion var weights = allowedSymbols.Select(s => s.InitialFrequency).ToList(); #pragma warning disable 612, 618 var seedSymbol = allowedSymbols.SelectRandom(weights, random); #pragma warning restore 612, 618 // replace the old node with the new node var seedNode = seedSymbol.CreateTreeNode(); if (seedNode.HasLocalParameters) { seedNode.ResetLocalParameters(random); } parent.RemoveSubtree(childIndex); parent.InsertSubtree(childIndex, seedNode); ProbabilisticTreeCreator.PTC2(random, seedNode, maxLength, maxDepth); JObject jsonReplaced; if (child == statement) { // child is executable, so is the new child jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(seedNode, pythonProcess, random, problemData, variables, variableSettings, timeout); } else { jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout); } if (JToken.EqualityComparer.Equals(jsonOriginal, jsonReplaced)) { SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(Equvivalent); } else { SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(Different); } MutationTypeParameter.ActualValue = new IntValue(RandomMutation); } else { SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(NoMutation); MutationTypeParameter.ActualValue = new IntValue(NoMutation); } }
private void ComboBoxSelectedIndexChanged(object sender, EventArgs e) { string freeVariable = (string)comboBox.SelectedItem; IEnumerable<string> fixedVariables = comboBox.Items.OfType<string>() .Except(new string[] { freeVariable }); variableNodes.Clear(); clonedTree = (ISymbolicExpressionTree)Content.Model.SymbolicExpressionTree.Clone(); foreach (var varNode in clonedTree.IterateNodesPrefix().OfType<VariableTreeNode>()) { if (fixedVariables.Contains(varNode.VariableName)) { if (!variableNodes.ContainsKey(varNode.VariableName)) variableNodes.Add(varNode.VariableName, new List<ISymbolicExpressionTreeNode>()); int childIndex = varNode.Parent.IndexOfSubtree(varNode); var replacementNode = MakeConstantTreeNode(medianValues[varNode.VariableName]); var parent = varNode.Parent; parent.RemoveSubtree(childIndex); parent.InsertSubtree(childIndex, MakeProduct(replacementNode, varNode.Weight)); variableNodes[varNode.VariableName].Add(replacementNode); } } CreateSliders(fixedVariables); UpdateScatterPlot(); UpdateResponseSeries(); }
public static bool DuplicateSubroutine( IRandom random, ISymbolicExpressionTree symbolicExpressionTree, int maxFunctionDefinitions, int maxFunctionArguments) { var functionDefiningBranches = symbolicExpressionTree.IterateNodesPrefix().OfType<DefunTreeNode>().ToList(); if (!functionDefiningBranches.Any() || functionDefiningBranches.Count() == maxFunctionDefinitions) // no function defining branches to duplicate or already reached the max number of ADFs return false; string formatString = new StringBuilder().Append('0', (int)Math.Log10(maxFunctionDefinitions) + 1).ToString(); // >= 100 functions => ### var allowedFunctionNames = from index in Enumerable.Range(0, maxFunctionDefinitions) select "ADF" + index.ToString(formatString); var selectedBranch = functionDefiningBranches.SampleRandom(random); var duplicatedDefunBranch = (DefunTreeNode)selectedBranch.Clone(); string newFunctionName = allowedFunctionNames.Except(UsedFunctionNames(symbolicExpressionTree)).First(); duplicatedDefunBranch.FunctionName = newFunctionName; symbolicExpressionTree.Root.AddSubtree(duplicatedDefunBranch); duplicatedDefunBranch.SetGrammar((ISymbolicExpressionTreeGrammar)selectedBranch.Grammar.Clone()); var allowedChildSymbols = selectedBranch.Grammar.GetAllowedChildSymbols(selectedBranch.Symbol); foreach (var allowedChildSymbol in allowedChildSymbols) duplicatedDefunBranch.Grammar.AddAllowedChildSymbol(duplicatedDefunBranch.Symbol, allowedChildSymbol); var maxSubtrees = selectedBranch.Grammar.GetMaximumSubtreeCount(selectedBranch.Symbol); for (int i = 0; i < maxSubtrees; i++) { foreach (var allowedChildSymbol in selectedBranch.Grammar.GetAllowedChildSymbols(selectedBranch.Symbol, i)) duplicatedDefunBranch.Grammar.AddAllowedChildSymbol(duplicatedDefunBranch.Symbol, allowedChildSymbol); } // add an invoke symbol for each branch that is allowed to invoke the original function foreach (var subtree in symbolicExpressionTree.Root.Subtrees.OfType<SymbolicExpressionTreeTopLevelNode>()) { var matchingInvokeSymbol = (from symb in subtree.Grammar.Symbols.OfType<InvokeFunction>() where symb.FunctionName == selectedBranch.FunctionName select symb).SingleOrDefault(); if (matchingInvokeSymbol != null) { var invokeSymbol = new InvokeFunction(duplicatedDefunBranch.FunctionName); subtree.Grammar.AddSymbol(invokeSymbol); subtree.Grammar.SetSubtreeCount(invokeSymbol, duplicatedDefunBranch.NumberOfArguments, duplicatedDefunBranch.NumberOfArguments); foreach (ISymbol symbol in subtree.Grammar.Symbols) { if (subtree.Grammar.IsAllowedChildSymbol(symbol, matchingInvokeSymbol)) subtree.Grammar.AddAllowedChildSymbol(symbol, invokeSymbol); else { for (int i = 0; i < subtree.Grammar.GetMaximumSubtreeCount(symbol); i++) if (subtree.Grammar.IsAllowedChildSymbol(symbol, matchingInvokeSymbol, i)) subtree.Grammar.AddAllowedChildSymbol(symbol, invokeSymbol, i); } } foreach (ISymbol symbol in subtree.Grammar.GetAllowedChildSymbols(matchingInvokeSymbol)) if (symbol != invokeSymbol) //avoid duplicate entry invokesymbol / invokesymbol subtree.Grammar.AddAllowedChildSymbol(invokeSymbol, symbol); for (int i = 0; i < subtree.Grammar.GetMaximumSubtreeCount(matchingInvokeSymbol); i++) { foreach (ISymbol symbol in subtree.Grammar.GetAllowedChildSymbols(matchingInvokeSymbol, i).Except(subtree.Grammar.GetAllowedChildSymbols(matchingInvokeSymbol))) subtree.Grammar.AddAllowedChildSymbol(invokeSymbol, symbol, i); } } // in the current subtree: // for all invoke nodes of the original function replace the invoke of the original function with an invoke of the new function randomly var originalFunctionInvocations = from node in subtree.IterateNodesPrefix().OfType<InvokeFunctionTreeNode>() where node.Symbol.FunctionName == selectedBranch.FunctionName select node; foreach (var originalFunctionInvokeNode in originalFunctionInvocations) { var newInvokeSymbol = (from symb in subtree.Grammar.Symbols.OfType<InvokeFunction>() where symb.FunctionName == duplicatedDefunBranch.FunctionName select symb).Single(); // flip coin wether to replace with newly defined function if (random.NextDouble() < 0.5) { originalFunctionInvokeNode.Symbol = newInvokeSymbol; } } } return true; }
/// <summary> /// Swaps the child node of the cutpoint with the selected branch. Semantics of the child branch will be removed from semantics0. Semantics from semantics1 which belonged to the selected branch will be added to semantics0. /// </summary> /// <param name="crossoverPoint">Defines parent and child node from the parent0</param> /// <param name="selectedBranch">Branch to crossover from parent1</param> /// <param name="parent0">Parent0</param> /// <param name="parent1">Parent1</param> /// <param name="semantics0">Semantics of parent0</param> /// <param name="semantics1">Semantics of parent1</param> protected static ItemArray <PythonStatementSemantic> SemanticSwap(CutPoint crossoverPoint, ISymbolicExpressionTreeNode selectedBranch, ISymbolicExpressionTree parent0, ISymbolicExpressionTree parent1, ItemArray <PythonStatementSemantic> semantics0, ItemArray <PythonStatementSemantic> semantics1) { var allNodes0Prefix = parent0.IterateNodesPrefix().ToList(); Dictionary <ISymbolicExpressionTreeNode, PythonStatementSemantic> sem0ByNode = new Dictionary <ISymbolicExpressionTreeNode, PythonStatementSemantic>(); var sem0ByNodeIndex = semantics0.ToDictionary(x => x.TreeNodePrefixPos, y => y); for (int i = 0; i < allNodes0Prefix.Count; i++) { if (sem0ByNodeIndex.ContainsKey(i)) { sem0ByNode.Add(allNodes0Prefix[i], sem0ByNodeIndex[i]); } } var allNodes1Prefix = parent1.IterateNodesPrefix().ToList(); Dictionary <ISymbolicExpressionTreeNode, PythonStatementSemantic> sem1ByNode = new Dictionary <ISymbolicExpressionTreeNode, PythonStatementSemantic>(); var sem1ByNodeIndex = semantics1.ToDictionary(x => x.TreeNodePrefixPos, y => y); for (int i = 0; i < allNodes1Prefix.Count; i++) { if (sem1ByNodeIndex.ContainsKey(i)) { sem1ByNode.Add(allNodes1Prefix[i], sem1ByNodeIndex[i]); } } if (crossoverPoint.Child != null) { // manipulate the tree of parent0 in place // replace the branch in tree0 with the selected branch from tree1 crossoverPoint.Parent.RemoveSubtree(crossoverPoint.ChildIndex); if (selectedBranch != null) { crossoverPoint.Parent.InsertSubtree(crossoverPoint.ChildIndex, selectedBranch); } } else { // child is null (additional child should be added under the parent) if (selectedBranch != null) { crossoverPoint.Parent.AddSubtree(selectedBranch); } } List <PythonStatementSemantic> newSemantics = new List <PythonStatementSemantic>(); var newNodes0Prefix = parent0.IterateNodesPrefix().ToList(); for (int i = 0; i < newNodes0Prefix.Count; i++) { PythonStatementSemantic sem = null; if (sem0ByNode.ContainsKey(newNodes0Prefix[i])) { sem = sem0ByNode[newNodes0Prefix[i]]; sem.TreeNodePrefixPos = i; } else if (sem1ByNode.ContainsKey(newNodes0Prefix[i])) { sem = sem1ByNode[newNodes0Prefix[i]]; sem.TreeNodePrefixPos = i; } if (sem != null) { newSemantics.Add(sem); } } return(new ItemArray <PythonStatementSemantic>(newSemantics)); }
private static void DeletionByRandomRegeneration(IRandom random, ISymbolicExpressionTree symbolicExpressionTree, DefunTreeNode selectedDefunBranch) { // find first invocation and replace it with a randomly generated tree // can't find all invocations in one step because once we replaced a top level invocation // the invocations below it are removed already var invocationCutPoint = (from node in symbolicExpressionTree.IterateNodesPrefix() from subtree in node.Subtrees.OfType<InvokeFunctionTreeNode>() where subtree.Symbol.FunctionName == selectedDefunBranch.FunctionName select new CutPoint(node, subtree)).FirstOrDefault(); while (invocationCutPoint != null) { // deletion by random regeneration ISymbolicExpressionTreeNode replacementTree = null; var allowedSymbolsList = invocationCutPoint.Parent.Grammar.GetAllowedChildSymbols(invocationCutPoint.Parent.Symbol, invocationCutPoint.ChildIndex).ToList(); var weights = allowedSymbolsList.Select(s => s.InitialFrequency); #pragma warning disable 612, 618 var selectedSymbol = allowedSymbolsList.SelectRandom(weights, random); #pragma warning restore 612, 618 int minPossibleLength = invocationCutPoint.Parent.Grammar.GetMinimumExpressionLength(selectedSymbol); int maxLength = Math.Max(minPossibleLength, invocationCutPoint.Child.GetLength()); int minPossibleDepth = invocationCutPoint.Parent.Grammar.GetMinimumExpressionDepth(selectedSymbol); int maxDepth = Math.Max(minPossibleDepth, invocationCutPoint.Child.GetDepth()); replacementTree = selectedSymbol.CreateTreeNode(); if (replacementTree.HasLocalParameters) replacementTree.ResetLocalParameters(random); invocationCutPoint.Parent.RemoveSubtree(invocationCutPoint.ChildIndex); invocationCutPoint.Parent.InsertSubtree(invocationCutPoint.ChildIndex, replacementTree); ProbabilisticTreeCreator.PTC2(random, replacementTree, maxLength, maxDepth); invocationCutPoint = (from node in symbolicExpressionTree.IterateNodesPrefix() from subtree in node.Subtrees.OfType<InvokeFunctionTreeNode>() where subtree.Symbol.FunctionName == selectedDefunBranch.FunctionName select new CutPoint(node, subtree)).FirstOrDefault(); } }
// ToDo: Remove workaround; use executed lines in PythonStatementSemantic public Tuple <IEnumerable <bool>, IEnumerable <double>, double, string, List <PythonStatementSemantic> > EvaluateAndTraceProgram(PythonProcess pythonProcess, string program, string input, string output, IEnumerable <int> indices, string header, string footer, ISymbolicExpressionTree tree, double timeout = 1) { string traceProgram = traceCodeWithVariables + program; traceProgram += traceCodeWithVariables == String.Empty ? String.Empty : traceTableReduceEntries; EvaluationScript es = pythonProcess.CreateEvaluationScript(traceProgram, input, output, timeout); es.Variables.Add("traceTable"); es.Variables.Add("traceTableBefore"); es.Variables.Add("executedLines"); JObject json = pythonProcess.SendAndEvaluateProgram(es); var baseResult = pythonProcess.GetVariablesFromJson(json, indices.Count()); if (json["traceTable"] == null) { return(new Tuple <IEnumerable <bool>, IEnumerable <double>, double, string, List <PythonStatementSemantic> >(baseResult.Item1, baseResult.Item2, baseResult.Item3, baseResult.Item4, new List <PythonStatementSemantic>())); } var traceTable = json["traceTable"].ToObject <IDictionary <int, IDictionary <string, IList> > >(); var traceTableBefore = json["traceTableBefore"].ToObject <IDictionary <int, IDictionary <string, IList> > >(); var executedLines = json["executedLines"].ToObject <IDictionary <int, List <int> > >(); List <PythonStatementSemantic> semantics = new List <PythonStatementSemantic>(); ISymbolicExpressionTreeNode root = tree.Root; var statementProductionNames = SemanticOperatorHelper.GetSemanticProductionNames(root.Grammar); // calculate the correct line the semantic evaluation starts from var code = CFGSymbolicExpressionTreeStringFormatter.StaticFormat(tree); int curline = es.Script.Count(c => c == '\n') - code.Count(c => c == '\n') - footer.Count(c => c == '\n') - traceTableReduceEntries.Count(c => c == '\n'); var symbolToLineDict = FindStatementSymbolsInTree(root, statementProductionNames, ref curline); #region workaround: empty line problem with while, can't fix, otherwise FindStatementSymbolsInTree won't work string[] sciptLines = es.Script.Split('\n'); var beginLineNumbers = symbolToLineDict.Values.Select(x => x[0]).Distinct().ToList(); var moveLines = new Dictionary <int, int>(beginLineNumbers.Count); foreach (var l in beginLineNumbers) { // decrease by one, as sciptLines is an array and start from zero, where lineNumbers started counting from 1 if (String.IsNullOrWhiteSpace(sciptLines[l - 1]) || sciptLines[l - 1].TrimStart().StartsWith("#")) { // empty line or comment var i = l + 1; while (i - 1 < sciptLines.Length && (String.IsNullOrWhiteSpace(sciptLines[i - 1]) || sciptLines[i - 1].TrimStart().StartsWith("#"))) { i++; } moveLines.Add(l, i); } else { moveLines.Add(l, l); } } foreach (var symbolLine in symbolToLineDict) { symbolLine.Value[0] = moveLines[symbolLine.Value[0]]; } #endregion #region fix Before line for <predefined> to have all variables initialised // not a great way to do it, but the python interpreter does not stop at e.g. 'while False:' int newMinBefore = traceTableBefore.OrderBy(x => x.Key).First(x => x.Value.ContainsKey("res0") && !x.Value["res0"].Contains(null)).Key; // first line that changes res0 foreach (var symbolToLine in symbolToLineDict) { symbolToLine.Value.Add(symbolToLine.Value[0]); // original beginning at [2], which is needed for executed lines if (symbolToLine.Value[0] < newMinBefore) { symbolToLine.Value[0] = newMinBefore; } } #endregion var prefixTreeNodes = tree.IterateNodesPrefix().ToList(); foreach (var symbolLine in symbolToLineDict) { // Before Dictionary <string, IList> before = new Dictionary <string, IList>(); foreach (var traceChange in traceTableBefore.Where(x => x.Key <= symbolLine.Value[0]).OrderByDescending(x => x.Key)) { foreach (var variableChange in traceChange.Value) { if (!before.ContainsKey(variableChange.Key)) { before.Add(variableChange.Key, variableChange.Value); } } } // After Dictionary <string, IList> after = new Dictionary <string, IList>(); foreach (var traceChange in traceTable.Where(x => x.Key >= symbolLine.Value[0] && x.Key <= symbolLine.Value[1]).OrderByDescending(x => x.Key)) { foreach (var variableChange in traceChange.Value) { if (!after.ContainsKey(variableChange.Key)) { after.Add(variableChange.Key, variableChange.Value); } } } // clean after with before foreach (var key in after.Keys.ToList()) { if (PythonSemanticComparer.CompareSequence(after[key], before[key])) { after.Remove(key); } } // add semantics var executedLinesWithinStatement = executedLines.Where(x => x.Key >= symbolLine.Value[2] && x.Key <= symbolLine.Value[1]); semantics.Add(new PythonStatementSemantic() { TreeNodePrefixPos = prefixTreeNodes.IndexOf(symbolLine.Key), ExecutedCases = executedLinesWithinStatement.Any() ? executedLinesWithinStatement.OrderByDescending(x => x.Value.Count).First().Value : new List <int>(), Before = before, After = after, }); } return(new Tuple <IEnumerable <bool>, IEnumerable <double>, double, string, List <PythonStatementSemantic> >(baseResult.Item1, baseResult.Item2, baseResult.Item3, baseResult.Item4, semantics)); }
protected virtual ISymbolicExpressionTree Cross(IRandom random, ISymbolicExpressionTree parent0, ISymbolicExpressionTree parent1, ItemArray <PythonStatementSemantic> semantic0, ItemArray <PythonStatementSemantic> semantic1, ICFGPythonProblemData problemData, int maxTreeLength, int maxTreeDepth, double internalCrossoverPointProbability, out ItemArray <PythonStatementSemantic> newSemantics) { if (semantic0 == null || semantic1 == null || semantic0.Length == 0 || semantic1.Length == 0) { parent0 = SubtreeCrossover.Cross(random, parent0, parent1, internalCrossoverPointProbability, maxTreeLength, maxTreeDepth); newSemantics = null; return(parent0); } newSemantics = semantic0; // select a random crossover point in the first parent CutPoint crossoverPoint0; SelectCrossoverPoint(random, parent0, internalCrossoverPointProbability, maxTreeLength, maxTreeDepth, out crossoverPoint0); int childLength = crossoverPoint0.Child != null?crossoverPoint0.Child.GetLength() : 0; // calculate the max length and depth that the inserted branch can have int maxInsertedBranchLength = maxTreeLength - (parent0.Length - childLength); int maxInsertedBranchDepth = maxTreeDepth - parent0.Root.GetBranchLevel(crossoverPoint0.Child); List <ISymbolicExpressionTreeNode> allowedBranches = new List <ISymbolicExpressionTreeNode>(); parent1.Root.ForEachNodePostfix((n) => { if (n.GetLength() <= maxInsertedBranchLength && n.GetDepth() <= maxInsertedBranchDepth && crossoverPoint0.IsMatchingPointType(n)) { allowedBranches.Add(n); } }); // empty branch if (crossoverPoint0.IsMatchingPointType(null)) { allowedBranches.Add(null); } if (allowedBranches.Count == 0) { return(parent0); } // select MaxCompares random crossover points // Use set to avoid having the same node multiple times HashSet <ISymbolicExpressionTreeNode> compBranches; if (allowedBranches.Count < MaxComparesParameter.Value.Value) { compBranches = new HashSet <ISymbolicExpressionTreeNode>(allowedBranches); } else { compBranches = new HashSet <ISymbolicExpressionTreeNode>(); for (int i = 0; i < MaxComparesParameter.Value.Value; i++) { var possibleBranch = SelectRandomBranch(random, allowedBranches, internalCrossoverPointProbability); allowedBranches.Remove(possibleBranch); compBranches.Add(possibleBranch); } } var statementProductionNames = SemanticOperatorHelper.GetSemanticProductionNames(crossoverPoint0.Parent.Grammar); // find first node that can be used for evaluation in parent0 ISymbolicExpressionTreeNode statement = SemanticOperatorHelper.GetStatementNode(crossoverPoint0.Child, statementProductionNames); if (statement == null) { newSemantics = SemanticSwap(crossoverPoint0, compBranches.SampleRandom(random), parent0, parent1, semantic0, semantic1); return(parent0); } var statementPos0 = parent0.IterateNodesPrefix().ToList().IndexOf(statement); string variableSettings; if (problemData.VariableSettings.Count == 0) { variableSettings = SemanticOperatorHelper.SemanticToPythonVariableSettings(semantic0.First(x => x.TreeNodePrefixPos == statementPos0).Before, problemData.Variables.GetVariableTypes()); } else { variableSettings = String.Join(Environment.NewLine, problemData.VariableSettings.Select(x => x.Value)); } var variables = problemData.Variables.GetVariableNames().ToList(); var json0 = SemanticOperatorHelper.EvaluateStatementNode(statement, PyProcess, random, problemData, variables, variableSettings, Timeout); ISymbolicExpressionTreeNode selectedBranch; if (!String.IsNullOrWhiteSpace((string)json0["exception"])) { selectedBranch = compBranches.SampleRandom(random); } else { selectedBranch = SelectBranch(statement, crossoverPoint0, compBranches, random, variables, variableSettings, json0, problemData); } // perform the actual swap newSemantics = SemanticSwap(crossoverPoint0, selectedBranch, parent0, parent1, semantic0, semantic1); return(parent0); }
private static IEnumerable <string> UsedFunctionNames(ISymbolicExpressionTree symbolicExpressionTree) { return(from node in symbolicExpressionTree.IterateNodesPrefix() where node.Symbol is Defun select((DefunTreeNode)node).FunctionName); }
public override void ReplaceBranch(IRandom random, ISymbolicExpressionTree symbolicExpressionTree, ICFGPythonProblemData problemData, ItemArray <PythonStatementSemantic> semantics, PythonProcess pythonProcess, double timeout, int maxTreeLength, int maxTreeDepth, int maximumSemanticTries) { if (semantics == null || semantics.Length == 0) { ReplaceBranchManipulation.ReplaceRandomBranch(random, symbolicExpressionTree, maxTreeLength, maxTreeDepth); SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(NoSemantics); MutationTypeParameter.ActualValue = new IntValue(RandomMutation); return; } var statementProductionNames = SemanticOperatorHelper.GetSemanticProductionNames(symbolicExpressionTree.Root.Grammar); var variables = problemData.Variables.GetVariableNames().ToList(); string variableSettings = problemData.VariableSettings.Count == 0 ? String.Empty : String.Join(Environment.NewLine, problemData.VariableSettings.Select(x => x.Value)); var allowedSymbols = new List <ISymbol>(); // repeat until a fitting parent and child are found (MAX_TRIES times) int tries = 0; int semanticTries = 0; List <JObject> saveOriginalSemantics = null; List <JObject> saveReplaceSemantics = null; List <Tuple <ISymbolicExpressionTreeNode, ISymbolicExpressionTreeNode, int> > possibleChildren = null; // Item1 = parent, Item2 = seedNode, Item3 = childIndex if (UsesAdditionalSemanticMeasure()) { saveOriginalSemantics = new List <JObject>(semanticTries); saveReplaceSemantics = new List <JObject>(semanticTries); possibleChildren = new List <Tuple <ISymbolicExpressionTreeNode, ISymbolicExpressionTreeNode, int> >(semanticTries); } bool success = false; do { #region find mutation point #pragma warning disable 612, 618 ISymbolicExpressionTreeNode parent = symbolicExpressionTree.Root.IterateNodesPrefix().Skip(1).Where(n => n.SubtreeCount > 0).SelectRandom(random); #pragma warning restore 612, 618 int childIndex = random.Next(parent.SubtreeCount); var child = parent.GetSubtree(childIndex); int maxLength = maxTreeLength - symbolicExpressionTree.Length + child.GetLength(); int maxDepth = maxTreeDepth - symbolicExpressionTree.Root.GetBranchLevel(child); allowedSymbols.Clear(); foreach (var symbol in parent.Grammar.GetAllowedChildSymbols(parent.Symbol, childIndex)) { // check basic properties that the new symbol must have if ((symbol.Name != child.Symbol.Name || symbol.MinimumArity > 0) && symbol.InitialFrequency > 0 && parent.Grammar.GetMinimumExpressionDepth(symbol) <= maxDepth && parent.Grammar.GetMinimumExpressionLength(symbol) <= maxLength) { allowedSymbols.Add(symbol); } } #endregion #region check for semantic difference with a new random tree if (allowedSymbols.Count > 0) { if (semanticTries <= maximumSemanticTries) { // do semantic mutation #region calculate original json output ISymbolicExpressionTreeNode statement = SemanticOperatorHelper.GetStatementNode(child, statementProductionNames); var statementPos0 = symbolicExpressionTree.IterateNodesPrefix().ToList().IndexOf(statement); PythonStatementSemantic curSemantics = null; if (String.IsNullOrEmpty(variableSettings)) { curSemantics = semantics.First(x => x.TreeNodePrefixPos == statementPos0); variableSettings = SemanticOperatorHelper.SemanticToPythonVariableSettings(curSemantics.Before, problemData.Variables.GetVariableTypes()); } var jsonOriginal = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout); // compare jsonOriginal to semantic after! Maybe avoid additional evaluation. #endregion var seedNode = GenerateAndInsertNewSubtree(random, parent, allowedSymbols, childIndex, maxLength, maxDepth); #region calculate new json output JObject jsonReplaced; if (child == statement) { // child is executable, so is the new child jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(seedNode, pythonProcess, random, problemData, variables, variableSettings, timeout); } else { jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout); } var exception = jsonOriginal["exception"] != null || jsonReplaced["exception"] != null; if (exception) { if (jsonOriginal["exception"] != null) { MutationExceptionsParameter.ActualValue.Add(new StringValue(jsonOriginal["exception"].ToString())); } if (jsonReplaced["exception"] != null) { MutationExceptionsParameter.ActualValue.Add(new StringValue(jsonReplaced["exception"].ToString())); } } if (curSemantics != null && !exception) { jsonOriginal = PythonSemanticComparer.ReplaceNotExecutedCases(jsonOriginal, curSemantics.Before, curSemantics.ExecutedCases); jsonReplaced = PythonSemanticComparer.ReplaceNotExecutedCases(jsonReplaced, curSemantics.Before, curSemantics.ExecutedCases); jsonOriginal = PythonSemanticComparer.ProduceDifference(jsonOriginal, curSemantics.Before); jsonReplaced = PythonSemanticComparer.ProduceDifference(jsonReplaced, curSemantics.Before); } if (!exception && SemanticMeasure(jsonOriginal, jsonReplaced)) { success = true; SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(Different); TypeSelectedForSimilarityParameter.ActualValue = new StringValue("First Semantic"); MutationTypeParameter.ActualValue = new IntValue(SemanticMutation); } else { // undo mutation parent.RemoveSubtree(childIndex); parent.InsertSubtree(childIndex, child); allowedSymbols.Clear(); if (!exception && UsesAdditionalSemanticMeasure()) { saveOriginalSemantics.Add(jsonOriginal); saveReplaceSemantics.Add(jsonReplaced); possibleChildren.Add(new Tuple <ISymbolicExpressionTreeNode, ISymbolicExpressionTreeNode, int>(parent, seedNode, childIndex)); } } if (problemData.VariableSettings.Count == 0) { // reset variableSettings variableSettings = String.Empty; } semanticTries++; #endregion #region try second semantic comparison if (!success && semanticTries >= maximumSemanticTries && UsesAdditionalSemanticMeasure()) { for (int index = 0; index < saveOriginalSemantics.Count; index++) { if (AdditionalSemanticMeasure(saveOriginalSemantics[index], saveReplaceSemantics[index])) { var mutation = possibleChildren[index]; mutation.Item1.RemoveSubtree(mutation.Item3); mutation.Item1.InsertSubtree(mutation.Item3, mutation.Item2); success = true; SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(Different); TypeSelectedForSimilarityParameter.ActualValue = new StringValue("Second Semantic"); MutationTypeParameter.ActualValue = new IntValue(SemanticMutation); break; } } } #endregion } else { // do random mutation #region calculate original json output ISymbolicExpressionTreeNode statement = SemanticOperatorHelper.GetStatementNode(child, statementProductionNames); var statementPos0 = symbolicExpressionTree.IterateNodesPrefix().ToList().IndexOf(statement); if (String.IsNullOrEmpty(variableSettings)) { variableSettings = SemanticOperatorHelper.SemanticToPythonVariableSettings(semantics.First(x => x.TreeNodePrefixPos == statementPos0).Before, problemData.Variables.GetVariableTypes()); } var jsonOriginal = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout); #endregion var seedNode = GenerateAndInsertNewSubtree(random, parent, allowedSymbols, childIndex, maxLength, maxDepth); JObject jsonReplaced; if (child == statement) { // child is executable, so is the new child jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(seedNode, pythonProcess, random, problemData, variables, variableSettings, timeout); } else { jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout); } if (JToken.EqualityComparer.Equals(jsonOriginal, jsonReplaced)) { SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(Equvivalent); } else { SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(Different); } TypeSelectedForSimilarityParameter.ActualValue = new StringValue("Random Crossover; Reached Max Semantic Tries"); MutationTypeParameter.ActualValue = new IntValue(RandomMutation); success = true; } } #endregion tries++; } while (tries < MAX_TRIES && !success); NumberOfTriesParameter.ActualValue = new IntValue(semanticTries); if (SemanticallyEquivalentMutationParameter.ActualValue == null) { SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(NoMutation); TypeSelectedForSimilarityParameter.ActualValue = new StringValue("No mutation"); MutationTypeParameter.ActualValue = new IntValue(NoMutation); } }
public static bool DuplicateSubroutine( IRandom random, ISymbolicExpressionTree symbolicExpressionTree, int maxFunctionDefinitions, int maxFunctionArguments) { var functionDefiningBranches = symbolicExpressionTree.IterateNodesPrefix().OfType <DefunTreeNode>().ToList(); if (!functionDefiningBranches.Any() || functionDefiningBranches.Count() == maxFunctionDefinitions) { // no function defining branches to duplicate or already reached the max number of ADFs return(false); } string formatString = new StringBuilder().Append('0', (int)Math.Log10(maxFunctionDefinitions) + 1).ToString(); // >= 100 functions => ### var allowedFunctionNames = from index in Enumerable.Range(0, maxFunctionDefinitions) select "ADF" + index.ToString(formatString); var selectedBranch = functionDefiningBranches.SampleRandom(random); var duplicatedDefunBranch = (DefunTreeNode)selectedBranch.Clone(); string newFunctionName = allowedFunctionNames.Except(UsedFunctionNames(symbolicExpressionTree)).First(); duplicatedDefunBranch.FunctionName = newFunctionName; symbolicExpressionTree.Root.AddSubtree(duplicatedDefunBranch); duplicatedDefunBranch.SetGrammar((ISymbolicExpressionTreeGrammar)selectedBranch.Grammar.Clone()); var allowedChildSymbols = selectedBranch.Grammar.GetAllowedChildSymbols(selectedBranch.Symbol); foreach (var allowedChildSymbol in allowedChildSymbols) { duplicatedDefunBranch.Grammar.AddAllowedChildSymbol(duplicatedDefunBranch.Symbol, allowedChildSymbol); } var maxSubtrees = selectedBranch.Grammar.GetMaximumSubtreeCount(selectedBranch.Symbol); for (int i = 0; i < maxSubtrees; i++) { foreach (var allowedChildSymbol in selectedBranch.Grammar.GetAllowedChildSymbols(selectedBranch.Symbol, i)) { duplicatedDefunBranch.Grammar.AddAllowedChildSymbol(duplicatedDefunBranch.Symbol, allowedChildSymbol); } } // add an invoke symbol for each branch that is allowed to invoke the original function foreach (var subtree in symbolicExpressionTree.Root.Subtrees.OfType <SymbolicExpressionTreeTopLevelNode>()) { var matchingInvokeSymbol = (from symb in subtree.Grammar.Symbols.OfType <InvokeFunction>() where symb.FunctionName == selectedBranch.FunctionName select symb).SingleOrDefault(); if (matchingInvokeSymbol != null) { var invokeSymbol = new InvokeFunction(duplicatedDefunBranch.FunctionName); subtree.Grammar.AddSymbol(invokeSymbol); subtree.Grammar.SetSubtreeCount(invokeSymbol, duplicatedDefunBranch.NumberOfArguments, duplicatedDefunBranch.NumberOfArguments); foreach (ISymbol symbol in subtree.Grammar.Symbols) { if (subtree.Grammar.IsAllowedChildSymbol(symbol, matchingInvokeSymbol)) { subtree.Grammar.AddAllowedChildSymbol(symbol, invokeSymbol); } else { for (int i = 0; i < subtree.Grammar.GetMaximumSubtreeCount(symbol); i++) { if (subtree.Grammar.IsAllowedChildSymbol(symbol, matchingInvokeSymbol, i)) { subtree.Grammar.AddAllowedChildSymbol(symbol, invokeSymbol, i); } } } } foreach (ISymbol symbol in subtree.Grammar.GetAllowedChildSymbols(matchingInvokeSymbol)) { if (symbol != invokeSymbol) //avoid duplicate entry invokesymbol / invokesymbol { subtree.Grammar.AddAllowedChildSymbol(invokeSymbol, symbol); } } for (int i = 0; i < subtree.Grammar.GetMaximumSubtreeCount(matchingInvokeSymbol); i++) { foreach (ISymbol symbol in subtree.Grammar.GetAllowedChildSymbols(matchingInvokeSymbol, i).Except(subtree.Grammar.GetAllowedChildSymbols(matchingInvokeSymbol))) { subtree.Grammar.AddAllowedChildSymbol(invokeSymbol, symbol, i); } } } // in the current subtree: // for all invoke nodes of the original function replace the invoke of the original function with an invoke of the new function randomly var originalFunctionInvocations = from node in subtree.IterateNodesPrefix().OfType <InvokeFunctionTreeNode>() where node.Symbol.FunctionName == selectedBranch.FunctionName select node; foreach (var originalFunctionInvokeNode in originalFunctionInvocations) { var newInvokeSymbol = (from symb in subtree.Grammar.Symbols.OfType <InvokeFunction>() where symb.FunctionName == duplicatedDefunBranch.FunctionName select symb).Single(); // flip coin wether to replace with newly defined function if (random.NextDouble() < 0.5) { originalFunctionInvokeNode.Symbol = newInvokeSymbol; } } } return(true); }
/// <summary> /// 1. find a mutation point /// 2. generate new random tree /// 3. calculate semantic of old and new subtree (or of the closest parent) /// 4. do mutation if semantically different /// 5. retry until a certain number of tries is reached /// /// if no mutation has happened, do random mutation /// </summary> public static void ReplaceSemanticallyDifferentBranch(IRandom random, ISymbolicExpressionTree symbolicExpressionTree, ICFGPythonProblemData problemData, ItemArray <PythonStatementSemantic> semantics, PythonProcess pythonProcess, double timeout, int maxTreeLength, int maxTreeDepth, int maximumSemanticTries) { if (semantics == null || semantics.Length == 0) { ReplaceBranchManipulation.ReplaceRandomBranch(random, symbolicExpressionTree, maxTreeLength, maxTreeDepth); return; } var statementProductionNames = SemanticOperatorHelper.GetSemanticProductionNames(symbolicExpressionTree.Root.Grammar); var variables = problemData.Variables.GetVariableNames().ToList(); string variableSettings = problemData.VariableSettings.Count == 0 ? String.Empty : String.Join(Environment.NewLine, problemData.VariableSettings.Select(x => x.Value)); var allowedSymbols = new List <ISymbol>(); // repeat until a fitting parent and child are found (MAX_TRIES times) int tries = 0; int semanticTries = 0; do { #region find mutation point #pragma warning disable 612, 618 ISymbolicExpressionTreeNode parent = symbolicExpressionTree.Root.IterateNodesPrefix().Skip(1).Where(n => n.SubtreeCount > 0).SelectRandom(random); #pragma warning restore 612, 618 int childIndex = random.Next(parent.SubtreeCount); var child = parent.GetSubtree(childIndex); int maxLength = maxTreeLength - symbolicExpressionTree.Length + child.GetLength(); int maxDepth = maxTreeDepth - symbolicExpressionTree.Root.GetBranchLevel(child); allowedSymbols.Clear(); foreach (var symbol in parent.Grammar.GetAllowedChildSymbols(parent.Symbol, childIndex)) { // check basic properties that the new symbol must have if ((symbol.Name != child.Symbol.Name || symbol.MinimumArity > 0) && symbol.InitialFrequency > 0 && parent.Grammar.GetMinimumExpressionDepth(symbol) <= maxDepth && parent.Grammar.GetMinimumExpressionLength(symbol) <= maxLength) { allowedSymbols.Add(symbol); } } #endregion #region check for semantic difference with a new random tree if (allowedSymbols.Count > 0) { if (semanticTries <= maximumSemanticTries) { // do semantic mutation #region calculate original json output ISymbolicExpressionTreeNode statement = SemanticOperatorHelper.GetStatementNode(child, statementProductionNames); var statementPos0 = symbolicExpressionTree.IterateNodesPrefix().ToList().IndexOf(statement); if (String.IsNullOrEmpty(variableSettings)) { variableSettings = SemanticOperatorHelper.SemanticToPythonVariableSettings(semantics.First(x => x.TreeNodePrefixPos == statementPos0).Before, problemData.Variables.GetVariableTypes()); } var jsonOriginal = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout); #endregion var seedNode = GenerateAndInsertNewSubtree(random, parent, allowedSymbols, childIndex, maxLength, maxDepth); #region calculate new json output JObject jsonReplaced; if (child == statement) { // child is executable, so is the new child jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(seedNode, pythonProcess, random, problemData, variables, variableSettings, timeout); } else { jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout); } if (JToken.EqualityComparer.Equals(jsonOriginal, jsonReplaced)) { // semantically equivalent. undo mutation parent.RemoveSubtree(childIndex); parent.InsertSubtree(childIndex, child); allowedSymbols.Clear(); } else { Console.WriteLine("never happens?"); } if (problemData.VariableSettings.Count == 0) { // reset variableSettings variableSettings = String.Empty; } semanticTries++; #endregion } else { // do random mutation GenerateAndInsertNewSubtree(random, parent, allowedSymbols, childIndex, maxLength, maxDepth); } } #endregion tries++; } while (tries < MAX_TRIES && allowedSymbols.Count == 0 && semanticTries <= maximumSemanticTries); }
public Tuple <IEnumerable <bool>, IEnumerable <double>, double, string, List <PythonStatementSemantic> > EvaluateAndTraceProgram(PythonProcess pythonProcess, string program, string input, string output, IEnumerable <int> indices, string header, string footer, ISymbolicExpressionTree tree, double timeout = 1) { string traceProgram = traceCodeWithVariables + program; traceProgram += traceCodeWithVariables == String.Empty ? String.Empty : traceTableReduceEntries; EvaluationScript es = pythonProcess.CreateEvaluationScript(traceProgram, input, output, timeout); es.Variables.Add("traceTable"); es.Variables.Add("traceTableBefore"); es.Variables.Add("executedLines"); JObject json = pythonProcess.SendAndEvaluateProgram(es); var baseResult = pythonProcess.GetVariablesFromJson(json, indices.Count()); if (json["traceTable"] == null || json["traceTableBefore"] == null) { return(new Tuple <IEnumerable <bool>, IEnumerable <double>, double, string, List <PythonStatementSemantic> >(baseResult.Item1, baseResult.Item2, baseResult.Item3, baseResult.Item4, new List <PythonStatementSemantic>())); } var traceTable = JsonConvert.DeserializeObject <IDictionary <int, IDictionary <string, IList> > >(json["traceTable"].ToString()); var traceTableBefore = JsonConvert.DeserializeObject <IDictionary <int, IDictionary <string, IList> > >(json["traceTableBefore"].ToString()); var executedLinesNew = JsonConvert.DeserializeObject <IDictionary <int, List <int> > >(json["executedLines"].ToString()); var executedLines = executedLinesNew.Keys.ToList(); executedLines.Sort(); IList <int> traceChanges = traceTable.Keys.OrderBy(x => x).ToList(); IList <int> traceBeforeChanges = traceTableBefore.Keys.OrderBy(x => x).ToList(); List <PythonStatementSemantic> semantics = new List <PythonStatementSemantic>(); ISymbolicExpressionTreeNode root = tree.Root; var statementProductionNames = SemanticOperatorHelper.GetSemanticProductionNames(root.Grammar); // calculate the correct line the semantic evaluation starts from var code = CFGSymbolicExpressionTreeStringFormatter.StaticFormat(tree); int curline = es.Script.Count(c => c == '\n') - code.Count(c => c == '\n') - footer.Count(c => c == '\n') - traceTableReduceEntries.Count(c => c == '\n'); var symbolToLineDict = FindStatementSymbolsInTree(root, statementProductionNames, ref curline); var prefixTreeNodes = tree.IterateNodesPrefix().ToList(); foreach (var symbolLine in symbolToLineDict) { // Before Dictionary <string, IList> before = new Dictionary <string, IList>(); foreach (var traceChange in traceTableBefore.Where(x => x.Key <= symbolLine.Value[0]).OrderByDescending(x => x.Key)) { foreach (var variableChange in traceChange.Value) { if (!before.ContainsKey(variableChange.Key)) { before.Add(variableChange.Key, variableChange.Value); } } } // After Dictionary <string, IList> after = new Dictionary <string, IList>(); foreach (var traceChange in traceTable.Where(x => x.Key >= symbolLine.Value[0] && x.Key <= symbolLine.Value[1]).OrderByDescending(x => x.Key)) { foreach (var variableChange in traceChange.Value) { if (!after.ContainsKey(variableChange.Key)) { after.Add(variableChange.Key, variableChange.Value); } } } // clean after with before foreach (var key in after.Keys.ToList()) { if (CompareSequence(after[key], before[key])) { after.Remove(key); } } semantics.Add(new PythonStatementSemantic() { TreeNodePrefixPos = prefixTreeNodes.IndexOf(symbolLine.Key), Before = before, After = after, }); } return(new Tuple <IEnumerable <bool>, IEnumerable <double>, double, string, List <PythonStatementSemantic> >(baseResult.Item1, baseResult.Item2, baseResult.Item3, baseResult.Item4, semantics)); }