public void Reevaluate(ICFGPythonProblemData problemData, double timeout, PythonProcess pythonProcess) {
foreach (var key in ResultKeys) {
if (this.ContainsKey(key)) { this.Remove(key); }
}
Evaluate(problemData, (this[ProgramResultName] as TextValue).Value, timeout, pythonProcess);
}
public void Reevaluate(ICFGPythonProblemData problemData, double timeout, PythonProcess pythonProcess)
{
foreach (var key in ResultKeys)
{
if (this.ContainsKey(key))
{
this.Remove(key);
}
}
Evaluate(problemData, (this[ProgramResultName] as TextValue).Value, timeout, pythonProcess);
}
public CFGPythonSolution(ISymbolicExpressionTree tree, ICFGPythonProblemData problemData, double timeout, PythonProcess pythonProcess)
: base(tree, problemData) {
name = ItemName;
description = ItemDescription;
string program = PythonHelper.FormatToProgram(tree, problemData.FullHeader, problemData.FullFooter);
Add(new Result(ProgramResultName, "The program with header and footer", new TextValue(program)));
string code = CFGSymbolicExpressionTreeStringFormatter.StaticFormat(tree);
Add(new Result(CodeResultName, "The code that was evolved", new TextValue(code)));
Evaluate(problemData, program, timeout, pythonProcess);
}
public CFGPythonSolution(ISymbolicExpressionTree tree, ICFGPythonProblemData problemData, double timeout, PythonProcess pythonProcess)
: base(tree, problemData)
{
name = ItemName;
description = ItemDescription;
string program = PythonHelper.FormatToProgram(tree, problemData.LoopBreakConst, problemData.FullHeader, problemData.FullFooter);
Add(new Result(ProgramResultName, "The program with header and footer", new TextValue(program)));
string code = CFGSymbolicExpressionTreeStringFormatter.StaticFormat(tree);
Add(new Result(CodeResultName, "The code that was evolved", new TextValue(code)));
Evaluate(problemData, program, timeout, pythonProcess);
}
public void Evaluate(ICFGPythonProblemData problemData, string program, double timeout, PythonProcess pythonProcess)
{
var trainingTimeout = timeout * 2; // increase timeout to make sure it finishes
var training = pythonProcess.EvaluateProgram(program, problemData.Input, problemData.Output, problemData.TrainingIndices, trainingTimeout);
// test timeout should be proportionally bigger than training timeout
var testTimeout = (double)problemData.TestIndices.Count() / (double)problemData.TrainingIndices.Count() * trainingTimeout;
testTimeout = testTimeout > timeout ? testTimeout : timeout;
var test = pythonProcess.EvaluateProgram(program, problemData.Input, problemData.Output, problemData.TestIndices, testTimeout);
if (String.IsNullOrEmpty(training.Item4))
{
Add(new Result(TrainingQuality, "Training quality", new DoubleValue(training.Item3)));
var cases = training.Item1.ToArray();
Add(new Result(TrainingSolvedCases, "Training cases which have been solved", new BoolArray(cases)));
Add(new Result(TrainingSolvedCasesPercentage, "Percentage of training cases which have been solved", new PercentValue((double)cases.Count(x => x) / (double)cases.Length)));
Add(new Result(TrainingSolvedCaseQualities, "The quality of each training case", new DoubleArray(training.Item2.ToArray())));
}
else
{
Add(new Result(TrainingException, "Exception occured during training", new TextValue(training.Item4)));
Add(new Result(TrainingQuality, "Training quality", new DoubleValue(Double.NaN)));
Add(new Result(TrainingSolvedCases, "Training cases which have been solved", new BoolArray(problemData.TrainingIndices.Count())));
Add(new Result(TrainingSolvedCasesPercentage, "Percentage of training cases which have been solved", new PercentValue(0)));
Add(new Result(TrainingSolvedCaseQualities, "The quality of each training case", new DoubleArray(Enumerable.Repeat(Double.NaN, problemData.TrainingIndices.Count()).ToArray())));
}
if (String.IsNullOrEmpty(test.Item4))
{
Add(new Result(TestQuality, "Test quality", new DoubleValue(test.Item3)));
var cases = test.Item1.ToArray();
Add(new Result(TestSolvedCases, "Test cases which have been solved", new BoolArray(cases)));
Add(new Result(TestSolvedCasesPercentage, "Percentage of test cases which have been solved", new PercentValue((double)cases.Count(x => x) / (double)cases.Length)));
Add(new Result(TestSolvedCaseQualities, "The quality of each test case", new DoubleArray(test.Item2.ToArray())));
}
else
{
Add(new Result(TestException, "Exception occured during test", new TextValue(test.Item4)));
Add(new Result(TestQuality, "Test quality", new DoubleValue(Double.NaN)));
Add(new Result(TestSolvedCases, "Test cases which have been solved", new BoolArray(problemData.TestIndices.Count())));
Add(new Result(TestSolvedCasesPercentage, "Percentage of test cases which have been solved", new PercentValue(0)));
Add(new Result(TestSolvedCaseQualities, "The quality of each test case", new DoubleArray(Enumerable.Repeat(Double.NaN, problemData.TestIndices.Count()).ToArray())));
}
}
protected override ISymbolicExpressionTree Cross(IRandom random, ISymbolicExpressionTree parent0, ISymbolicExpressionTree parent1, ItemArray <PythonStatementSemantic> semantic0, ItemArray <PythonStatementSemantic> semantic1, ICFGPythonProblemData problemData, int maxTreeLength, int maxTreeDepth, double internalCrossoverPointProbability, out ItemArray <PythonStatementSemantic> newSemantics)
{
newSemantics = semantic0;
if (semantic0 == null || semantic1 == null || semantic0.Length == 0 || semantic1.Length == 0)
{
return(parent0);
}
// select a random crossover point in the first parent
CutPoint crossoverPoint0;
SelectCrossoverPoint(random, parent0, internalCrossoverPointProbability, maxTreeLength, maxTreeDepth, out crossoverPoint0);
int childLength = crossoverPoint0.Child != null?crossoverPoint0.Child.GetLength() : 0;
// calculate the max length and depth that the inserted branch can have
int maxInsertedBranchLength = Math.Max(0, maxTreeLength - (parent0.Length - childLength));
int maxInsertedBranchDepth = Math.Max(0, maxTreeDepth - parent0.Root.GetBranchLevel(crossoverPoint0.Parent));
List <ISymbolicExpressionTreeNode> allowedBranches = new List <ISymbolicExpressionTreeNode>();
parent1.Root.ForEachNodePostfix((n) => {
if (n.GetLength() <= maxInsertedBranchLength &&
n.GetDepth() <= maxInsertedBranchDepth && crossoverPoint0.IsMatchingPointType(n))
{
allowedBranches.Add(n);
}
});
// empty branch
if (crossoverPoint0.IsMatchingPointType(null))
{
allowedBranches.Add(null);
}
if (allowedBranches.Count == 0)
{
return(parent0);
}
var selectedBranch = SelectRandomBranch(random, allowedBranches, internalCrossoverPointProbability);
newSemantics = SemanticSwap(crossoverPoint0, selectedBranch, parent0, parent1, semantic0, semantic1);
return(parent0);
}
public BrokenIndividual(ISymbolicExpressionTree tree, string exception, bool includeTree, ICFGPythonProblemData problemData)
{
name = String.Format("{0}: {1}", ItemName, exception);
description = ItemDescription;
Add(new Result(ModelLengthResultName, "Length of the symbolic regression model.", new IntValue(tree.Length)));
Add(new Result(ModelDepthResultName, "Depth of the symbolic regression model.", new IntValue(tree.Depth)));
if (includeTree)
{
Add(new Result(ModelResultName, "The CFG model.", tree));
}
Add(new Result(ExceptiontName, "The CFG model.", new StringValue(exception)));
string program = PythonHelper.FormatToProgram(tree, problemData.LoopBreakConst, problemData.FullHeader, problemData.FullFooter);
Add(new Result(ProgramResultName, "The program with header and footer", new TextValue(program)));
string code = CFGSymbolicExpressionTreeStringFormatter.StaticFormat(tree);
Add(new Result(CodeResultName, "The code that was evolved", new TextValue(code)));
}
public override void ReplaceBranch(IRandom random, ISymbolicExpressionTree symbolicExpressionTree, ICFGPythonProblemData problemData, ItemArray <PythonStatementSemantic> semantics, PythonProcess pythonProcess, double timeout, int maxTreeLength, int maxTreeDepth, int maximumSemanticTries)
{
if (semantics == null || semantics.Length == 0)
{
ReplaceBranchManipulation.ReplaceRandomBranch(random, symbolicExpressionTree, maxTreeLength, maxTreeDepth);
SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(NoSemantics);
MutationTypeParameter.ActualValue = new IntValue(RandomMutation);
return;
}
var statementProductionNames = SemanticOperatorHelper.GetSemanticProductionNames(symbolicExpressionTree.Root.Grammar);
var variables = problemData.Variables.GetVariableNames().ToList();
string variableSettings = problemData.VariableSettings.Count == 0 ? String.Empty : String.Join(Environment.NewLine, problemData.VariableSettings.Select(x => x.Value));
var allowedSymbols = new List <ISymbol>();
// repeat until a fitting parent and child are found (MAX_TRIES times)
int tries = 0;
int semanticTries = 0;
List <JObject> saveOriginalSemantics = null;
List <JObject> saveReplaceSemantics = null;
List <Tuple <ISymbolicExpressionTreeNode, ISymbolicExpressionTreeNode, int> > possibleChildren = null; // Item1 = parent, Item2 = seedNode, Item3 = childIndex
if (UsesAdditionalSemanticMeasure())
{
saveOriginalSemantics = new List <JObject>(semanticTries);
saveReplaceSemantics = new List <JObject>(semanticTries);
possibleChildren = new List <Tuple <ISymbolicExpressionTreeNode, ISymbolicExpressionTreeNode, int> >(semanticTries);
}
bool success = false;
do
{
#region find mutation point
#pragma warning disable 612, 618
ISymbolicExpressionTreeNode parent = symbolicExpressionTree.Root.IterateNodesPrefix().Skip(1).Where(n => n.SubtreeCount > 0).SelectRandom(random);
#pragma warning restore 612, 618
int childIndex = random.Next(parent.SubtreeCount);
var child = parent.GetSubtree(childIndex);
int maxLength = maxTreeLength - symbolicExpressionTree.Length + child.GetLength();
int maxDepth = maxTreeDepth - symbolicExpressionTree.Root.GetBranchLevel(child);
allowedSymbols.Clear();
foreach (var symbol in parent.Grammar.GetAllowedChildSymbols(parent.Symbol, childIndex))
{
// check basic properties that the new symbol must have
if ((symbol.Name != child.Symbol.Name || symbol.MinimumArity > 0) &&
symbol.InitialFrequency > 0 &&
parent.Grammar.GetMinimumExpressionDepth(symbol) <= maxDepth &&
parent.Grammar.GetMinimumExpressionLength(symbol) <= maxLength)
{
allowedSymbols.Add(symbol);
}
}
#endregion
#region check for semantic difference with a new random tree
if (allowedSymbols.Count > 0)
{
if (semanticTries <= maximumSemanticTries)
{
// do semantic mutation
#region calculate original json output
ISymbolicExpressionTreeNode statement = SemanticOperatorHelper.GetStatementNode(child, statementProductionNames);
var statementPos0 = symbolicExpressionTree.IterateNodesPrefix().ToList().IndexOf(statement);
PythonStatementSemantic curSemantics = null;
if (String.IsNullOrEmpty(variableSettings))
{
curSemantics = semantics.First(x => x.TreeNodePrefixPos == statementPos0);
variableSettings = SemanticOperatorHelper.SemanticToPythonVariableSettings(curSemantics.Before, problemData.Variables.GetVariableTypes());
}
var jsonOriginal = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout);
// compare jsonOriginal to semantic after! Maybe avoid additional evaluation.
#endregion
var seedNode = GenerateAndInsertNewSubtree(random, parent, allowedSymbols, childIndex, maxLength, maxDepth);
#region calculate new json output
JObject jsonReplaced;
if (child == statement)
{
// child is executable, so is the new child
jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(seedNode, pythonProcess, random, problemData, variables, variableSettings, timeout);
}
else
{
jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout);
}
var exception = jsonOriginal["exception"] != null || jsonReplaced["exception"] != null;
if (exception)
{
if (jsonOriginal["exception"] != null)
{
MutationExceptionsParameter.ActualValue.Add(new StringValue(jsonOriginal["exception"].ToString()));
}
if (jsonReplaced["exception"] != null)
{
MutationExceptionsParameter.ActualValue.Add(new StringValue(jsonReplaced["exception"].ToString()));
}
}
if (curSemantics != null && !exception)
{
jsonOriginal = PythonSemanticComparer.ReplaceNotExecutedCases(jsonOriginal, curSemantics.Before, curSemantics.ExecutedCases);
jsonReplaced = PythonSemanticComparer.ReplaceNotExecutedCases(jsonReplaced, curSemantics.Before, curSemantics.ExecutedCases);
jsonOriginal = PythonSemanticComparer.ProduceDifference(jsonOriginal, curSemantics.Before);
jsonReplaced = PythonSemanticComparer.ProduceDifference(jsonReplaced, curSemantics.Before);
}
if (!exception && SemanticMeasure(jsonOriginal, jsonReplaced))
{
success = true;
SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(Different);
TypeSelectedForSimilarityParameter.ActualValue = new StringValue("First Semantic");
MutationTypeParameter.ActualValue = new IntValue(SemanticMutation);
}
else
{
// undo mutation
parent.RemoveSubtree(childIndex);
parent.InsertSubtree(childIndex, child);
allowedSymbols.Clear();
if (!exception && UsesAdditionalSemanticMeasure())
{
saveOriginalSemantics.Add(jsonOriginal);
saveReplaceSemantics.Add(jsonReplaced);
possibleChildren.Add(new Tuple <ISymbolicExpressionTreeNode, ISymbolicExpressionTreeNode, int>(parent, seedNode, childIndex));
}
}
if (problemData.VariableSettings.Count == 0)
{
// reset variableSettings
variableSettings = String.Empty;
}
semanticTries++;
#endregion
#region try second semantic comparison
if (!success && semanticTries >= maximumSemanticTries && UsesAdditionalSemanticMeasure())
{
for (int index = 0; index < saveOriginalSemantics.Count; index++)
{
if (AdditionalSemanticMeasure(saveOriginalSemantics[index], saveReplaceSemantics[index]))
{
var mutation = possibleChildren[index];
mutation.Item1.RemoveSubtree(mutation.Item3);
mutation.Item1.InsertSubtree(mutation.Item3, mutation.Item2);
success = true;
SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(Different);
TypeSelectedForSimilarityParameter.ActualValue = new StringValue("Second Semantic");
MutationTypeParameter.ActualValue = new IntValue(SemanticMutation);
break;
}
}
}
#endregion
}
else
{
// do random mutation
#region calculate original json output
ISymbolicExpressionTreeNode statement = SemanticOperatorHelper.GetStatementNode(child, statementProductionNames);
var statementPos0 = symbolicExpressionTree.IterateNodesPrefix().ToList().IndexOf(statement);
if (String.IsNullOrEmpty(variableSettings))
{
variableSettings = SemanticOperatorHelper.SemanticToPythonVariableSettings(semantics.First(x => x.TreeNodePrefixPos == statementPos0).Before, problemData.Variables.GetVariableTypes());
}
var jsonOriginal = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout);
#endregion
var seedNode = GenerateAndInsertNewSubtree(random, parent, allowedSymbols, childIndex, maxLength, maxDepth);
JObject jsonReplaced;
if (child == statement)
{
// child is executable, so is the new child
jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(seedNode, pythonProcess, random, problemData, variables, variableSettings, timeout);
}
else
{
jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout);
}
if (JToken.EqualityComparer.Equals(jsonOriginal, jsonReplaced))
{
SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(Equvivalent);
}
else
{
SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(Different);
}
TypeSelectedForSimilarityParameter.ActualValue = new StringValue("Random Crossover; Reached Max Semantic Tries");
MutationTypeParameter.ActualValue = new IntValue(RandomMutation);
success = true;
}
}
#endregion
tries++;
} while (tries < MAX_TRIES && !success);
NumberOfTriesParameter.ActualValue = new IntValue(semanticTries);
if (SemanticallyEquivalentMutationParameter.ActualValue == null)
{
SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(NoMutation);
TypeSelectedForSimilarityParameter.ActualValue = new StringValue("No mutation");
MutationTypeParameter.ActualValue = new IntValue(NoMutation);
}
}
private void SelectBranchFromExecutableNodes(IList <CutPoint> crossoverPoints0, IList <IEnumerable <ISymbolicExpressionTreeNode> > allowedBranchesPerCutpoint, IRandom random, List <string> variables, string variableSettings, ICFGPythonProblemData problemData, out ISymbolicExpressionTreeNode selectedBranch, out CutPoint selectedCutPoint)
{
var jsonOutput = new Dictionary <ISymbolicExpressionTreeNode, JObject>();
for (int i = 0; i < crossoverPoints0.Count; i++)
{
var cutPoint = crossoverPoints0[i];
var jsonCur = SemanticOperatorHelper.EvaluateStatementNode(cutPoint.Child, PyProcess, random, problemData, variables, variableSettings, Timeout);
if (!String.IsNullOrWhiteSpace((string)jsonCur["exception"]))
{
continue;
}
foreach (var possibleBranch in allowedBranchesPerCutpoint[i])
{
JObject jsonPossibleBranch;
if (!jsonOutput.ContainsKey(possibleBranch))
{
jsonPossibleBranch = SemanticOperatorHelper.EvaluateStatementNode(possibleBranch, PyProcess, random, problemData, variables, variableSettings, Timeout);
jsonOutput.Add(possibleBranch, jsonPossibleBranch);
}
else
{
jsonPossibleBranch = jsonOutput[possibleBranch];
}
if (!String.IsNullOrWhiteSpace((string)jsonPossibleBranch["exception"]))
{
continue;
}
if (JToken.EqualityComparer.Equals(jsonCur, jsonPossibleBranch))
{
continue;
}
selectedBranch = possibleBranch;
selectedCutPoint = cutPoint;
return;
}
}
// no difference was found with any comparison, select random
selectedBranch = allowedBranchesPerCutpoint[0].SampleRandom(random);
selectedCutPoint = crossoverPoints0[0];
}
private static string FormatScript(ISymbolicExpressionTree symbolicExpressionTree, ICFGPythonProblemData problemData, int loopBreakConst, IEnumerable <string> variables, string variableSettings)
{
Regex r = new Regex(@"^(.*?)\s*=", RegexOptions.Multiline);
string variableSettingsSubstitute = r.Replace(variableSettings, "${1}_setting =");
return(String.Format(EVAL_TRACE_SCRIPT, problemData.HelperCode.Value,
String.Format("'{0}'", String.Join("','", variables)),
variableSettingsSubstitute,
String.Join(",", variables),
String.Join(",", variables.Select(x => x + "_setting")),
PythonHelper.FormatToProgram(symbolicExpressionTree, loopBreakConst, " ")));
}
public static JObject EvaluateStatementNode(ISymbolicExpressionTreeNode statement, PythonProcess pythonProcess, IRandom random, ICFGPythonProblemData problemData, IList <string> variables, string variableSettings, double timeout)
{
var statementParent = statement.Parent;
EvaluationScript crossoverPointScript0 = new EvaluationScript()
{
Script = FormatScript(CreateTreeFromNode(random, statement, RootSymbol, StartSymbol), problemData, problemData.LoopBreakConst, variables, variableSettings),
Variables = variables,
Timeout = timeout
};
JObject json0 = pythonProcess.SendAndEvaluateProgram(crossoverPointScript0);
statement.Parent = statementParent; // restore parent
return(json0);
}
private JObject GenerateOriginalJson(CutPoint crossoverPoint0, IEnumerable <string> statementProductionNames, ISymbolicExpressionTree parent0, string variableSettings, ItemArray <PythonStatementSemantic> semantic0, ICFGPythonProblemData problemData, IRandom random, List <string> variables)
{
ISymbolicExpressionTreeNode statementOriginal = SemanticOperatorHelper.GetStatementNode(crossoverPoint0.Child, statementProductionNames); // statementOriginal is not always the same as statement
var statementPos0 = parent0.IterateNodesPrefix().ToList().IndexOf(statementOriginal);
if (String.IsNullOrEmpty(variableSettings))
{
var curSemantics = semantic0.First(x => x.TreeNodePrefixPos == statementPos0);
variableSettings = SemanticOperatorHelper.SemanticToPythonVariableSettings(curSemantics.Before, problemData.Variables.GetVariableTypes());
}
return(SemanticOperatorHelper.EvaluateStatementNode(statementOriginal, PyProcess, random, problemData, variables, variableSettings, Timeout));
}
public override void ReplaceBranch(IRandom random, ISymbolicExpressionTree symbolicExpressionTree, ICFGPythonProblemData problemData, ItemArray <PythonStatementSemantic> semantics, PythonProcess pythonProcess, double timeout, int maxTreeLength, int maxTreeDepth, int maximumSemanticTries)
{
if (semantics == null || semantics.Length == 0)
{
ReplaceBranchManipulation.ReplaceRandomBranch(random, symbolicExpressionTree, maxTreeLength, maxTreeDepth);
SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(NoSemantics);
MutationTypeParameter.ActualValue = new IntValue(RandomMutation);
return;
}
var allowedSymbols = new List <ISymbol>();
ISymbolicExpressionTreeNode parent;
int childIndex;
int maxLength;
int maxDepth;
// repeat until a fitting parent and child are found (MAX_TRIES times)
int tries = 0;
ISymbolicExpressionTreeNode child;
do
{
#pragma warning disable 612, 618
parent = symbolicExpressionTree.Root.IterateNodesPrefix().Skip(1).Where(n => n.SubtreeCount > 0).SelectRandom(random);
#pragma warning restore 612, 618
childIndex = random.Next(parent.SubtreeCount);
child = parent.GetSubtree(childIndex);
maxLength = maxTreeLength - symbolicExpressionTree.Length + child.GetLength();
maxDepth = maxTreeDepth - symbolicExpressionTree.Root.GetBranchLevel(child);
allowedSymbols.Clear();
foreach (var symbol in parent.Grammar.GetAllowedChildSymbols(parent.Symbol, childIndex))
{
// check basic properties that the new symbol must have
if ((symbol.Name != child.Symbol.Name || symbol.MinimumArity > 0) &&
symbol.InitialFrequency > 0 &&
parent.Grammar.GetMinimumExpressionDepth(symbol) <= maxDepth &&
parent.Grammar.GetMinimumExpressionLength(symbol) <= maxLength)
{
allowedSymbols.Add(symbol);
}
}
tries++;
} while (tries < MAX_TRIES && allowedSymbols.Count == 0);
NumberOfTriesParameter.ActualValue = new IntValue(tries); //normal tries. not semantic tries
if (tries < MAX_TRIES)
{
var statementProductionNames = SemanticOperatorHelper.GetSemanticProductionNames(symbolicExpressionTree.Root.Grammar);
var variables = problemData.Variables.GetVariableNames().ToList();
#region calculate original json output
ISymbolicExpressionTreeNode statement = SemanticOperatorHelper.GetStatementNode(child, statementProductionNames);
string variableSettings = problemData.VariableSettings.Count == 0 ? String.Empty : String.Join(Environment.NewLine, problemData.VariableSettings.Select(x => x.Value));
var statementPos0 = symbolicExpressionTree.IterateNodesPrefix().ToList().IndexOf(statement);
if (String.IsNullOrEmpty(variableSettings))
{
variableSettings = SemanticOperatorHelper.SemanticToPythonVariableSettings(semantics.First(x => x.TreeNodePrefixPos == statementPos0).Before, problemData.Variables.GetVariableTypes());
}
var jsonOriginal = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout);
#endregion
var weights = allowedSymbols.Select(s => s.InitialFrequency).ToList();
#pragma warning disable 612, 618
var seedSymbol = allowedSymbols.SelectRandom(weights, random);
#pragma warning restore 612, 618
// replace the old node with the new node
var seedNode = seedSymbol.CreateTreeNode();
if (seedNode.HasLocalParameters)
{
seedNode.ResetLocalParameters(random);
}
parent.RemoveSubtree(childIndex);
parent.InsertSubtree(childIndex, seedNode);
ProbabilisticTreeCreator.PTC2(random, seedNode, maxLength, maxDepth);
JObject jsonReplaced;
if (child == statement)
{
// child is executable, so is the new child
jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(seedNode, pythonProcess, random, problemData, variables, variableSettings, timeout);
}
else
{
jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout);
}
if (JToken.EqualityComparer.Equals(jsonOriginal, jsonReplaced))
{
SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(Equvivalent);
}
else
{
SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(Different);
}
MutationTypeParameter.ActualValue = new IntValue(RandomMutation);
}
else
{
SemanticallyEquivalentMutationParameter.ActualValue = new IntValue(NoMutation);
MutationTypeParameter.ActualValue = new IntValue(NoMutation);
}
}
public void Evaluate(ICFGPythonProblemData problemData, string program, double timeout, PythonProcess pythonProcess) {
var trainingTimeout = timeout * 2; // increase timeout to make sure it finishes
var training = pythonProcess.EvaluateProgram(program, problemData.Input, problemData.Output, problemData.TrainingIndices, trainingTimeout);
// test timeout should be proportionally bigger than training timeout
var testTimeout = (double)problemData.TestIndices.Count() / (double)problemData.TrainingIndices.Count() * trainingTimeout;
testTimeout = testTimeout > timeout ? testTimeout : timeout;
var test = pythonProcess.EvaluateProgram(program, problemData.Input, problemData.Output, problemData.TestIndices, testTimeout);
if (String.IsNullOrEmpty(training.Item4)) {
Add(new Result(TrainingQuality, "Training quality", new DoubleValue(training.Item3)));
var cases = training.Item1.ToArray();
Add(new Result(TrainingSolvedCases, "Training cases which have been solved", new BoolArray(cases)));
Add(new Result(TrainingSolvedCasesPercentage, "Percentage of training cases which have been solved", new PercentValue((double)cases.Count(x => x) / (double)cases.Length)));
Add(new Result(TrainingSolvedCaseQualities, "The quality of each training case", new DoubleArray(training.Item2.ToArray())));
} else {
Add(new Result(TrainingException, "Exception occured during training", new TextValue(training.Item4)));
Add(new Result(TrainingQuality, "Training quality", new DoubleValue(Double.NaN)));
Add(new Result(TrainingSolvedCases, "Training cases which have been solved", new BoolArray(problemData.TrainingIndices.Count())));
Add(new Result(TrainingSolvedCasesPercentage, "Percentage of training cases which have been solved", new PercentValue(0)));
Add(new Result(TrainingSolvedCaseQualities, "The quality of each training case", new DoubleArray(Enumerable.Repeat(Double.NaN, problemData.TrainingIndices.Count()).ToArray())));
}
if (String.IsNullOrEmpty(test.Item4)) {
Add(new Result(TestQuality, "Test quality", new DoubleValue(test.Item3)));
var cases = test.Item1.ToArray();
Add(new Result(TestSolvedCases, "Test cases which have been solved", new BoolArray(cases)));
Add(new Result(TestSolvedCasesPercentage, "Percentage of test cases which have been solved", new PercentValue((double)cases.Count(x => x) / (double)cases.Length)));
Add(new Result(TestSolvedCaseQualities, "The quality of each test case", new DoubleArray(test.Item2.ToArray())));
} else {
Add(new Result(TestException, "Exception occured during test", new TextValue(test.Item4)));
Add(new Result(TestQuality, "Test quality", new DoubleValue(Double.NaN)));
Add(new Result(TestSolvedCases, "Test cases which have been solved", new BoolArray(problemData.TestIndices.Count())));
Add(new Result(TestSolvedCasesPercentage, "Percentage of test cases which have been solved", new PercentValue(0)));
Add(new Result(TestSolvedCaseQualities, "The quality of each test case", new DoubleArray(Enumerable.Repeat(Double.NaN, problemData.TestIndices.Count()).ToArray())));
}
}
protected virtual ISymbolicExpressionTree Cross(IRandom random, ISymbolicExpressionTree parent0, ISymbolicExpressionTree parent1, ItemArray <PythonStatementSemantic> semantic0, ItemArray <PythonStatementSemantic> semantic1, ICFGPythonProblemData problemData, int maxTreeLength, int maxTreeDepth, double internalCrossoverPointProbability, out ItemArray <PythonStatementSemantic> newSemantics)
{
if (semantic0 == null || semantic1 == null || semantic0.Length == 0 || semantic1.Length == 0)
{
parent0 = SubtreeCrossover.Cross(random, parent0, parent1, internalCrossoverPointProbability, maxTreeLength, maxTreeDepth);
newSemantics = null;
return(parent0);
}
newSemantics = semantic0;
// select a random crossover point in the first parent
CutPoint crossoverPoint0;
SelectCrossoverPoint(random, parent0, internalCrossoverPointProbability, maxTreeLength, maxTreeDepth, out crossoverPoint0);
int childLength = crossoverPoint0.Child != null?crossoverPoint0.Child.GetLength() : 0;
// calculate the max length and depth that the inserted branch can have
int maxInsertedBranchLength = maxTreeLength - (parent0.Length - childLength);
int maxInsertedBranchDepth = maxTreeDepth - parent0.Root.GetBranchLevel(crossoverPoint0.Child);
List <ISymbolicExpressionTreeNode> allowedBranches = new List <ISymbolicExpressionTreeNode>();
parent1.Root.ForEachNodePostfix((n) => {
if (n.GetLength() <= maxInsertedBranchLength &&
n.GetDepth() <= maxInsertedBranchDepth && crossoverPoint0.IsMatchingPointType(n))
{
allowedBranches.Add(n);
}
});
// empty branch
if (crossoverPoint0.IsMatchingPointType(null))
{
allowedBranches.Add(null);
}
if (allowedBranches.Count == 0)
{
return(parent0);
}
// select MaxCompares random crossover points
// Use set to avoid having the same node multiple times
HashSet <ISymbolicExpressionTreeNode> compBranches;
if (allowedBranches.Count < MaxComparesParameter.Value.Value)
{
compBranches = new HashSet <ISymbolicExpressionTreeNode>(allowedBranches);
}
else
{
compBranches = new HashSet <ISymbolicExpressionTreeNode>();
for (int i = 0; i < MaxComparesParameter.Value.Value; i++)
{
var possibleBranch = SelectRandomBranch(random, allowedBranches, internalCrossoverPointProbability);
allowedBranches.Remove(possibleBranch);
compBranches.Add(possibleBranch);
}
}
var statementProductionNames = SemanticOperatorHelper.GetSemanticProductionNames(crossoverPoint0.Parent.Grammar);
// find first node that can be used for evaluation in parent0
ISymbolicExpressionTreeNode statement = SemanticOperatorHelper.GetStatementNode(crossoverPoint0.Child, statementProductionNames);
if (statement == null)
{
newSemantics = SemanticSwap(crossoverPoint0, compBranches.SampleRandom(random), parent0, parent1, semantic0, semantic1);
return(parent0);
}
var statementPos0 = parent0.IterateNodesPrefix().ToList().IndexOf(statement);
string variableSettings;
if (problemData.VariableSettings.Count == 0)
{
variableSettings = SemanticOperatorHelper.SemanticToPythonVariableSettings(semantic0.First(x => x.TreeNodePrefixPos == statementPos0).Before, problemData.Variables.GetVariableTypes());
}
else
{
variableSettings = String.Join(Environment.NewLine, problemData.VariableSettings.Select(x => x.Value));
}
var variables = problemData.Variables.GetVariableNames().ToList();
var json0 = SemanticOperatorHelper.EvaluateStatementNode(statement, PyProcess, random, problemData, variables, variableSettings, Timeout);
ISymbolicExpressionTreeNode selectedBranch;
if (!String.IsNullOrWhiteSpace((string)json0["exception"]))
{
selectedBranch = compBranches.SampleRandom(random);
}
else
{
selectedBranch = SelectBranch(statement, crossoverPoint0, compBranches, random, variables, variableSettings, json0, problemData);
}
// perform the actual swap
newSemantics = SemanticSwap(crossoverPoint0, selectedBranch, parent0, parent1, semantic0, semantic1);
return(parent0);
}
// version 1
// only use primary cut point
private void SelectBranchFromNodes1(ISymbolicExpressionTreeNode statementNode, IList <CutPoint> crossoverPoints0, IList <IEnumerable <ISymbolicExpressionTreeNode> > allowedBranchesPerCutpoint, IRandom random, List <string> variables, string variableSettings, ICFGPythonProblemData problemData, out ISymbolicExpressionTreeNode selectedBranch, out CutPoint selectedCutPoint)
{
var jsonOutput = new Dictionary <ISymbolicExpressionTreeNode, JObject>();
var primaryCutPoint = crossoverPoints0[0];
primaryCutPoint.Parent.RemoveSubtree(primaryCutPoint.ChildIndex); // removes parent from child
for (int i = 0; i < crossoverPoints0.Count; i++)
{
var cutPoint = crossoverPoints0[i];
primaryCutPoint.Parent.InsertSubtree(primaryCutPoint.ChildIndex, cutPoint.Child); // this will affect cutPoint.Parent
var jsonCur = SemanticOperatorHelper.EvaluateStatementNode(statementNode, PyProcess, random, problemData, variables, variableSettings, Timeout);
primaryCutPoint.Parent.RemoveSubtree(primaryCutPoint.ChildIndex); // removes intermediate parent from node
cutPoint.Child.Parent = cutPoint.Parent; // restore parent
if (!String.IsNullOrWhiteSpace((string)jsonCur["exception"]))
{
continue;
}
foreach (var possibleBranch in allowedBranchesPerCutpoint[i])
{
JObject jsonPossibleBranch;
if (!jsonOutput.ContainsKey(possibleBranch))
{
var parent = possibleBranch.Parent; // save parent
primaryCutPoint.Parent.InsertSubtree(primaryCutPoint.ChildIndex, possibleBranch); // this will affect node.Parent
jsonPossibleBranch = SemanticOperatorHelper.EvaluateStatementNode(statementNode, PyProcess, random, problemData, variables, variableSettings, Timeout);
primaryCutPoint.Parent.RemoveSubtree(primaryCutPoint.ChildIndex); // removes intermediate parent from node
possibleBranch.Parent = parent; // restore parent
jsonOutput.Add(possibleBranch, jsonPossibleBranch);
}
else
{
jsonPossibleBranch = jsonOutput[possibleBranch];
}
if (!String.IsNullOrWhiteSpace((string)jsonPossibleBranch["exception"]))
{
continue;
}
if (!JToken.EqualityComparer.Equals(jsonCur, jsonPossibleBranch))
{
primaryCutPoint.Parent.InsertSubtree(primaryCutPoint.ChildIndex, primaryCutPoint.Child); // restore primaryCutPoint
selectedBranch = possibleBranch;
selectedCutPoint = cutPoint;
return;
}
}
}
primaryCutPoint.Parent.InsertSubtree(primaryCutPoint.ChildIndex, primaryCutPoint.Child); // restore primaryCutPoint
// no difference was found with any comparison, select random
selectedBranch = allowedBranchesPerCutpoint[0].SampleRandom(random);
selectedCutPoint = crossoverPoints0[0];
}
private ISymbolicExpressionTreeNode SelectBranch(ISymbolicExpressionTreeNode statementNode, CutPoint crossoverPoint0, IEnumerable <ISymbolicExpressionTreeNode> compBranches, IRandom random, List <string> variables, string variableSettings, JObject jsonParent0, ICFGPythonProblemData problemData)
{
List <JObject> evaluationPerNode = new List <JObject>();
List <double> similarity = new List <double>();
crossoverPoint0.Parent.RemoveSubtree(crossoverPoint0.ChildIndex); // removes parent from child
foreach (var node in compBranches)
{
JObject json;
if (statementNode == crossoverPoint0.Child)
{
json = SemanticOperatorHelper.EvaluateStatementNode(node, PyProcess, random, problemData, variables, variableSettings, Timeout);
}
else
{
var parent = node.Parent; // save parent
crossoverPoint0.Parent.InsertSubtree(crossoverPoint0.ChildIndex, node); // this will affect node.Parent
json = SemanticOperatorHelper.EvaluateStatementNode(statementNode, PyProcess, random, problemData, variables, variableSettings, Timeout);
crossoverPoint0.Parent.RemoveSubtree(crossoverPoint0.ChildIndex); // removes intermediate parent from node
node.Parent = parent; // restore parent
}
evaluationPerNode.Add(json);
similarity.Add(0);
}
crossoverPoint0.Parent.InsertSubtree(crossoverPoint0.ChildIndex, crossoverPoint0.Child); // restore crossoverPoint0
#region remove branches that threw an exception
List <int> branchesCausedExceptions = new List <int>();
for (int i = evaluationPerNode.Count - 1; i >= 0; i--)
{
if (evaluationPerNode[i]["exception"] != null)
{
branchesCausedExceptions.Add(i);
}
}
var branchesWithoutException = compBranches.ToList();
foreach (int index in branchesCausedExceptions)
{
branchesWithoutException.RemoveAt(index);
evaluationPerNode.RemoveAt(index);
similarity.RemoveAt(index);
}
#endregion
Dictionary <VariableType, List <string> > differencesPerType = new Dictionary <VariableType, List <string> >();
foreach (var entry in problemData.Variables.GetTypesOfVariables())
{
List <string> differences = new List <string>();
foreach (var variableName in entry.Value)
{
if (evaluationPerNode.Any(x => !JToken.EqualityComparer.Equals(jsonParent0[variableName], x[variableName])))
{
differences.Add(variableName);
}
}
if (differences.Count > 0)
{
differencesPerType.Add(entry.Key, differences);
}
}
if (differencesPerType.Count == 0)
{
return(compBranches.SampleRandom(random)); // no difference found, crossover with any branch
}
var typeDifference = differencesPerType.SampleRandom(random);
foreach (var variableName in typeDifference.Value)
{
var variableSimilarity = CalculateDifference(jsonParent0[variableName], evaluationPerNode.Select(x => x[variableName]), typeDifference.Key, true);
similarity = similarity.Zip(variableSimilarity, (x, y) => x + y).ToList();
}
similarity = similarity.Select(x => x / typeDifference.Value.Count).ToList(); // normalize between 0 and 1 again (actually not necessary)
double best = Double.MaxValue;
int pos = -1;
for (int i = 0; i < similarity.Count; i++)
{
if (similarity[i] > 0 && similarity[i] < best)
{
best = similarity[i];
pos = i;
}
}
return(pos >= 0 ? branchesWithoutException.ElementAt(pos) : compBranches.SampleRandom(random));
}
// version 2
// check the statement node of every cut point in the parent
// takes longer, but really checks for differences
private void SelectBranchFromNodes2(IEnumerable <string> statementProductionNames, IList <CutPoint> crossoverPoints0, IList <IEnumerable <ISymbolicExpressionTreeNode> > allowedBranchesPerCutpoint, IRandom random, List <string> variables, string variableSettings, ICFGPythonProblemData problemData, out ISymbolicExpressionTreeNode selectedBranch, out CutPoint selectedCutPoint)
{
for (int i = 0; i < crossoverPoints0.Count; i++)
{
var cutPoint = crossoverPoints0[i];
ISymbolicExpressionTreeNode curStatementNode = SemanticOperatorHelper.GetStatementNode(cutPoint.Child, statementProductionNames);
var jsonCur = SemanticOperatorHelper.EvaluateStatementNode(curStatementNode, PyProcess, random, problemData, variables, variableSettings, Timeout);
if (!String.IsNullOrWhiteSpace((string)jsonCur["exception"]))
{
continue;
}
cutPoint.Parent.RemoveSubtree(cutPoint.ChildIndex); // removes parent from node
foreach (var possibleBranch in allowedBranchesPerCutpoint[i])
{
JObject jsonPossibleBranch;
if (curStatementNode == cutPoint.Child)
{
// shouldn't actually happen
jsonPossibleBranch = SemanticOperatorHelper.EvaluateStatementNode(possibleBranch, PyProcess, random, problemData, variables, variableSettings, Timeout);
}
else
{
var parent = possibleBranch.Parent; // save parent
cutPoint.Parent.InsertSubtree(cutPoint.ChildIndex, possibleBranch); // this will affect node.Parent
jsonPossibleBranch = SemanticOperatorHelper.EvaluateStatementNode(curStatementNode, PyProcess, random, problemData, variables, variableSettings, Timeout);
cutPoint.Parent.RemoveSubtree(cutPoint.ChildIndex); // removes intermediate parent from node
possibleBranch.Parent = parent; // restore parent
}
if (!String.IsNullOrWhiteSpace((string)jsonPossibleBranch["exception"]))
{
continue;
}
if (JToken.EqualityComparer.Equals(jsonCur, jsonPossibleBranch))
{
continue;
} // equal json, therefore continue
cutPoint.Parent.InsertSubtree(cutPoint.ChildIndex, cutPoint.Child); // restore cutPoint
selectedBranch = possibleBranch;
selectedCutPoint = cutPoint;
return;
}
cutPoint.Parent.InsertSubtree(cutPoint.ChildIndex, cutPoint.Child); // restore cutPoint
}
// no difference was found with any comparison, select randomly
// only select form the first cut point, as the other cut points might not have allowedBranchesPerCutpoint
selectedBranch = allowedBranchesPerCutpoint[0].SampleRandom(random);
selectedCutPoint = crossoverPoints0[0];
}
/// <summary>
/// 1. find a mutation point
/// 2. generate new random tree
/// 3. calculate semantic of old and new subtree (or of the closest parent)
/// 4. do mutation if semantically different
/// 5. retry until a certain number of tries is reached
///
/// if no mutation has happened, do random mutation
/// </summary>
public static void ReplaceSemanticallyDifferentBranch(IRandom random, ISymbolicExpressionTree symbolicExpressionTree, ICFGPythonProblemData problemData, ItemArray <PythonStatementSemantic> semantics, PythonProcess pythonProcess, double timeout, int maxTreeLength, int maxTreeDepth, int maximumSemanticTries)
{
if (semantics == null || semantics.Length == 0)
{
ReplaceBranchManipulation.ReplaceRandomBranch(random, symbolicExpressionTree, maxTreeLength, maxTreeDepth);
return;
}
var statementProductionNames = SemanticOperatorHelper.GetSemanticProductionNames(symbolicExpressionTree.Root.Grammar);
var variables = problemData.Variables.GetVariableNames().ToList();
string variableSettings = problemData.VariableSettings.Count == 0 ? String.Empty : String.Join(Environment.NewLine, problemData.VariableSettings.Select(x => x.Value));
var allowedSymbols = new List <ISymbol>();
// repeat until a fitting parent and child are found (MAX_TRIES times)
int tries = 0;
int semanticTries = 0;
do
{
#region find mutation point
#pragma warning disable 612, 618
ISymbolicExpressionTreeNode parent = symbolicExpressionTree.Root.IterateNodesPrefix().Skip(1).Where(n => n.SubtreeCount > 0).SelectRandom(random);
#pragma warning restore 612, 618
int childIndex = random.Next(parent.SubtreeCount);
var child = parent.GetSubtree(childIndex);
int maxLength = maxTreeLength - symbolicExpressionTree.Length + child.GetLength();
int maxDepth = maxTreeDepth - symbolicExpressionTree.Root.GetBranchLevel(child);
allowedSymbols.Clear();
foreach (var symbol in parent.Grammar.GetAllowedChildSymbols(parent.Symbol, childIndex))
{
// check basic properties that the new symbol must have
if ((symbol.Name != child.Symbol.Name || symbol.MinimumArity > 0) &&
symbol.InitialFrequency > 0 &&
parent.Grammar.GetMinimumExpressionDepth(symbol) <= maxDepth &&
parent.Grammar.GetMinimumExpressionLength(symbol) <= maxLength)
{
allowedSymbols.Add(symbol);
}
}
#endregion
#region check for semantic difference with a new random tree
if (allowedSymbols.Count > 0)
{
if (semanticTries <= maximumSemanticTries)
{
// do semantic mutation
#region calculate original json output
ISymbolicExpressionTreeNode statement = SemanticOperatorHelper.GetStatementNode(child, statementProductionNames);
var statementPos0 = symbolicExpressionTree.IterateNodesPrefix().ToList().IndexOf(statement);
if (String.IsNullOrEmpty(variableSettings))
{
variableSettings = SemanticOperatorHelper.SemanticToPythonVariableSettings(semantics.First(x => x.TreeNodePrefixPos == statementPos0).Before, problemData.Variables.GetVariableTypes());
}
var jsonOriginal = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout);
#endregion
var seedNode = GenerateAndInsertNewSubtree(random, parent, allowedSymbols, childIndex, maxLength, maxDepth);
#region calculate new json output
JObject jsonReplaced;
if (child == statement)
{
// child is executable, so is the new child
jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(seedNode, pythonProcess, random, problemData, variables, variableSettings, timeout);
}
else
{
jsonReplaced = SemanticOperatorHelper.EvaluateStatementNode(statement, pythonProcess, random, problemData, variables, variableSettings, timeout);
}
if (JToken.EqualityComparer.Equals(jsonOriginal, jsonReplaced))
{
// semantically equivalent. undo mutation
parent.RemoveSubtree(childIndex);
parent.InsertSubtree(childIndex, child);
allowedSymbols.Clear();
}
else
{
Console.WriteLine("never happens?");
}
if (problemData.VariableSettings.Count == 0)
{
// reset variableSettings
variableSettings = String.Empty;
}
semanticTries++;
#endregion
}
else
{
// do random mutation
GenerateAndInsertNewSubtree(random, parent, allowedSymbols, childIndex, maxLength, maxDepth);
}
}
#endregion
tries++;
} while (tries < MAX_TRIES && allowedSymbols.Count == 0 && semanticTries <= maximumSemanticTries);
}
public abstract void ReplaceBranch(IRandom random, ISymbolicExpressionTree symbolicExpressionTree, ICFGPythonProblemData problemData, ItemArray <PythonStatementSemantic> semantics, PythonProcess pythonProcess, double timeout, int maxTreeLength, int maxTreeDepth, int maximumSemanticTries);
