public void Reevaluate(ICFGPythonProblemData problemData, double timeout, PythonProcess pythonProcess) {
foreach (var key in ResultKeys) {
if (this.ContainsKey(key)) { this.Remove(key); }
}
Evaluate(problemData, (this[ProgramResultName] as TextValue).Value, timeout, pythonProcess);
}
protected PythonProcess(PythonProcess original, Cloner cloner)
: base(original, cloner)
{
executable = original.executable;
arguments = original.arguments;
degreeOfParallelism = original.degreeOfParallelism;
UpdateName();
evaluationThreadPool = new EvaluationThreadPool(executable, arguments, degreeOfParallelism);
}
public void Reevaluate(ICFGPythonProblemData problemData, double timeout, PythonProcess pythonProcess)
{
foreach (var key in ResultKeys)
{
if (this.ContainsKey(key))
{
this.Remove(key);
}
}
Evaluate(problemData, (this[ProgramResultName] as TextValue).Value, timeout, pythonProcess);
}
public CFGPythonSolution(ISymbolicExpressionTree tree, ICFGPythonProblemData problemData, double timeout, PythonProcess pythonProcess)
: base(tree, problemData) {
name = ItemName;
description = ItemDescription;
string program = PythonHelper.FormatToProgram(tree, problemData.FullHeader, problemData.FullFooter);
Add(new Result(ProgramResultName, "The program with header and footer", new TextValue(program)));
string code = CFGSymbolicExpressionTreeStringFormatter.StaticFormat(tree);
Add(new Result(CodeResultName, "The code that was evolved", new TextValue(code)));
Evaluate(problemData, program, timeout, pythonProcess);
}
public CFGPythonSolution(ISymbolicExpressionTree tree, ICFGPythonProblemData problemData, double timeout, PythonProcess pythonProcess)
: base(tree, problemData)
{
name = ItemName;
description = ItemDescription;
string program = PythonHelper.FormatToProgram(tree, problemData.LoopBreakConst, problemData.FullHeader, problemData.FullFooter);
Add(new Result(ProgramResultName, "The program with header and footer", new TextValue(program)));
string code = CFGSymbolicExpressionTreeStringFormatter.StaticFormat(tree);
Add(new Result(CodeResultName, "The code that was evolved", new TextValue(code)));
Evaluate(problemData, program, timeout, pythonProcess);
}
public void Evaluate(ICFGPythonProblemData problemData, string program, double timeout, PythonProcess pythonProcess)
{
var trainingTimeout = timeout * 2; // increase timeout to make sure it finishes
var training = pythonProcess.EvaluateProgram(program, problemData.Input, problemData.Output, problemData.TrainingIndices, trainingTimeout);
// test timeout should be proportionally bigger than training timeout
var testTimeout = (double)problemData.TestIndices.Count() / (double)problemData.TrainingIndices.Count() * trainingTimeout;
testTimeout = testTimeout > timeout ? testTimeout : timeout;
var test = pythonProcess.EvaluateProgram(program, problemData.Input, problemData.Output, problemData.TestIndices, testTimeout);
if (String.IsNullOrEmpty(training.Item4))
{
Add(new Result(TrainingQuality, "Training quality", new DoubleValue(training.Item3)));
var cases = training.Item1.ToArray();
Add(new Result(TrainingSolvedCases, "Training cases which have been solved", new BoolArray(cases)));
Add(new Result(TrainingSolvedCasesPercentage, "Percentage of training cases which have been solved", new PercentValue((double)cases.Count(x => x) / (double)cases.Length)));
Add(new Result(TrainingSolvedCaseQualities, "The quality of each training case", new DoubleArray(training.Item2.ToArray())));
}
else
{
Add(new Result(TrainingException, "Exception occured during training", new TextValue(training.Item4)));
Add(new Result(TrainingQuality, "Training quality", new DoubleValue(Double.NaN)));
Add(new Result(TrainingSolvedCases, "Training cases which have been solved", new BoolArray(problemData.TrainingIndices.Count())));
Add(new Result(TrainingSolvedCasesPercentage, "Percentage of training cases which have been solved", new PercentValue(0)));
Add(new Result(TrainingSolvedCaseQualities, "The quality of each training case", new DoubleArray(Enumerable.Repeat(Double.NaN, problemData.TrainingIndices.Count()).ToArray())));
}
if (String.IsNullOrEmpty(test.Item4))
{
Add(new Result(TestQuality, "Test quality", new DoubleValue(test.Item3)));
var cases = test.Item1.ToArray();
Add(new Result(TestSolvedCases, "Test cases which have been solved", new BoolArray(cases)));
Add(new Result(TestSolvedCasesPercentage, "Percentage of test cases which have been solved", new PercentValue((double)cases.Count(x => x) / (double)cases.Length)));
Add(new Result(TestSolvedCaseQualities, "The quality of each test case", new DoubleArray(test.Item2.ToArray())));
}
else
{
Add(new Result(TestException, "Exception occured during test", new TextValue(test.Item4)));
Add(new Result(TestQuality, "Test quality", new DoubleValue(Double.NaN)));
Add(new Result(TestSolvedCases, "Test cases which have been solved", new BoolArray(problemData.TestIndices.Count())));
Add(new Result(TestSolvedCasesPercentage, "Percentage of test cases which have been solved", new PercentValue(0)));
Add(new Result(TestSolvedCaseQualities, "The quality of each test case", new DoubleArray(Enumerable.Repeat(Double.NaN, problemData.TestIndices.Count()).ToArray())));
}
}
protected PythonProcess(PythonProcess original, Cloner cloner)
: base(original, cloner) {
executable = original.executable;
arguments = original.arguments;
degreeOfParallelism = original.degreeOfParallelism;
UpdateName();
evaluationThreadPool = new EvaluationThreadPool(executable, arguments, degreeOfParallelism);
}
public void Evaluate(ICFGPythonProblemData problemData, string program, double timeout, PythonProcess pythonProcess) {
var trainingTimeout = timeout * 2; // increase timeout to make sure it finishes
var training = pythonProcess.EvaluateProgram(program, problemData.Input, problemData.Output, problemData.TrainingIndices, trainingTimeout);
// test timeout should be proportionally bigger than training timeout
var testTimeout = (double)problemData.TestIndices.Count() / (double)problemData.TrainingIndices.Count() * trainingTimeout;
testTimeout = testTimeout > timeout ? testTimeout : timeout;
var test = pythonProcess.EvaluateProgram(program, problemData.Input, problemData.Output, problemData.TestIndices, testTimeout);
if (String.IsNullOrEmpty(training.Item4)) {
Add(new Result(TrainingQuality, "Training quality", new DoubleValue(training.Item3)));
var cases = training.Item1.ToArray();
Add(new Result(TrainingSolvedCases, "Training cases which have been solved", new BoolArray(cases)));
Add(new Result(TrainingSolvedCasesPercentage, "Percentage of training cases which have been solved", new PercentValue((double)cases.Count(x => x) / (double)cases.Length)));
Add(new Result(TrainingSolvedCaseQualities, "The quality of each training case", new DoubleArray(training.Item2.ToArray())));
} else {
Add(new Result(TrainingException, "Exception occured during training", new TextValue(training.Item4)));
Add(new Result(TrainingQuality, "Training quality", new DoubleValue(Double.NaN)));
Add(new Result(TrainingSolvedCases, "Training cases which have been solved", new BoolArray(problemData.TrainingIndices.Count())));
Add(new Result(TrainingSolvedCasesPercentage, "Percentage of training cases which have been solved", new PercentValue(0)));
Add(new Result(TrainingSolvedCaseQualities, "The quality of each training case", new DoubleArray(Enumerable.Repeat(Double.NaN, problemData.TrainingIndices.Count()).ToArray())));
}
if (String.IsNullOrEmpty(test.Item4)) {
Add(new Result(TestQuality, "Test quality", new DoubleValue(test.Item3)));
var cases = test.Item1.ToArray();
Add(new Result(TestSolvedCases, "Test cases which have been solved", new BoolArray(cases)));
Add(new Result(TestSolvedCasesPercentage, "Percentage of test cases which have been solved", new PercentValue((double)cases.Count(x => x) / (double)cases.Length)));
Add(new Result(TestSolvedCaseQualities, "The quality of each test case", new DoubleArray(test.Item2.ToArray())));
} else {
Add(new Result(TestException, "Exception occured during test", new TextValue(test.Item4)));
Add(new Result(TestQuality, "Test quality", new DoubleValue(Double.NaN)));
Add(new Result(TestSolvedCases, "Test cases which have been solved", new BoolArray(problemData.TestIndices.Count())));
Add(new Result(TestSolvedCasesPercentage, "Percentage of test cases which have been solved", new PercentValue(0)));
Add(new Result(TestSolvedCaseQualities, "The quality of each test case", new DoubleArray(Enumerable.Repeat(Double.NaN, problemData.TestIndices.Count()).ToArray())));
}
}
public void Initialize() {
pp = new PythonProcess("python.exe", "");
pp.DegreeOfParallelism = 1;
}