private static void CheckAssertionsInParallel(string directory, string tag, ProgramAttributes attributes)
{
//Parallel.For(0, attributes.numSplits,
// new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount },
// i => CheckAssertion(directory, tag, i, attributes));
var delim = Options.IsLinux() ? @"/" : @"\";
List<Tuple<string, string>> solvers = new List<Tuple<string, string>>();
AddSolver(solvers, "Z3_440", @"/z3exe:", "." + delim + "references" + delim + "z3.4.4.0.exe", "/z3opt:smt.RELEVANCY=0 /z3opt:smt.CASE_SPLIT=0");
AddSolver(solvers, "Z3_441", @"/z3exe:", "." + delim + "references" + delim + "z3.4.4.1.exe", "/z3opt:smt.RELEVANCY=0 /z3opt:smt.CASE_SPLIT=0");
AddSolver(solvers, "Z3_441_OPTIMIZE_BV", @"/z3exe:", "." + delim + "references" + delim + "z3.4.4.1.exe", "/proverOpt:OPTIMIZE_FOR_BV=true");
// work stealing parallel implementation
workItems = new System.Collections.Concurrent.ConcurrentBag<Tuple<string, string, int, ProgramAttributes>>();
foreach (Tuple<string, string> solver in solvers)
{
attributes.solverName = solver.Item1;
attributes.solverCallArg = solver.Item2;
for (int i = 0; i < attributes.numSplits; i++)
workItems.Add(Tuple.Create(directory, tag, i, attributes));
}
var threads = new List<Thread>();
for (int i = 0; i < Environment.ProcessorCount * 0.5; i++)
{
threads.Add(new Thread(new ThreadStart(CheckAssertions)));
}
threads.ForEach(t => t.Start());
threads.ForEach(t => t.Join());
}
public static Tuple<int, int, int> RunBenchmarks(List<Tuple<string, string, string, string>> benchmarks, bool doNotRunBenchmarks, bool splitMemory, bool optimizeStore, bool optimizeLoad)
{
//item1 = benchmarks\StackExample\func_0000000000001000, item2 = dllfunc.bpl, item3 = "", item4 = "baseline"
var delim = Options.IsLinux() ? @"/" : @"\";
int numVerified = 0, numError = 0, numUnknown = 0;
if (splitMemory)
{
optimizeLoad = true;
optimizeStore = true;
}
//item1: directory, item2: bpl file, item3: options, item4: tag
foreach (Tuple<string, string, string, string> benchmark in benchmarks)
{
if (!Directory.Exists(benchmark.Item1))
{
throw new Exception("Directory " + benchmark.Item1 + " does not exist");
}
else
{
Console.WriteLine("\n>>>>>>> Processing directory {0}", benchmark.Item1);
}
//Func<string, string> WrapPath = delegate(string s) { return benchmark.Item1 + separator + s; };
string args = @" " + benchmark.Item1 + delim + benchmark.Item2;
args += @" /instrumentedFile:" + benchmark.Item1 + delim + "dll_func_instrumented.bpl";
args += @" /splitFiles /splitFilesDir:" + benchmark.Item1 + " ";
args += @" " + benchmark.Item3;
args += @" /tag:" + benchmark.Item4;
if (splitMemory) { args += @" /splitMemoryModel+"; }
if (optimizeStore) { args += @" /optimizeStoreITE+"; }
if (optimizeLoad) { args += @" /optimizeLoadITE+"; }
ProgramAttributes attributes = ExecuteCfiVerifierBinary(args);
if (attributes.numSplits < 0)
{
resultFileNameWriter.WriteLine("-- benchmark " + benchmark.Item1 + " did not generate any assertions");
Console.WriteLine("Benchmark " + benchmark.Item1 + " did not generate any assertions");
continue;
}
Console.WriteLine("\tFOUND {0} assertions in benchmark {1}, Running them in parallel...",
attributes.numSplits,
benchmark.Item1);
if (!doNotRunBenchmarks) {
CheckAssertionsInParallel(benchmark.Item1, benchmark.Item4, attributes);
Tuple<int, int, int> stats = ComputeStatisticsForDirectory(benchmark.Item1);
numVerified += stats.Item1;
numError += stats.Item2;
numUnknown += stats.Item3;
EmitBenchmarkResults(resultFileName, benchmark.Item1);
}
// generate a script in case we want to run benchmarks later manually
//TextWriter fileWriter = new StreamWriter("script");
//for (int i = 0; i < attributes.numSplits; i++)
//{
// string boogie_args = @" " + benchmark.Item1 + delim + @"split_" + i.ToString() + @"." +
// benchmark.Item4 + ".bpl /timeLimit:" + Options.timeoutPerProcess +
// @" /contractInfer /z3opt:smt.RELEVANCY=0 /z3opt:smt.CASE_SPLIT=0 /errorLimit:1";
// string boogie_bin = @"." + delim + "references" + delim + "Boogie.exe";
// fileWriter.WriteLine(boogie_bin + boogie_args);
//}
//fileWriter.Flush();
//fileWriter.Close();
}
return new Tuple<int, int, int>(numVerified, numError, numUnknown);
}
// first: number of assertions, second: found a loop
private static ProgramAttributes ExecuteCfiVerifierBinary(string arguments)
{
string binaryName = GetBinaryPath() + @"CfiVerifier.exe";
//Func<string, string> ProcessOutput = delegate(string s) { return ("The number of lines in output = " + s.Split('\n').Count().ToString()); };
Func<string, int> BenchmarkCount = delegate(string s)
{
string HEADER = "VCSplitter generated ";
string BODY = " assertions";
if (s.Contains(HEADER) && s.Contains(BODY))
{
string countString = s.Split(new string[] { HEADER }, StringSplitOptions.None)[1].
Split(new string[] { BODY }, StringSplitOptions.None)[0];
return Int32.Parse(countString);
}
return -1;
};
Func<string, bool> FoundLoop = delegate(string s)
{
return (s.Contains("CfiVerifier found one or more loops"));
};
Func<string, int> BasicBlockCount = delegate(string s)
{
string HEADER = "CfiVerifier found ";
string BODY = " basic blocks";
if (s.Contains(HEADER) && s.Contains(BODY))
{
string countString = s.Split(new string[] { HEADER }, StringSplitOptions.None)[1].
Split(new string[] { BODY }, StringSplitOptions.None)[0];
return Int32.Parse(countString);
}
return -1;
};
Func<string, Dictionary<int, String>> AssertionType = delegate(string input)
{
String HEADER = "VCSplitter found split id ";
String BODY = " with type ";
//Console.WriteLine(s);
Dictionary<int,string> result = new Dictionary<int, string>();
String[] splits = input.Split(new String[] { HEADER }, StringSplitOptions.None);
foreach (String s in splits) {
if (!s.Contains(BODY)) { continue; }
int splitId = Int32.Parse(s.Split(new String[] { BODY }, StringSplitOptions.None)[0]);
string assertionType = s.Split(new String[] { BODY }, StringSplitOptions.None)[1];
result.Add(splitId, assertionType);
}
return result;
};
Console.WriteLine("\tSTART Executing {0} {1}", binaryName, arguments);
try
{
ProcessStartInfo procInfo = new ProcessStartInfo();
//System.Diagnostics.Process proc = new System.Diagnostics.Process();
procInfo.UseShellExecute = false;
procInfo.FileName = binaryName;
procInfo.Arguments = arguments;
procInfo.WindowStyle = ProcessWindowStyle.Hidden;
procInfo.RedirectStandardOutput = true;
Process proc = new Process();
proc.StartInfo = procInfo;
proc.EnableRaisingEvents = false;
proc.Start();
string output = "";
output = proc.StandardOutput.ReadToEnd();
proc.WaitForExit();
# region deprecated logic for forcing a timeout
//
// Caution: Using proc.WaitForExit(1200000) causes processes to be idle
//
//bool exitInTime = proc.WaitForExit(1200000);
//if (!exitInTime) { Console.WriteLine("TIMEOUT"); }
//while (proc.StandardOutput.Peek() > -1)
//{
// output = output + proc.StandardOutput.ReadLine() + "\n";
//}
//if (!exitInTime)
//{
// proc.Kill();
// return new Tuple<string, string, string, string>("TIMEOUT", "UNKNOWN", "TIMEOUT", "TIMEOUT");
//}
#endregion
Console.WriteLine("\tEND Executing {0} {1}", binaryName, arguments);
ProgramAttributes attributes = new ProgramAttributes();
attributes.foundLoop = FoundLoop(output);
attributes.numSplits = BenchmarkCount(output);
attributes.assertionTypes = AssertionType(output);
attributes.numBasicBlocks = BasicBlockCount(output);
return attributes;
}
private static void CheckAssertion(string directory, string tag, int splitId, ProgramAttributes attributes)
{
//join is commutative. join(VERIFIED,_) = VERIFIED. join(UNKNOWN,x) = x. join(ERROR,ERROR) = ERROR.
Func<BoogieResult, BoogieResult, BoogieResult> join = delegate(BoogieResult r1, BoogieResult r2)
{
if (r1 == BoogieResult.VERIFIED || r2 == BoogieResult.VERIFIED) { return BoogieResult.VERIFIED; }
else if (r1 == BoogieResult.UNKNOWN) { return r2; }
else if (r2 == BoogieResult.UNKNOWN) { return r1; }
else { return BoogieResult.ERROR; } //both r1 and r2 are ERROR in this case }
};
var delim = Options.IsLinux() ? @"/" : @"\";
string args = @" " + directory + delim + @"split_" + splitId.ToString() + @"." + tag + ".bpl " +
"/timeLimit:" + Options.timeoutPerProcess + " " + attributes.solverCallArg +
@" /contractInfer /z3opt:smt.RELEVANCY=0 /z3opt:smt.CASE_SPLIT=0 /errorLimit:1";
//string args1 = args0 + @" /contractInfer /z3opt:smt.RELEVANCY=0 /z3opt:smt.CASE_SPLIT=0 /errorLimit:1";
//item1: error / unknown / verified, item2: time spent in boogie
Tuple<BoogieResult, int> args1_result = ExecuteBoogieBinary(args);
RegisterResult(directory, tag, splitId, attributes, args1_result.Item1, args1_result.Item2);
}
