// ugly regexp matching of clousot output to get our results
public void ParseAndLogResults(string analysisResults, WarningInfo log)
{
Contract.Requires(analysisResults != null);
Contract.Requires(log != null);
var hashNameDiscrepancies = Regex.Matches(analysisResults, "HashName Discrepancy:(.*)");
List <string> interestingMethods = new List <string>();
foreach (Match match in hashNameDiscrepancies)
{
interestingMethods.Add(match.Groups[1].Value.ToString());
}
var methodMatchFailures = Regex.Match(analysisResults, "Failed to match ([0-9]+) methods");
int matchFailures = 0;
if (methodMatchFailures.Success)
{
matchFailures = Convert.ToInt32(methodMatchFailures.Groups[1].Value);
}
log.LogMethodMatchFailures(matchFailures);
int assertionsChecked = 0, assertionsCorrect = 0, assertionsUnknown = 0, assertionsFalse = 0, assertionsMasked = 0, assertionsUnreached = 0;
int index = analysisResults.IndexOf("OutputSuggestion,,Checked ");
if (index != -1)
{
string resultsLine = analysisResults.Substring(index);
Console.WriteLine(resultsLine);
var assertionsCheckedMatch = Regex.Match(resultsLine, "Checked ([0-9]+) assertion");
var assertionsCorrectMatch = Regex.Match(resultsLine, "([0-9]+) correct");
var assertionsUnknownMatch = Regex.Match(resultsLine, "([0-9]+) unknown");
var assertionsFalseMatch = Regex.Match(resultsLine, "([0-9]+) false");
var assertionsMaskedMatch = Regex.Match(resultsLine, "([0-9]+) masked");
var assertionsUnreachedMatch = Regex.Match(resultsLine, "([0-9]+) unreached");
if (assertionsCheckedMatch.Success)
{
assertionsChecked = Convert.ToInt32(assertionsCheckedMatch.Groups[1].Value);
}
if (assertionsCorrectMatch.Success)
{
assertionsCorrect = Convert.ToInt32(assertionsCorrectMatch.Groups[1].Value);
}
if (assertionsUnknownMatch.Success)
{
assertionsUnknown = Convert.ToInt32(assertionsUnknownMatch.Groups[1].Value);
}
if (assertionsFalseMatch.Success)
{
assertionsFalse = Convert.ToInt32(assertionsFalseMatch.Groups[1].Value);
}
if (assertionsMaskedMatch.Success)
{
assertionsMasked = Convert.ToInt32(assertionsMaskedMatch.Groups[1].Value);
}
if (assertionsUnreachedMatch.Success)
{
assertionsUnreached = Convert.ToInt32(assertionsUnreachedMatch.Groups[1].Value);
}
}
log.LogWarningInfo(assertionsChecked, assertionsCorrect, assertionsUnknown, assertionsFalse, assertionsMasked, assertionsUnreached, interestingMethods);
}
// do the actual work of baselining
private void BaselineImpl(bool rebuild, string baselineStrategy, int?startAtVersion = -1)
{
List <string> reversed = new List <string>();
// version numbers are most recent first by default; reverse this ordering
foreach (string vnum in Repo.GetVersionNumbers())
{
reversed.Add(vnum);
}
reversed.Reverse();
List <string> baselines = new List <string>(); // holds baseline files of previous revision
List <string> nextBaselines = new List <string>(); // holds baseline files of current revision
// set up logging
string logFileName = Path.Combine(Repo.WorkingDirectory, baselineStrategy + "_log.txt");
var logFile = File.AppendText(logFileName);
logFile.AutoFlush = true;
// for each version of the program from first to last
foreach (string version in reversed)
{
if (Convert.ToInt32(version) < startAtVersion)
{
continue;
}
Console.WriteLine("Baselining " + version);
WarningInfo log = new WarningInfo(version);
if (!Repo.CheckedOut(version))
{
Contract.Assert(Repo.Checkout(version) != null); // must check out repo in order to baseline it!
}
if (rebuild || !Repo.Built(version))
{
Console.WriteLine("version " + version + " not built; building");
if (!Repo.Build(version))
{
Console.WriteLine("Build of version " + version + " failed!");
continue;
}
//Contract.Assert(Build(version)); // must successfully build repo in order to baseline it!
}
string versionDir = Repo.MakeRepoPath(version);
// get repro.bat file for all built projects
List <string> repros = Util.GetRepros(versionDir);
// else, run each repro.bat with the baseline flag pointing at the baseline file for the previous version, if such a file exists
foreach (string repro in repros)
{
bool newProj = false;
string baselineFileName = repro.Replace("repro.bat", "base.xml");
nextBaselines.Add(baselineFileName);
// cleanup old baseline files to avoid accidentally suppressing all errors or other weirdness
if (File.Exists(baselineFileName))
{
File.Delete(baselineFileName);
}
if (File.Exists(baselineFileName + ".new"))
{
File.Delete(baselineFileName + ".new");
}
string stem = Path.Combine(Repo.WorkingDirectory, Repo.Name);
var match = Regex.Match(repro.Replace("repro.bat", ""), "[0-9]+(.*)");
Contract.Assert(match.Success);
string repoStem = match.Groups[1].Value;
repoStem = repoStem.Substring(0, repoStem.IndexOf("obj")); // try to make stem robust to silly changes in hierarchy of binary directory
// try to be robust to changes like changing from '\Theme\' to '\Source\Theme'
string altRepoStem = repoStem.Substring(repoStem.IndexOf('\\', 1));
//Console.WriteLine("repoStem is " + repoStem);
//Console.WriteLine("altRepoStem is " + altRepoStem);
string baselineMatch = null;
// pick appropriate baseline for this repro file
foreach (string baseline in baselines)
{
Console.WriteLine("baseline is " + baseline);
if (baseline.Contains(repoStem))
{
// only one baseline should match the stem; otherwise it's ambiguous which one we should use
Contract.Assert(baselineMatch == null, "ambiguous which baseline file to use " + baselineMatch + " or " + baseline);
baselineMatch = baseline;
}
}
// try alternate baseline matching if needed
if (baselineMatch == null && altRepoStem.Length > 2)
{
foreach (string baseline in baselines)
{
Console.WriteLine("baseline is " + baseline);
if (baseline.Contains(altRepoStem))
{
baselineMatch = baseline;
}
}
}
if (baselineMatch == null)
{
Console.WriteLine("Found new project " + repro + "; using empty baseline");
// we have no previous baseline file for this project; it must be new
baselineMatch = baselineFileName;
newProj = true;
}
Contract.Assert(baselineMatch != null, "No baseline file matches stem " + repoStem);
Console.WriteLine("Using baseline file " + baselineMatch);
Console.WriteLine("Running analysis...");
string analysisResults = RunAnalysisWithBaseLine(repro, baselineMatch, baselineStrategy, newProj);
ParseAndLogResults(analysisResults, log);
Console.WriteLine("Done.");
}
// update baseline files
baselines = nextBaselines;
nextBaselines = new List <string>();
// write log data
logFile.WriteLine(log.DumpToCSV());
}
}
// do the actual work of baselining
private void BaselineImpl(bool rebuild, string baselineStrategy, int? startAtVersion = -1)
{
List<string> reversed = new List<string>();
// version numbers are most recent first by default; reverse this ordering
foreach (string vnum in Repo.GetVersionNumbers()) reversed.Add(vnum);
reversed.Reverse();
List<string> baselines = new List<string>(); // holds baseline files of previous revision
List<string> nextBaselines = new List<string>(); // holds baseline files of current revision
// set up logging
string logFileName = Path.Combine(Repo.WorkingDirectory, baselineStrategy + "_log.txt");
var logFile = File.AppendText(logFileName);
logFile.AutoFlush = true;
// for each version of the program from first to last
foreach (string version in reversed)
{
if (Convert.ToInt32(version) < startAtVersion) continue;
Console.WriteLine("Baselining " + version);
WarningInfo log = new WarningInfo(version);
if (!Repo.CheckedOut(version)) Contract.Assert(Repo.Checkout(version) != null); // must check out repo in order to baseline it!
if (rebuild || !Repo.Built(version))
{
Console.WriteLine("version " + version + " not built; building");
if (!Repo.Build(version))
{
Console.WriteLine("Build of version " + version + " failed!");
continue;
}
//Contract.Assert(Build(version)); // must successfully build repo in order to baseline it!
}
string versionDir = Repo.MakeRepoPath(version);
// get repro.bat file for all built projects
List<string> repros = Util.GetRepros(versionDir);
// else, run each repro.bat with the baseline flag pointing at the baseline file for the previous version, if such a file exists
foreach (string repro in repros)
{
bool newProj = false;
string baselineFileName = repro.Replace("repro.bat", "base.xml");
nextBaselines.Add(baselineFileName);
// cleanup old baseline files to avoid accidentally suppressing all errors or other weirdness
if (File.Exists(baselineFileName))
{
File.Delete(baselineFileName);
}
if (File.Exists(baselineFileName + ".new"))
{
File.Delete(baselineFileName + ".new");
}
string stem = Path.Combine(Repo.WorkingDirectory, Repo.Name);
var match = Regex.Match(repro.Replace("repro.bat", ""), "[0-9]+(.*)");
Contract.Assert(match.Success);
string repoStem = match.Groups[1].Value;
repoStem = repoStem.Substring(0, repoStem.IndexOf("obj")); // try to make stem robust to silly changes in hierarchy of binary directory
// try to be robust to changes like changing from '\Theme\' to '\Source\Theme'
string altRepoStem = repoStem.Substring(repoStem.IndexOf('\\', 1));
//Console.WriteLine("repoStem is " + repoStem);
//Console.WriteLine("altRepoStem is " + altRepoStem);
string baselineMatch = null;
// pick appropriate baseline for this repro file
foreach (string baseline in baselines)
{
Console.WriteLine("baseline is " + baseline);
if (baseline.Contains(repoStem))
{
// only one baseline should match the stem; otherwise it's ambiguous which one we should use
Contract.Assert(baselineMatch == null, "ambiguous which baseline file to use " + baselineMatch + " or " + baseline);
baselineMatch = baseline;
}
}
// try alternate baseline matching if needed
if (baselineMatch == null && altRepoStem.Length > 2)
{
foreach (string baseline in baselines)
{
Console.WriteLine("baseline is " + baseline);
if (baseline.Contains(altRepoStem))
{
baselineMatch = baseline;
}
}
}
if (baselineMatch == null)
{
Console.WriteLine("Found new project " + repro + "; using empty baseline");
// we have no previous baseline file for this project; it must be new
baselineMatch = baselineFileName;
newProj = true;
}
Contract.Assert(baselineMatch != null, "No baseline file matches stem " + repoStem);
Console.WriteLine("Using baseline file " + baselineMatch);
Console.WriteLine("Running analysis...");
string analysisResults = RunAnalysisWithBaseLine(repro, baselineMatch, baselineStrategy, newProj);
ParseAndLogResults(analysisResults, log);
Console.WriteLine("Done.");
}
// update baseline files
baselines = nextBaselines;
nextBaselines = new List<string>();
// write log data
logFile.WriteLine(log.DumpToCSV());
}
}
// ugly regexp matching of clousot output to get our results
public void ParseAndLogResults(string analysisResults, WarningInfo log)
{
Contract.Requires(analysisResults != null);
Contract.Requires(log != null);
var hashNameDiscrepancies = Regex.Matches(analysisResults, "HashName Discrepancy:(.*)");
List<string> interestingMethods = new List<string>();
foreach (Match match in hashNameDiscrepancies)
{
interestingMethods.Add(match.Groups[1].Value.ToString());
}
var methodMatchFailures = Regex.Match(analysisResults, "Failed to match ([0-9]+) methods");
int matchFailures = 0;
if (methodMatchFailures.Success) matchFailures = Convert.ToInt32(methodMatchFailures.Groups[1].Value);
log.LogMethodMatchFailures(matchFailures);
int assertionsChecked = 0, assertionsCorrect = 0, assertionsUnknown = 0, assertionsFalse = 0, assertionsMasked = 0, assertionsUnreached = 0;
int index = analysisResults.IndexOf("OutputSuggestion,,Checked ");
if (index != -1)
{
string resultsLine = analysisResults.Substring(index);
Console.WriteLine(resultsLine);
var assertionsCheckedMatch = Regex.Match(resultsLine, "Checked ([0-9]+) assertion");
var assertionsCorrectMatch = Regex.Match(resultsLine, "([0-9]+) correct");
var assertionsUnknownMatch = Regex.Match(resultsLine, "([0-9]+) unknown");
var assertionsFalseMatch = Regex.Match(resultsLine, "([0-9]+) false");
var assertionsMaskedMatch = Regex.Match(resultsLine, "([0-9]+) masked");
var assertionsUnreachedMatch = Regex.Match(resultsLine, "([0-9]+) unreached");
if (assertionsCheckedMatch.Success) assertionsChecked = Convert.ToInt32(assertionsCheckedMatch.Groups[1].Value);
if (assertionsCorrectMatch.Success) assertionsCorrect = Convert.ToInt32(assertionsCorrectMatch.Groups[1].Value);
if (assertionsUnknownMatch.Success) assertionsUnknown = Convert.ToInt32(assertionsUnknownMatch.Groups[1].Value);
if (assertionsFalseMatch.Success) assertionsFalse = Convert.ToInt32(assertionsFalseMatch.Groups[1].Value);
if (assertionsMaskedMatch.Success) assertionsMasked = Convert.ToInt32(assertionsMaskedMatch.Groups[1].Value);
if (assertionsUnreachedMatch.Success) assertionsUnreached = Convert.ToInt32(assertionsUnreachedMatch.Groups[1].Value);
}
log.LogWarningInfo(assertionsChecked, assertionsCorrect, assertionsUnknown, assertionsFalse, assertionsMasked, assertionsUnreached, interestingMethods);
}