public void MeasureProcessExitCode()
{
Measurement.ProcessRunMeasure m = ProcessMeasurer.Measure("Delay.exe", "10000 extraArgument", TimeSpan.FromMilliseconds(10000));
Assert.AreEqual(Measure.LimitsStatus.WithinLimits, m.Limits);
Assert.AreEqual(42, m.ExitCode, "Exit code");
Assert.IsTrue(m.PeakMemorySizeMB > 1, "Memory size seems too low");
var ptime = m.TotalProcessorTime.TotalMilliseconds;
var wctime = m.WallClockTime.TotalMilliseconds;
Assert.IsTrue(ptime > 0 && ptime <= 100, "Total processor time");
Assert.IsTrue(wctime > 0 && wctime <= 10000, "Wall-clock time must be greater than given timeout");
StreamReader reader = new StreamReader(m.StdOut);
string output = reader.ReadToEnd();
Assert.IsTrue(output.Contains("Use: Delay.exe"), "Output must contain certain text.");
reader = new StreamReader(m.StdErr);
string error = reader.ReadToEnd();
Assert.IsTrue(String.IsNullOrEmpty(error), "Error output");
}
public void MeasureProcessRun()
{
Measurement.ProcessRunMeasure m = ProcessMeasurer.Measure("Delay.exe", "100", TimeSpan.FromMilliseconds(1000));
Assert.AreEqual(0, m.ExitCode, "Exit code");
Assert.AreEqual(Measure.LimitsStatus.WithinLimits, m.Limits);
Assert.IsTrue(m.PeakMemorySizeMB > 1, "Memory size seems too low");
var ptime = m.TotalProcessorTime.TotalMilliseconds;
var wctime = m.WallClockTime.TotalMilliseconds;
Assert.IsTrue(ptime <= 1000 && wctime >= ptime, "Total processor time must be very small because Delay.exe mostly sleeps but it is " + ptime);
Assert.IsTrue(wctime >= 100, "Wall-clock time must be greater than given delay");
Assert.IsTrue(wctime < 1000, "Wall-clock time must be less");
StreamReader reader = new StreamReader(m.StdOut);
string output = reader.ReadToEnd();
Assert.IsTrue(output.Contains("Done."), "Output must contain certain text.");
reader = new StreamReader(m.StdErr);
string error = reader.ReadToEnd();
Assert.IsTrue(error.Contains("Sample error text."), "Error output must contain certain text.");
}
public void ZipWithMultipleExecutablesWithoutRelationshipFail()
{
var zipname = Path.GetTempFileName();
using (var zip = new ZipFile())
{
zip.AddFiles(new string[] { "Delay.exe", "FailingTool.exe" });
zip.Save(zipname);
}
bool fail = false;
try
{
Measurement.ProcessRunMeasure m = ProcessMeasurer.Measure(zipname, "100", TimeSpan.FromMilliseconds(1000));
}
catch
{
fail = true;
}
finally
{
File.Delete(zipname);
}
Assert.IsTrue(fail, "Zip with multiple executables and no relationships should be considered incorrect.");
}
public void MeasureProcessRunFromPackage()
{
var zipname = Path.GetTempFileName();
using (var zip = new ZipFile())
{
zip.AddFiles(new string[] { "Delay.exe", "FailingTool.exe" });
string mainexe = "Delay.exe";
string content_types = "<?xml version=\"1.0\" encoding=\"utf-8\"?><Types xmlns=\"http://schemas.openxmlformats.org/package/2006/content-types\"><Default Extension=\"exe\" ContentType=\"application/octet-stream\" /><Default Extension=\"dll\" ContentType=\"application/octet-stream\" /><Default Extension=\"rels\" ContentType=\"application/vnd.openxmlformats-package.relationships+xml\" /></Types>";
string rels = "<?xml version=\"1.0\" encoding=\"utf-8\"?><Relationships xmlns=\"http://schemas.openxmlformats.org/package/2006/relationships\"><Relationship Type=\"http://schemas.openxmlformats.org/package/2006/relationships/meta data/thumbnail\" Target=\"/" + mainexe + "\" Id=\"R17bb7f6124fd45fe\" /></Relationships>";
zip.AddEntry("[Content_Types].xml", content_types);
zip.AddEntry("_rels\\.rels", rels);
zip.Save(zipname);
}
try
{
Measurement.ProcessRunMeasure m = ProcessMeasurer.Measure(zipname, "100", TimeSpan.FromMilliseconds(1000));
Assert.AreEqual(0, m.ExitCode, "Exit code");
Assert.AreEqual(Measure.LimitsStatus.WithinLimits, m.Limits);
Assert.IsTrue(m.PeakMemorySizeMB > 1, "Memory size seems too low");
var ptime = m.TotalProcessorTime.TotalMilliseconds;
var wctime = m.WallClockTime.TotalMilliseconds;
Assert.IsTrue(ptime <= 1000 && wctime >= ptime, "Total processor time must be very small because Delay.exe mostly sleeps but it is " + ptime);
Assert.IsTrue(wctime >= 100 && wctime < 1000, "Wall-clock time must be greater than given delay");
StreamReader reader = new StreamReader(m.StdOut);
string output = reader.ReadToEnd();
Assert.IsTrue(output.Contains("Done."), "Output must contain certain text.");
reader = new StreamReader(m.StdErr);
string error = reader.ReadToEnd();
Assert.IsTrue(error.Contains("Sample error text."), "Error output must contain certain text.");
}
finally
{
File.Delete(zipname);
}
}
public void MeasureProcessMemoryLimit()
{
Measurement.ProcessRunMeasure m = ProcessMeasurer.Measure("FailingTool.exe", "out-of-memory", TimeSpan.FromMinutes(10), memoryLimit: 1);
Assert.AreEqual(Measure.LimitsStatus.MemoryOut, m.Limits);
Assert.IsTrue(m.ExitCode == null, "Exit code");
Assert.IsTrue(m.PeakMemorySizeMB > 1, "Memory size seems too low");
var ptime = m.TotalProcessorTime.TotalMilliseconds;
var wctime = m.WallClockTime.TotalMilliseconds;
Assert.IsTrue(ptime <= 100000, "Total processor time");
Assert.IsTrue(wctime <= 100000, "Wall-clock time");
StreamReader reader = new StreamReader(m.StdOut);
string output = reader.ReadToEnd();
reader = new StreamReader(m.StdErr);
string error = reader.ReadToEnd();
}
private ResultStatus GetBugCode(ProcessRunMeasure measure)
{
ResultStatus status = ResultStatus.Error; // no bug found means general error.
StreamReader reader = new StreamReader(measure.StdErr);
while (!reader.EndOfStream)
{
string l = reader.ReadLine();
if (l.StartsWith("(error") && l.Contains("check annotation"))
{
status = ResultStatus.Bug;
break;
}
}
measure.StdErr.Position = 0L;
if (status == ResultStatus.Error)
{
reader = new StreamReader(measure.StdOut);
while (!reader.EndOfStream)
{
string l = reader.ReadLine();
if (l.StartsWith("(error") && l.Contains("check annotation"))
{
status = ResultStatus.Bug;
break;
}
else if (l.StartsWith("(error \"out of memory\")"))
{
status = ResultStatus.OutOfMemory;
break;
}
}
measure.StdOut.Position = 0L;
}
return(status);
}
public override ProcessRunAnalysis Analyze(string inputFile, ProcessRunMeasure measure)
{
ResultStatus status;
switch (measure.Limits)
{
case Measure.LimitsStatus.WithinLimits:
status = measure.ExitCode == 0 ? ResultStatus.Success : ResultStatus.Error;
break;
case Measure.LimitsStatus.MemoryOut:
status = ResultStatus.OutOfMemory;
break;
case Measure.LimitsStatus.TimeOut:
status = ResultStatus.Timeout;
break;
default:
throw new NotSupportedException("Unknown status");
}
return(new ProcessRunAnalysis(status, new Dictionary <string, string>()));
}
public void MeasureProcessOutOfMemory()
{
Measurement.ProcessRunMeasure m = ProcessMeasurer.Measure("FailingTool.exe", "out-of-memory", TimeSpan.FromMinutes(10));
Assert.AreEqual(Measure.LimitsStatus.WithinLimits, m.Limits);
Assert.IsTrue(m.ExitCode < 0, "Exit code");
Assert.IsTrue(m.PeakMemorySizeMB > 1, "Memory size seems too low");
var ptime = m.TotalProcessorTime.TotalMilliseconds;
var wctime = m.WallClockTime.TotalMilliseconds;
Assert.IsTrue(ptime <= 100000, "Total processor time");
Assert.IsTrue(wctime <= 100000, "Wall-clock time");
StreamReader reader = new StreamReader(m.StdOut);
string output = reader.ReadToEnd();
reader = new StreamReader(m.StdErr);
string error = reader.ReadToEnd();
Assert.IsTrue(output.Contains("i = 0"), "Output");
Assert.IsTrue(error.Contains("OutOfMemoryException"), "Error must contain certain text.");
}
/// <summary> /// Determines status of the process run result and its custom domain-specific properties. /// </summary> public abstract ProcessRunAnalysis Analyze(string inputFile, ProcessRunMeasure measure);
public override ProcessRunAnalysis Analyze(string inputFile, ProcessRunMeasure measure)
{
if (!measure.StdOut.CanSeek)
{
throw new NotSupportedException("Standard output stream doesn't support seeking");
}
if (!measure.StdErr.CanSeek)
{
throw new NotSupportedException("Standard error stream doesn't support seeking");
}
measure.StdOut.Position = 0L;
measure.StdErr.Position = 0L;
Counts countsTargets;
try
{
using (Stream input = new FileStream(inputFile, FileMode.Open, FileAccess.Read))
{
countsTargets = CountInput(input);
}
}
catch (Exception ex)
{
Trace.WriteLine("Failed to read input file: " + ex);
countsTargets = new Counts();
}
Counts countsResults = CountResults(measure.StdOut);
int? exitCode = measure.ExitCode;
LimitsStatus limits = measure.Limits;
ResultStatus status;
if (limits == LimitsStatus.TimeOut)
{
status = ResultStatus.Timeout;
}
else if (limits == LimitsStatus.MemoryOut || exitCode == 101)
{
status = ResultStatus.OutOfMemory;
}
else if (exitCode == 0)
{
if (countsResults.sat == 0 && countsResults.unsat == 0 && countsResults.other == 0)
{
status = ResultStatus.Error;
}
else
{
status = ResultStatus.Success;
}
}
else
{
status = GetBugCode(measure);
}
return(new ProcessRunAnalysis(status,
new Dictionary <string, string>
{
{ KeySat, countsResults.sat.ToString() },
{ KeyUnsat, countsResults.unsat.ToString() },
{ KeyUnknown, countsResults.other.ToString() },
{ KeyTargetSat, countsTargets.sat.ToString() },
{ KeyTargetUnsat, countsTargets.unsat.ToString() },
{ KeyTargetUnknown, countsTargets.other.ToString() }
}));
}
