public void MeasureProcessExitCode()
{
Measurement.ProcessRunMeasure m = ProcessMeasurer.Measure("Delay.exe", "10000 extraArgument", TimeSpan.FromMilliseconds(10000));
Assert.AreEqual(Measure.LimitsStatus.WithinLimits, m.Limits);
Assert.AreEqual(42, m.ExitCode, "Exit code");
Assert.IsTrue(m.PeakMemorySizeMB > 1, "Memory size seems too low");
var ptime = m.TotalProcessorTime.TotalMilliseconds;
var wctime = m.WallClockTime.TotalMilliseconds;
Assert.IsTrue(ptime > 0 && ptime <= 100, "Total processor time");
Assert.IsTrue(wctime > 0 && wctime <= 10000, "Wall-clock time must be greater than given timeout");
StreamReader reader = new StreamReader(m.StdOut);
string output = reader.ReadToEnd();
Assert.IsTrue(output.Contains("Use: Delay.exe"), "Output must contain certain text.");
reader = new StreamReader(m.StdErr);
string error = reader.ReadToEnd();
Assert.IsTrue(String.IsNullOrEmpty(error), "Error output");
}
public void MeasureProcessRun()
{
Measurement.ProcessRunMeasure m = ProcessMeasurer.Measure("Delay.exe", "100", TimeSpan.FromMilliseconds(1000));
Assert.AreEqual(0, m.ExitCode, "Exit code");
Assert.AreEqual(Measure.LimitsStatus.WithinLimits, m.Limits);
Assert.IsTrue(m.PeakMemorySizeMB > 1, "Memory size seems too low");
var ptime = m.TotalProcessorTime.TotalMilliseconds;
var wctime = m.WallClockTime.TotalMilliseconds;
Assert.IsTrue(ptime <= 1000 && wctime >= ptime, "Total processor time must be very small because Delay.exe mostly sleeps but it is " + ptime);
Assert.IsTrue(wctime >= 100, "Wall-clock time must be greater than given delay");
Assert.IsTrue(wctime < 1000, "Wall-clock time must be less");
StreamReader reader = new StreamReader(m.StdOut);
string output = reader.ReadToEnd();
Assert.IsTrue(output.Contains("Done."), "Output must contain certain text.");
reader = new StreamReader(m.StdErr);
string error = reader.ReadToEnd();
Assert.IsTrue(error.Contains("Sample error text."), "Error output must contain certain text.");
}
public void ZipWithMultipleExecutablesWithoutRelationshipFail()
{
var zipname = Path.GetTempFileName();
using (var zip = new ZipFile())
{
zip.AddFiles(new string[] { "Delay.exe", "FailingTool.exe" });
zip.Save(zipname);
}
bool fail = false;
try
{
Measurement.ProcessRunMeasure m = ProcessMeasurer.Measure(zipname, "100", TimeSpan.FromMilliseconds(1000));
}
catch
{
fail = true;
}
finally
{
File.Delete(zipname);
}
Assert.IsTrue(fail, "Zip with multiple executables and no relationships should be considered incorrect.");
}
public static BenchmarkResult RunBenchmark(int experimentId, string executable, string args, string inputDisplayName, string inputFullPath, int repetitions, TimeSpan timeOut, double memLimitMB, long?ouputLimit, long?errorLimit, Domain domain, double normal, int maxRepetitions = 10, double maxTimeInSeconds = 10)
{
if (domain == null)
{
throw new ArgumentNullException("domain");
}
if (args != null)
{
args = args.Replace("{0}", inputFullPath);
}
else
{
args = "";
}
DateTime acq = DateTime.Now;
int maxCount = repetitions == 0 ? maxRepetitions : repetitions;
TimeSpan maxTime = TimeSpan.FromSeconds(maxTimeInSeconds);
int count = 0;
List <ProcessRunMeasure> measures = new List <ProcessRunMeasure>();
TimeSpan total = TimeSpan.FromSeconds(0);
ProcessRunAnalysis analysis = null;
ProcessRunMeasure m;
do
{
m = ProcessMeasurer.Measure(executable, args, timeOut, memLimitMB, ouputLimit, errorLimit, domain);
measures.Add(m);
count++;
total += m.WallClockTime;
if (analysis == null) // analyzed only once, repetitions are for more confident run time
{
analysis = domain.Analyze(inputFullPath, m);
}
} while ((repetitions != 0 || total < maxTime) && count < maxCount && m.Limits == Measure.LimitsStatus.WithinLimits);
ProcessRunMeasure finalMeasure = Utils.AggregateMeasures(measures.ToArray());
Trace.WriteLine(String.Format("Done in {0} (aggregated by {1} runs)", finalMeasure.WallClockTime, count));
var performanceIndex = normal * finalMeasure.TotalProcessorTime.TotalSeconds;
var result = new BenchmarkResult(
experimentId, inputDisplayName,
acq, performanceIndex,
finalMeasure.TotalProcessorTime, finalMeasure.WallClockTime, finalMeasure.PeakMemorySizeMB,
analysis.Status,
finalMeasure.ExitCode, finalMeasure.StdOut, finalMeasure.StdErr,
analysis.OutputProperties);
return(result);
}
public void MeasureProcessRunFromPackage()
{
var zipname = Path.GetTempFileName();
using (var zip = new ZipFile())
{
zip.AddFiles(new string[] { "Delay.exe", "FailingTool.exe" });
string mainexe = "Delay.exe";
string content_types = "<?xml version=\"1.0\" encoding=\"utf-8\"?><Types xmlns=\"http://schemas.openxmlformats.org/package/2006/content-types\"><Default Extension=\"exe\" ContentType=\"application/octet-stream\" /><Default Extension=\"dll\" ContentType=\"application/octet-stream\" /><Default Extension=\"rels\" ContentType=\"application/vnd.openxmlformats-package.relationships+xml\" /></Types>";
string rels = "<?xml version=\"1.0\" encoding=\"utf-8\"?><Relationships xmlns=\"http://schemas.openxmlformats.org/package/2006/relationships\"><Relationship Type=\"http://schemas.openxmlformats.org/package/2006/relationships/meta data/thumbnail\" Target=\"/" + mainexe + "\" Id=\"R17bb7f6124fd45fe\" /></Relationships>";
zip.AddEntry("[Content_Types].xml", content_types);
zip.AddEntry("_rels\\.rels", rels);
zip.Save(zipname);
}
try
{
Measurement.ProcessRunMeasure m = ProcessMeasurer.Measure(zipname, "100", TimeSpan.FromMilliseconds(1000));
Assert.AreEqual(0, m.ExitCode, "Exit code");
Assert.AreEqual(Measure.LimitsStatus.WithinLimits, m.Limits);
Assert.IsTrue(m.PeakMemorySizeMB > 1, "Memory size seems too low");
var ptime = m.TotalProcessorTime.TotalMilliseconds;
var wctime = m.WallClockTime.TotalMilliseconds;
Assert.IsTrue(ptime <= 1000 && wctime >= ptime, "Total processor time must be very small because Delay.exe mostly sleeps but it is " + ptime);
Assert.IsTrue(wctime >= 100 && wctime < 1000, "Wall-clock time must be greater than given delay");
StreamReader reader = new StreamReader(m.StdOut);
string output = reader.ReadToEnd();
Assert.IsTrue(output.Contains("Done."), "Output must contain certain text.");
reader = new StreamReader(m.StdErr);
string error = reader.ReadToEnd();
Assert.IsTrue(error.Contains("Sample error text."), "Error output must contain certain text.");
}
finally
{
File.Delete(zipname);
}
}
public void MeasureProcessMemoryLimit()
{
Measurement.ProcessRunMeasure m = ProcessMeasurer.Measure("FailingTool.exe", "out-of-memory", TimeSpan.FromMinutes(10), memoryLimit: 1);
Assert.AreEqual(Measure.LimitsStatus.MemoryOut, m.Limits);
Assert.IsTrue(m.ExitCode == null, "Exit code");
Assert.IsTrue(m.PeakMemorySizeMB > 1, "Memory size seems too low");
var ptime = m.TotalProcessorTime.TotalMilliseconds;
var wctime = m.WallClockTime.TotalMilliseconds;
Assert.IsTrue(ptime <= 100000, "Total processor time");
Assert.IsTrue(wctime <= 100000, "Wall-clock time");
StreamReader reader = new StreamReader(m.StdOut);
string output = reader.ReadToEnd();
reader = new StreamReader(m.StdErr);
string error = reader.ReadToEnd();
}
public void MeasureProcessOutOfMemory()
{
Measurement.ProcessRunMeasure m = ProcessMeasurer.Measure("FailingTool.exe", "out-of-memory", TimeSpan.FromMinutes(10));
Assert.AreEqual(Measure.LimitsStatus.WithinLimits, m.Limits);
Assert.IsTrue(m.ExitCode < 0, "Exit code");
Assert.IsTrue(m.PeakMemorySizeMB > 1, "Memory size seems too low");
var ptime = m.TotalProcessorTime.TotalMilliseconds;
var wctime = m.WallClockTime.TotalMilliseconds;
Assert.IsTrue(ptime <= 100000, "Total processor time");
Assert.IsTrue(wctime <= 100000, "Wall-clock time");
StreamReader reader = new StreamReader(m.StdOut);
string output = reader.ReadToEnd();
reader = new StreamReader(m.StdErr);
string error = reader.ReadToEnd();
Assert.IsTrue(output.Contains("i = 0"), "Output");
Assert.IsTrue(error.Contains("OutOfMemoryException"), "Error must contain certain text.");
}
