public void MeasureProcessExitCode() { Measurement.ProcessRunMeasure m = ProcessMeasurer.Measure("Delay.exe", "10000 extraArgument", TimeSpan.FromMilliseconds(10000)); Assert.AreEqual(Measure.LimitsStatus.WithinLimits, m.Limits); Assert.AreEqual(42, m.ExitCode, "Exit code"); Assert.IsTrue(m.PeakMemorySizeMB > 1, "Memory size seems too low"); var ptime = m.TotalProcessorTime.TotalMilliseconds; var wctime = m.WallClockTime.TotalMilliseconds; Assert.IsTrue(ptime > 0 && ptime <= 100, "Total processor time"); Assert.IsTrue(wctime > 0 && wctime <= 10000, "Wall-clock time must be greater than given timeout"); StreamReader reader = new StreamReader(m.StdOut); string output = reader.ReadToEnd(); Assert.IsTrue(output.Contains("Use: Delay.exe"), "Output must contain certain text."); reader = new StreamReader(m.StdErr); string error = reader.ReadToEnd(); Assert.IsTrue(String.IsNullOrEmpty(error), "Error output"); }
public void MeasureProcessRun() { Measurement.ProcessRunMeasure m = ProcessMeasurer.Measure("Delay.exe", "100", TimeSpan.FromMilliseconds(1000)); Assert.AreEqual(0, m.ExitCode, "Exit code"); Assert.AreEqual(Measure.LimitsStatus.WithinLimits, m.Limits); Assert.IsTrue(m.PeakMemorySizeMB > 1, "Memory size seems too low"); var ptime = m.TotalProcessorTime.TotalMilliseconds; var wctime = m.WallClockTime.TotalMilliseconds; Assert.IsTrue(ptime <= 1000 && wctime >= ptime, "Total processor time must be very small because Delay.exe mostly sleeps but it is " + ptime); Assert.IsTrue(wctime >= 100, "Wall-clock time must be greater than given delay"); Assert.IsTrue(wctime < 1000, "Wall-clock time must be less"); StreamReader reader = new StreamReader(m.StdOut); string output = reader.ReadToEnd(); Assert.IsTrue(output.Contains("Done."), "Output must contain certain text."); reader = new StreamReader(m.StdErr); string error = reader.ReadToEnd(); Assert.IsTrue(error.Contains("Sample error text."), "Error output must contain certain text."); }
public void ZipWithMultipleExecutablesWithoutRelationshipFail() { var zipname = Path.GetTempFileName(); using (var zip = new ZipFile()) { zip.AddFiles(new string[] { "Delay.exe", "FailingTool.exe" }); zip.Save(zipname); } bool fail = false; try { Measurement.ProcessRunMeasure m = ProcessMeasurer.Measure(zipname, "100", TimeSpan.FromMilliseconds(1000)); } catch { fail = true; } finally { File.Delete(zipname); } Assert.IsTrue(fail, "Zip with multiple executables and no relationships should be considered incorrect."); }
public static BenchmarkResult RunBenchmark(int experimentId, string executable, string args, string inputDisplayName, string inputFullPath, int repetitions, TimeSpan timeOut, double memLimitMB, long?ouputLimit, long?errorLimit, Domain domain, double normal, int maxRepetitions = 10, double maxTimeInSeconds = 10) { if (domain == null) { throw new ArgumentNullException("domain"); } if (args != null) { args = args.Replace("{0}", inputFullPath); } else { args = ""; } DateTime acq = DateTime.Now; int maxCount = repetitions == 0 ? maxRepetitions : repetitions; TimeSpan maxTime = TimeSpan.FromSeconds(maxTimeInSeconds); int count = 0; List <ProcessRunMeasure> measures = new List <ProcessRunMeasure>(); TimeSpan total = TimeSpan.FromSeconds(0); ProcessRunAnalysis analysis = null; ProcessRunMeasure m; do { m = ProcessMeasurer.Measure(executable, args, timeOut, memLimitMB, ouputLimit, errorLimit, domain); measures.Add(m); count++; total += m.WallClockTime; if (analysis == null) // analyzed only once, repetitions are for more confident run time { analysis = domain.Analyze(inputFullPath, m); } } while ((repetitions != 0 || total < maxTime) && count < maxCount && m.Limits == Measure.LimitsStatus.WithinLimits); ProcessRunMeasure finalMeasure = Utils.AggregateMeasures(measures.ToArray()); Trace.WriteLine(String.Format("Done in {0} (aggregated by {1} runs)", finalMeasure.WallClockTime, count)); var performanceIndex = normal * finalMeasure.TotalProcessorTime.TotalSeconds; var result = new BenchmarkResult( experimentId, inputDisplayName, acq, performanceIndex, finalMeasure.TotalProcessorTime, finalMeasure.WallClockTime, finalMeasure.PeakMemorySizeMB, analysis.Status, finalMeasure.ExitCode, finalMeasure.StdOut, finalMeasure.StdErr, analysis.OutputProperties); return(result); }
public void MeasureProcessRunFromPackage() { var zipname = Path.GetTempFileName(); using (var zip = new ZipFile()) { zip.AddFiles(new string[] { "Delay.exe", "FailingTool.exe" }); string mainexe = "Delay.exe"; string content_types = "<?xml version=\"1.0\" encoding=\"utf-8\"?><Types xmlns=\"http://schemas.openxmlformats.org/package/2006/content-types\"><Default Extension=\"exe\" ContentType=\"application/octet-stream\" /><Default Extension=\"dll\" ContentType=\"application/octet-stream\" /><Default Extension=\"rels\" ContentType=\"application/vnd.openxmlformats-package.relationships+xml\" /></Types>"; string rels = "<?xml version=\"1.0\" encoding=\"utf-8\"?><Relationships xmlns=\"http://schemas.openxmlformats.org/package/2006/relationships\"><Relationship Type=\"http://schemas.openxmlformats.org/package/2006/relationships/meta data/thumbnail\" Target=\"/" + mainexe + "\" Id=\"R17bb7f6124fd45fe\" /></Relationships>"; zip.AddEntry("[Content_Types].xml", content_types); zip.AddEntry("_rels\\.rels", rels); zip.Save(zipname); } try { Measurement.ProcessRunMeasure m = ProcessMeasurer.Measure(zipname, "100", TimeSpan.FromMilliseconds(1000)); Assert.AreEqual(0, m.ExitCode, "Exit code"); Assert.AreEqual(Measure.LimitsStatus.WithinLimits, m.Limits); Assert.IsTrue(m.PeakMemorySizeMB > 1, "Memory size seems too low"); var ptime = m.TotalProcessorTime.TotalMilliseconds; var wctime = m.WallClockTime.TotalMilliseconds; Assert.IsTrue(ptime <= 1000 && wctime >= ptime, "Total processor time must be very small because Delay.exe mostly sleeps but it is " + ptime); Assert.IsTrue(wctime >= 100 && wctime < 1000, "Wall-clock time must be greater than given delay"); StreamReader reader = new StreamReader(m.StdOut); string output = reader.ReadToEnd(); Assert.IsTrue(output.Contains("Done."), "Output must contain certain text."); reader = new StreamReader(m.StdErr); string error = reader.ReadToEnd(); Assert.IsTrue(error.Contains("Sample error text."), "Error output must contain certain text."); } finally { File.Delete(zipname); } }
public void MeasureProcessMemoryLimit() { Measurement.ProcessRunMeasure m = ProcessMeasurer.Measure("FailingTool.exe", "out-of-memory", TimeSpan.FromMinutes(10), memoryLimit: 1); Assert.AreEqual(Measure.LimitsStatus.MemoryOut, m.Limits); Assert.IsTrue(m.ExitCode == null, "Exit code"); Assert.IsTrue(m.PeakMemorySizeMB > 1, "Memory size seems too low"); var ptime = m.TotalProcessorTime.TotalMilliseconds; var wctime = m.WallClockTime.TotalMilliseconds; Assert.IsTrue(ptime <= 100000, "Total processor time"); Assert.IsTrue(wctime <= 100000, "Wall-clock time"); StreamReader reader = new StreamReader(m.StdOut); string output = reader.ReadToEnd(); reader = new StreamReader(m.StdErr); string error = reader.ReadToEnd(); }
public void MeasureProcessOutOfMemory() { Measurement.ProcessRunMeasure m = ProcessMeasurer.Measure("FailingTool.exe", "out-of-memory", TimeSpan.FromMinutes(10)); Assert.AreEqual(Measure.LimitsStatus.WithinLimits, m.Limits); Assert.IsTrue(m.ExitCode < 0, "Exit code"); Assert.IsTrue(m.PeakMemorySizeMB > 1, "Memory size seems too low"); var ptime = m.TotalProcessorTime.TotalMilliseconds; var wctime = m.WallClockTime.TotalMilliseconds; Assert.IsTrue(ptime <= 100000, "Total processor time"); Assert.IsTrue(wctime <= 100000, "Wall-clock time"); StreamReader reader = new StreamReader(m.StdOut); string output = reader.ReadToEnd(); reader = new StreamReader(m.StdErr); string error = reader.ReadToEnd(); Assert.IsTrue(output.Contains("i = 0"), "Output"); Assert.IsTrue(error.Contains("OutOfMemoryException"), "Error must contain certain text."); }