Example #1
0
		static int Main ()
		{
			var depDir = Path.Combine ("..", "external", "benchmarker");
			var benchDir = Path.Combine (depDir, "benchmarks");
			var testDir = Path.Combine (depDir, "tests");

			var benchmarks = Directory.EnumerateFiles (benchDir, "*.benchmark")
			                 .Select (Benchmark.Load)
			                 .Where (b => !b.OnlyExplicit && b.ClientCommandLine == null)
			                 .OrderBy (b => b.Name)
			                 .ToArray ();

			var monoPath = Path.GetFullPath (Path.Combine ("..", "..", "runtime", "mono-wrapper"));
			var classDir = Path.GetFullPath (Path.Combine ("..", "..", "mcs", "class", "lib", "net_4_x"));

			var rand = new Random ();
			var cpus = Environment.ProcessorCount;

			var results = new List<TestResult> (benchmarks.Length);

			var sw = Stopwatch.StartNew ();

			for (var i = 0; i < benchmarks.Length; i++) {
				var bench = benchmarks [i];

				var sampleFreq = rand.Next (0, 1001);
				var sampleMode = rand.Next (0, 2) == 1 ? "real" : "process";
				var maxSamples = rand.Next (0, cpus * 2000 + 1);
				var heapShotFreq = rand.Next (0, 11);
				var maxFrames = rand.Next (0, 33);
				var allocMode = rand.Next (0, 2) == 1 ? "alloc" : "noalloc";

				var profOptions = $"sample=cycles/{sampleFreq},sampling-{sampleMode},maxsamples={maxSamples},heapshot={heapShotFreq}gc,maxframes={maxFrames},{allocMode},output=/dev/null";

				var info = new ProcessStartInfo {
					UseShellExecute = false,
					WorkingDirectory = Path.Combine (testDir, bench.TestDirectory),
					FileName = monoPath,
					Arguments = $"--debug --profile=log:{profOptions} " + string.Join (" ", bench.CommandLine),
					RedirectStandardOutput = true,
					RedirectStandardError = true,
				};

				info.EnvironmentVariables.Clear ();
				info.EnvironmentVariables.Add ("MONO_PATH", classDir);

				var progress = $"({i + 1}/{benchmarks.Length})";

				Console.ForegroundColor = ConsoleColor.Blue;
				Console.WriteLine ($"[{sw.Elapsed.ToString ("G")}] {progress} Running {bench.Name} with profiler options: {profOptions}");
				Console.ResetColor ();

				var result = new TestResult {
					Benchmark = bench,
					StartInfo = info,
				};

				using (var proc = new Process ()) {
					proc.StartInfo = info;

					proc.OutputDataReceived += (sender, args) => {
						if (args.Data != null)
							result.StandardOutput.AppendLine (args.Data);
					};

					proc.ErrorDataReceived += (sender, args) => {
						if (args.Data != null)
							result.StandardError.AppendLine (args.Data);
					};

					result.Stopwatch.Start ();

					proc.Start ();

					proc.BeginOutputReadLine ();
					proc.BeginErrorReadLine ();

					if (!proc.WaitForExit ((int) _timeout.TotalMilliseconds)) {
						// Force a thread dump.
						Syscall.kill (proc.Id, Signum.SIGQUIT);
						Thread.Sleep (1000);

						try {
							proc.Kill ();
						} catch (Exception) {
						}
					} else
						result.ExitCode = proc.ExitCode;

					result.Stopwatch.Stop ();
				}

				var resultStr = result.ExitCode == null ? "timed out" : $"exited with code: {result.ExitCode}";

				Console.ForegroundColor = result.ExitCode != 0 ? ConsoleColor.Red : ConsoleColor.Green;
				Console.WriteLine ($"[{sw.Elapsed.ToString ("G")}] {progress} {bench.Name} took {result.Stopwatch.Elapsed.ToString ("G")} and {resultStr}");
				Console.ResetColor ();

				if (result.ExitCode != 0) {
					Console.ForegroundColor = ConsoleColor.Red;
					Console.WriteLine ("===== stdout =====");
					Console.ResetColor ();

					Console.WriteLine (result.StandardOutput.ToString ());

					Console.ForegroundColor = ConsoleColor.Red;
					Console.WriteLine ("===== stderr =====");
					Console.ResetColor ();

					Console.WriteLine (result.StandardError.ToString ());
				}

				results.Add (result);
			}

			sw.Stop ();

			var successes = results.Count (r => r.ExitCode == 0);
			var failures = results.Count (r => r.ExitCode != null && r.ExitCode != 0);
			var timeouts = results.Count (r => r.ExitCode == null);

			var settings = new XmlWriterSettings {
				NewLineOnAttributes = true,
				Indent = true,
			};

			using (var writer = XmlWriter.Create ("TestResult-profiler-stress.xml", settings)) {
				writer.WriteStartDocument ();
				writer.WriteComment ("This file represents the results of running a test suite");

				writer.WriteStartElement ("test-results");
				writer.WriteAttributeString ("name", "profiler-stress-tests.dummy");
				writer.WriteAttributeString ("total", results.Count.ToString ());
				writer.WriteAttributeString ("failures", failures.ToString ());
				writer.WriteAttributeString ("not-run", "0");
				writer.WriteAttributeString ("date", DateTime.Now.ToString ("yyyy-MM-dd"));
				writer.WriteAttributeString ("time", DateTime.Now.ToString ("HH:mm:ss"));

				writer.WriteStartElement ("environment");
				writer.WriteAttributeString ("nunit-version", "2.4.8.0");
				writer.WriteAttributeString ("clr-version", Environment.Version.ToString ());
				writer.WriteAttributeString ("os-version", Environment.OSVersion.ToString ());
				writer.WriteAttributeString ("platform", Environment.OSVersion.Platform.ToString ());
				writer.WriteAttributeString ("cwd", Environment.CurrentDirectory);
				writer.WriteAttributeString ("machine-name", Environment.MachineName);
				writer.WriteAttributeString ("user", Environment.UserName);
				writer.WriteAttributeString ("user-domain", Environment.UserDomainName);
				writer.WriteEndElement ();

				writer.WriteStartElement ("culture-info");
				writer.WriteAttributeString ("current-culture", CultureInfo.CurrentCulture.Name);
				writer.WriteAttributeString ("current-uiculture", CultureInfo.CurrentUICulture.Name);
				writer.WriteEndElement ();

				writer.WriteStartElement ("test-suite");
				writer.WriteAttributeString ("name", "profiler-stress-tests.dummy");
				writer.WriteAttributeString ("success", (failures + timeouts == 0).ToString ());
				writer.WriteAttributeString ("time", ((int) sw.Elapsed.TotalSeconds).ToString ());
				writer.WriteAttributeString ("asserts", (failures + timeouts).ToString ());
				writer.WriteStartElement ("results");

				writer.WriteStartElement ("test-suite");
				writer.WriteAttributeString ("name", "MonoTests");
				writer.WriteAttributeString ("success", (failures + timeouts == 0).ToString ());
				writer.WriteAttributeString ("time", ((int) sw.Elapsed.TotalSeconds).ToString ());
				writer.WriteAttributeString ("asserts", (failures + timeouts).ToString ());
				writer.WriteStartElement ("results");

				writer.WriteStartElement ("test-suite");
				writer.WriteAttributeString ("name", "profiler-stress");
				writer.WriteAttributeString ("success", (failures + timeouts == 0).ToString ());
				writer.WriteAttributeString ("time", ((int) sw.Elapsed.TotalSeconds).ToString ());
				writer.WriteAttributeString ("asserts", (failures + timeouts).ToString ());
				writer.WriteStartElement ("results");

				foreach (var result in results) {
					var timeoutStr = result.ExitCode == null ? "_timeout" : string.Empty;

					writer.WriteStartElement ("test-case");
					writer.WriteAttributeString ("name", $"MonoTests.profiler-stress.{result.Benchmark.Name}{timeoutStr}");
					writer.WriteAttributeString ("executed", "True");
					writer.WriteAttributeString ("success", (result.ExitCode == 0).ToString ());
					writer.WriteAttributeString ("time", ((int) result.Stopwatch.Elapsed.TotalSeconds).ToString ());
					writer.WriteAttributeString ("asserts", result.ExitCode == 0 ? "0" : "1");

					if (result.ExitCode != 0) {
						writer.WriteStartElement ("failure");

						writer.WriteStartElement ("message");
						writer.WriteCData (FilterInvalidXmlChars (result.StandardOutput.ToString ()));
						writer.WriteEndElement ();

						writer.WriteStartElement ("stack-trace");
						writer.WriteCData (FilterInvalidXmlChars (result.StandardError.ToString ()));
						writer.WriteEndElement ();

						writer.WriteEndElement ();
					}

					writer.WriteEndElement ();
				}

				writer.WriteEndElement ();
				writer.WriteEndElement ();

				writer.WriteEndElement ();
				writer.WriteEndElement ();

				writer.WriteEndElement ();
				writer.WriteEndElement ();

				writer.WriteEndElement ();

				writer.WriteEndDocument ();
			}

			var failureStr = failures + timeouts != 0 ? $" ({failures} failures, {timeouts} timeouts)" : string.Empty;

			Console.ForegroundColor = failures + timeouts != 0 ? ConsoleColor.Red : ConsoleColor.Green;
			Console.WriteLine ($"[{sw.Elapsed.ToString ("G")}] Finished with {successes}/{results.Count} passing tests{failureStr}");
			Console.ResetColor ();

			return failures + timeouts;
		}
Example #2
0
        static int Main()
        {
            var depDir   = Path.Combine("..", "external", "benchmarker");
            var benchDir = Path.Combine(depDir, "benchmarks");
            var testDir  = Path.Combine(depDir, "tests");

            var benchmarks = Directory.EnumerateFiles(benchDir, "*.benchmark")
                             .Select(Benchmark.Load)
                             .Where(b => !b.OnlyExplicit && b.ClientCommandLine == null)
                             .OrderBy(b => b.Name)
                             .ToArray();

            var monoPath = Path.GetFullPath(Path.Combine("..", "..", "runtime", "mono-wrapper"));
            var classDir = Path.GetFullPath(Path.Combine("..", "..", "mcs", "class", "lib", "net_4_x"));

            var rand = new Random();
            var cpus = Environment.ProcessorCount;

            var results = new List <TestResult> (benchmarks.Length);

            var sw = Stopwatch.StartNew();

            for (var i = 0; i < benchmarks.Length; i++)
            {
                var bench = benchmarks [i];

                var sampleFreq   = rand.Next(-1000, 1001);
                var sampleMode   = rand.Next(0, 2) == 1 ? "-real" : string.Empty;
                var maxSamples   = rand.Next(0, cpus * 2000 + 1);
                var heapShotFreq = rand.Next(-10, 11);
                var maxFrames    = rand.Next(0, 33);
                var options      = _options.ToDictionary(x => x, _ => rand.Next(0, 2) == 1)
                                   .Select(x => (x.Value ? string.Empty : "no") + x.Key)
                                   .ToArray();

                var profOptions = $"maxframes={maxFrames},{string.Join (",", options)},output=/dev/null";

                if (sampleFreq > 0)
                {
                    profOptions += $",sample{sampleMode}={sampleFreq},maxsamples={maxSamples}";
                }

                if (heapShotFreq > 0)
                {
                    profOptions += $",heapshot={heapShotFreq}gc";
                }

                var info = new ProcessStartInfo {
                    UseShellExecute        = false,
                    WorkingDirectory       = Path.Combine(testDir, bench.TestDirectory),
                    FileName               = monoPath,
                    Arguments              = $"--debug --profile=log:{profOptions} " + string.Join(" ", bench.CommandLine),
                    RedirectStandardOutput = true,
                    RedirectStandardError  = true,
                };

                info.EnvironmentVariables.Clear();
                info.EnvironmentVariables.Add("MONO_PATH", classDir);

                var progress = $"({i + 1}/{benchmarks.Length})";

                Console.ForegroundColor = ConsoleColor.Blue;
                Console.WriteLine($"[{sw.Elapsed.ToString ("G")}] {progress} Running {bench.Name} with profiler options: {profOptions}");
                Console.ResetColor();

                var result = new TestResult {
                    Benchmark = bench,
                    StartInfo = info,
                };

                using (var proc = new Process()) {
                    proc.StartInfo = info;

                    proc.OutputDataReceived += (sender, args) => {
                        if (args.Data != null)
                        {
                            result.StandardOutput.AppendLine(args.Data);
                        }
                    };

                    proc.ErrorDataReceived += (sender, args) => {
                        if (args.Data != null)
                        {
                            result.StandardError.AppendLine(args.Data);
                        }
                    };

                    result.Stopwatch.Start();

                    proc.Start();

                    proc.BeginOutputReadLine();
                    proc.BeginErrorReadLine();

                    if (!proc.WaitForExit((int)_timeout.TotalMilliseconds))
                    {
                        // Force a thread dump.
                        Syscall.kill(proc.Id, Signum.SIGQUIT);
                        Thread.Sleep(1000);

                        try {
                            proc.Kill();
                        } catch (Exception) {
                        }
                    }
                    else
                    {
                        result.ExitCode = proc.ExitCode;
                    }

                    result.Stopwatch.Stop();
                }

                var resultStr = result.ExitCode == null ? "timed out" : $"exited with code: {result.ExitCode}";

                Console.ForegroundColor = result.ExitCode != 0 ? ConsoleColor.Red : ConsoleColor.Green;
                Console.WriteLine($"[{sw.Elapsed.ToString ("G")}] {progress} {bench.Name} took {result.Stopwatch.Elapsed.ToString ("G")} and {resultStr}");
                Console.ResetColor();

                if (result.ExitCode != 0)
                {
                    Console.ForegroundColor = ConsoleColor.Red;
                    Console.WriteLine("===== stdout =====");
                    Console.ResetColor();

                    Console.WriteLine(result.StandardOutput.ToString());

                    Console.ForegroundColor = ConsoleColor.Red;
                    Console.WriteLine("===== stderr =====");
                    Console.ResetColor();

                    Console.WriteLine(result.StandardError.ToString());
                }

                results.Add(result);
            }

            sw.Stop();

            var successes = results.Count(r => r.ExitCode == 0);
            var failures  = results.Count(r => r.ExitCode != null && r.ExitCode != 0);
            var timeouts  = results.Count(r => r.ExitCode == null);

            var settings = new XmlWriterSettings {
                NewLineOnAttributes = true,
                Indent = true,
            };

            using (var writer = XmlWriter.Create("TestResult-profiler-stress.xml", settings)) {
                writer.WriteStartDocument();
                writer.WriteComment("This file represents the results of running a test suite");

                writer.WriteStartElement("test-results");
                writer.WriteAttributeString("name", "profiler-stress-tests.dummy");
                writer.WriteAttributeString("total", results.Count.ToString());
                writer.WriteAttributeString("failures", failures.ToString());
                writer.WriteAttributeString("not-run", "0");
                writer.WriteAttributeString("date", DateTime.Now.ToString("yyyy-MM-dd"));
                writer.WriteAttributeString("time", DateTime.Now.ToString("HH:mm:ss"));

                writer.WriteStartElement("environment");
                writer.WriteAttributeString("nunit-version", "2.4.8.0");
                writer.WriteAttributeString("clr-version", Environment.Version.ToString());
                writer.WriteAttributeString("os-version", Environment.OSVersion.ToString());
                writer.WriteAttributeString("platform", Environment.OSVersion.Platform.ToString());
                writer.WriteAttributeString("cwd", Environment.CurrentDirectory);
                writer.WriteAttributeString("machine-name", Environment.MachineName);
                writer.WriteAttributeString("user", Environment.UserName);
                writer.WriteAttributeString("user-domain", Environment.UserDomainName);
                writer.WriteEndElement();

                writer.WriteStartElement("culture-info");
                writer.WriteAttributeString("current-culture", CultureInfo.CurrentCulture.Name);
                writer.WriteAttributeString("current-uiculture", CultureInfo.CurrentUICulture.Name);
                writer.WriteEndElement();

                writer.WriteStartElement("test-suite");
                writer.WriteAttributeString("name", "profiler-stress-tests.dummy");
                writer.WriteAttributeString("success", (failures + timeouts == 0).ToString());
                writer.WriteAttributeString("time", ((int)sw.Elapsed.TotalSeconds).ToString());
                writer.WriteAttributeString("asserts", (failures + timeouts).ToString());
                writer.WriteStartElement("results");

                writer.WriteStartElement("test-suite");
                writer.WriteAttributeString("name", "MonoTests");
                writer.WriteAttributeString("success", (failures + timeouts == 0).ToString());
                writer.WriteAttributeString("time", ((int)sw.Elapsed.TotalSeconds).ToString());
                writer.WriteAttributeString("asserts", (failures + timeouts).ToString());
                writer.WriteStartElement("results");

                writer.WriteStartElement("test-suite");
                writer.WriteAttributeString("name", "profiler-stress");
                writer.WriteAttributeString("success", (failures + timeouts == 0).ToString());
                writer.WriteAttributeString("time", ((int)sw.Elapsed.TotalSeconds).ToString());
                writer.WriteAttributeString("asserts", (failures + timeouts).ToString());
                writer.WriteStartElement("results");

                foreach (var result in results)
                {
                    var timeoutStr = result.ExitCode == null ? "_timeout" : string.Empty;

                    writer.WriteStartElement("test-case");
                    writer.WriteAttributeString("name", $"MonoTests.profiler-stress.{result.Benchmark.Name}{timeoutStr}");
                    writer.WriteAttributeString("executed", "True");
                    writer.WriteAttributeString("success", (result.ExitCode == 0).ToString());
                    writer.WriteAttributeString("time", ((int)result.Stopwatch.Elapsed.TotalSeconds).ToString());
                    writer.WriteAttributeString("asserts", result.ExitCode == 0 ? "0" : "1");

                    if (result.ExitCode != 0)
                    {
                        writer.WriteStartElement("failure");

                        writer.WriteStartElement("message");
                        writer.WriteCData(FilterInvalidXmlChars(result.StandardOutput.ToString()));
                        writer.WriteEndElement();

                        writer.WriteStartElement("stack-trace");
                        writer.WriteCData(FilterInvalidXmlChars(result.StandardError.ToString()));
                        writer.WriteEndElement();

                        writer.WriteEndElement();
                    }

                    writer.WriteEndElement();
                }

                writer.WriteEndElement();
                writer.WriteEndElement();

                writer.WriteEndElement();
                writer.WriteEndElement();

                writer.WriteEndElement();
                writer.WriteEndElement();

                writer.WriteEndElement();

                writer.WriteEndDocument();
            }

            var failureStr = failures + timeouts != 0 ? $" ({failures} failures, {timeouts} timeouts)" : string.Empty;

            Console.ForegroundColor = failures + timeouts != 0 ? ConsoleColor.Red : ConsoleColor.Green;
            Console.WriteLine($"[{sw.Elapsed.ToString ("G")}] Finished with {successes}/{results.Count} passing tests{failureStr}");
            Console.ResetColor();

            return(failures + timeouts);
        }