public void RunTests(IEnumerable <string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { this._log = frameworkHandle; this.Log(TestMessageLevel.Informational, "running tests (src) ..."); var sourcesList = sources as IList <string> ?? sources.ToList(); if (!this.SetupEnvironment(sourcesList.First())) { return; } this.Log(TestMessageLevel.Informational, "using configuration: " + this._config.ActiveConfiguration); this._runningFromSources = true; var logFileDir = this._config.CacheDir + "\\Testing\\Temporary"; this.Log(TestMessageLevel.Informational, "logs are written to (" + TestContainerHelper.ToLinkPath(logFileDir) + ")"); foreach (var source in sourcesList) { var cases = TestContainerHelper.ParseTestContainerFile( source, frameworkHandle, null, this._config.ActiveConfiguration); this.RunTests(cases.Values, runContext, frameworkHandle); } this._runningFromSources = false; this.Log(TestMessageLevel.Informational, "running tests (src) done"); }
public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { this._log = frameworkHandle; this.Log(TestMessageLevel.Informational, "running tests ..."); var testCases = tests as IList <TestCase> ?? tests.ToList(); if (!testCases.Any()) { return; } if (!this._runningFromSources) { if (!this.SetupEnvironment(testCases.First().Source)) { return; } } // make sure a configuration is set if (!this._config.ActiveConfiguration.Any()) { if (this._config.TrySetActiveConfigFromConfigTypes()) { this.Log(TestMessageLevel.Warning, "Configuration fallback to: " + this._config.ActiveConfiguration); } else { this.Log(TestMessageLevel.Error, "could not set Configuration"); return; } } // make sure we have a ctest executable if (!File.Exists(this._config.CTestExecutable)) { this._config.CTestExecutable = TestContainerHelper.FindCTestExe(this._config.CacheDir); } if (!File.Exists(this._config.CTestExecutable)) { this.Log(TestMessageLevel.Error, "ctest not found, tried: \"" + this._config.CTestExecutable + "\""); return; } if (!Directory.Exists(this._config.CacheDir)) { this.Log(TestMessageLevel.Error, "working directory not found: " + TestContainerHelper.ToLinkPath(this._config.CacheDir)); return; } this.Log(TestMessageLevel.Informational, "working directory is " + TestContainerHelper.ToLinkPath(this._config.CacheDir)); var logFileDir = this._config.CacheDir + "\\Testing\\Temporary"; if (!this._runningFromSources) { this.Log(TestMessageLevel.Informational, "ctest (" + this._config.CTestExecutable + ")"); this.Log(TestMessageLevel.Informational, "logs are written to (" + TestContainerHelper.ToLinkPath(logFileDir) + ")"); } this._parentProcess = Process.GetCurrentProcess(); this._ctestProcess = new Process(); if (this._procParam == null) { this._procParam = new ProcessStartInfo { CreateNoWindow = true, RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, WindowStyle = ProcessWindowStyle.Hidden }; } // run test cases foreach (var test in testCases) { var testResult = new TestResult(test) { ComputerName = Environment.MachineName, Outcome = TestOutcome.Skipped }; // verify we have a run directory and a ctest executable var args = "-R \"^" + test.FullyQualifiedName + "$\""; if (this._config.ActiveConfiguration.Any()) { args += " -C \"" + this._config.ActiveConfiguration + "\""; } this._procParam.Arguments = args; this._procParam.FileName = this._config.CTestExecutable; this._procParam.WorkingDirectory = this._config.CacheDir; this._ctestProcess.StartInfo = this._procParam; var logFileName = logFileDir + "\\LastTest.log"; if (File.Exists(logFileName)) { File.Delete(logFileName); } var logMsg = "ctest " + test.FullyQualifiedName; if (this._config.ActiveConfiguration.Any()) { logMsg += " -C " + this._config.ActiveConfiguration; } this.Log(TestMessageLevel.Informational, logMsg); if (this._cancelled) { break; } if (runContext.IsBeingDebugged) { /// @todo check if child process debugging is available?!? this._ctestProcess.Start(); } else { this._ctestProcess.Start(); } this._ctestProcess.WaitForExit(); if (this._cancelled) { break; } var output = this._ctestProcess.StandardOutput.ReadToEnd(); if (!File.Exists(logFileName)) { this.Log(TestMessageLevel.Warning, "logfile not found: " + TestContainerHelper.ToLinkPath(logFileName)); } var content = File.ReadAllText(logFileName); var logFileBackup = test.FullyQualifiedName + ".log"; var invalidChars = new string(Path.GetInvalidFileNameChars()) + new string(Path.GetInvalidPathChars()); foreach (char c in invalidChars) { logFileBackup = logFileBackup.Replace(c.ToString(), "_"); } logFileBackup = logFileDir + "\\" + logFileBackup; File.Copy(logFileName, logFileBackup, true); var matchesDuration = CTestExecutor.RegexDuration.Match(content); var timeSpan = new TimeSpan(); if (matchesDuration.Success) { timeSpan = TimeSpan.FromSeconds( double.Parse(matchesDuration.Groups[CTestExecutor.RegexFieldDuration].Value, System.Globalization.CultureInfo.InvariantCulture.NumberFormat)); } else { this.Log(TestMessageLevel.Warning, "could not get runtime of test " + test.FullyQualifiedName); } testResult.Duration = timeSpan; testResult.Outcome = this._ctestProcess.ExitCode == 0 ? TestOutcome.Passed : TestOutcome.Failed; if (this._ctestProcess.ExitCode != 0) { var matchesOutput = CTestExecutor.RegexOutput.Match(content); testResult.ErrorMessage = matchesOutput.Groups[CTestExecutor.RegexFieldOutput].Value; this.Log(TestMessageLevel.Error, "ERROR IN TEST " + test.FullyQualifiedName + ":"); this.Log(TestMessageLevel.Error, output); this.Log(TestMessageLevel.Error, "END OF TEST OUTPUT FROM " + test.FullyQualifiedName); } this.Log(TestMessageLevel.Informational, "Log saved to " + TestContainerHelper.ToLinkPath(logFileBackup)); frameworkHandle.RecordResult(testResult); } this._ctestProcess.Dispose(); this._ctestProcess = null; this._parentProcess = null; this.Log(TestMessageLevel.Informational, "running tests done"); }
public static Dictionary <string, TestCase> ParseTestContainerFile(string source, IMessageLogger log, CTestTestCollection collection, string activeConfiguration) { log.SendMessage(TestMessageLevel.Informational, "Parsing CTest file: " + TestContainerHelper.ToLinkPath(source)); var cases = new Dictionary <string, TestCase>(); var content = File.ReadLines(source); var skipFoundTests = false; var lineNumber = 0; foreach (var line in content) { lineNumber++; var ifcheck = TestContainerHelper.IfLevelRegex.Match(line); if (ifcheck.Success) { var cfgRegexString = ifcheck.Groups[FieldNameCfgRegex].Value; var cfgRegex = new Regex(cfgRegexString); skipFoundTests = !cfgRegex.IsMatch(activeConfiguration); continue; } else if (line == "else()") { skipFoundTests = true; continue; } else if (line == "endif()") { skipFoundTests = false; continue; } var test = TestContainerHelper.AddTestRegex.Match(line); if (test.Success) { var testname = test.Groups[FieldNameTestname].Value; if (skipFoundTests) { //log.SendMessage(TestMessageLevel.Warning, //"skipping test because of Configuration mismatch: line " + lineNumber + ", test " + testname); continue; } if (null != collection) { if (!collection.TestExists(testname)) { log.SendMessage(TestMessageLevel.Warning, "test not listed by ctest -N : " + testname); } } if (cases.ContainsKey(testname)) { continue; } var testcase = new TestCase(testname, CTestExecutor.ExecutorUri, source) { CodeFilePath = source, DisplayName = testname, LineNumber = lineNumber, }; if (null != collection) { if (collection.TestExists(testname)) { testcase.DisplayName = collection[testname].Number.ToString().PadLeft(3, '0') + ": " + testname; } } cases.Add(testname, testcase); } } return(cases); }