private TestExecutionResults ExecuteTests(string testModule, IEnumerable <IMemberDefinition> testsToRun) { if (!args.RunInIsolation && args.InstrumentationStrategy == InstrumentationStrategy.StaticConstructor) { throw new Exception("When StaticConstructor instrumentation strategy is used tests should be run in Isolation."); } ITestExecutor executor = null; if (args.RunInIsolation) { executor = new IsolationTestExecutor(args.TestingFramework.GetExecutor()); } else { executor = args.TestingFramework.GetExecutor(); } TestExecutionResults results = executor.Execute(testModule, testsToRun, args.TestingFrameworkArguments); int numOfExecuted = args.DependencyCollectionGranularity == DependencyCollectionGranularity.Class ? results.ExecutedTestClassesCount : results.ExecutedTestMethodsCount; if (testsToRun.Count() != numOfExecuted) { logger.Warn("Number of tests scheduled for run does not match number of actually run tests"); } return(results); }
private TestExecutionResults ProcessExecutionResults(XmlNode xml) { TestExecutionResults result = new TestExecutionResults(); var assemblyNodes = xml.GetChildren("assembly"); if (assemblyNodes.Count != 1) { logger.Error("There should be results from exactly one assembly since we are running a single assembly."); return(null); } XmlNode assemblyNode = assemblyNodes[0]; result.PassedTestMethodsCount = assemblyNode.ParseAttribute <int>(PassedTestCaseCountAttribute); result.FailedTestMethodsCount = assemblyNode.ParseAttribute <int>(FailedTestCaseCountAttribute); result.SkippedTestMethodsCount = assemblyNode.ParseAttribute <int>(SkippedTestCaseCountAttribute); result.ExecutionTime = assemblyNode.ParseAttribute <decimal>(TimeAttribute); var tests = assemblyNode.GetChildren("test"); foreach (var test in tests) { string testClassName = test.ParseAttribute <string>("type"); string testMethodName = test.ParseAttribute <string>("name"); if (testMethodName.Contains('(')) { // convert trait name into test method name testMethodName = testMethodName.CutTillFirstOccurence('('); } result.ExecutedTestClasses.Add(testClassName); result.ExecutedTestMethods.Add(testMethodName); } return(result); }
// TODO: Check if we can avoid using operator public static TestExecutionResults operator +(TestExecutionResults one, TestExecutionResults two) { TestExecutionResults cumalative = new TestExecutionResults(); cumalative.PassedTestMethodsCount = one.PassedTestMethodsCount + two.PassedTestMethodsCount; cumalative.FailedTestMethodsCount = one.FailedTestMethodsCount + two.FailedTestMethodsCount; cumalative.SkippedTestMethodsCount = one.SkippedTestMethodsCount + two.SkippedTestMethodsCount; cumalative.ExecutionTime = one.ExecutionTime + two.ExecutionTime; cumalative.TestClasses.UnionWith(one.TestClasses); cumalative.TestClasses.UnionWith(two.TestClasses); cumalative.ExecutedTestClasses.UnionWith(one.ExecutedTestClasses); cumalative.ExecutedTestClasses.UnionWith(two.ExecutedTestClasses); cumalative.NotExecutedTestClasses.UnionWith(one.NotExecutedTestClasses); cumalative.NotExecutedTestClasses.UnionWith(two.NotExecutedTestClasses); cumalative.TestMethods.UnionWith(one.TestMethods); cumalative.TestMethods.UnionWith(two.TestMethods); cumalative.ExecutedTestMethods.UnionWith(one.ExecutedTestMethods); cumalative.ExecutedTestMethods.UnionWith(two.ExecutedTestMethods); cumalative.NotExecutedTestMethods.UnionWith(one.NotExecutedTestMethods); cumalative.NotExecutedTestMethods.UnionWith(two.NotExecutedTestMethods); cumalative.FailedTestMethods.UnionWith(one.FailedTestMethods); cumalative.FailedTestMethods.UnionWith(two.FailedTestMethods); return(cumalative); }
protected override TestExecutionResults ExecuteImpl(string testModule, IEnumerable <IMemberDefinition> testsToRun, string[] args) { if (!testsToRun.Any()) { // discovery cannot be done // When running with empty list of tests XUnit hangs return(new TestExecutionResults()); } executedTests.Clear(); this.testsToRun = testsToRun; executionCanceled = false; result = new TestExecutionResults(); runner = AssemblyRunner.WithAppDomain(testModule); logger.Debug($"Running XUnit executor in App domain: {AppDomain.CurrentDomain.FriendlyName}"); runner.TestCaseFilter = TestCaseFilterHandler; runner.OnExecutionComplete = ExecutionCompleteHandler; runner.OnTestFailed = TestFailedHandler; runner.OnTestFinished += TestFinishedHandler; runner.OnDiscoveryComplete += DiscoveryCompleteHandler; runner.OnErrorMessage += ErrorMessageHandler; logger.Debug("Starting tests"); runner.Start(maxParallelThreads: 1); finished.WaitOne(); finished.Dispose(); while (runner.Status != AssemblyRunnerStatus.Idle && !executionCanceled) { // spin until runner finishes } if (executionCanceled) { try { runner.Dispose(); } catch (InvalidOperationException e) { // for now ignore this exception // due to XUnit bug of not resetting Status to Idle when cancelling run } } else { runner.Dispose(); } var temp = result; // reset result variable result = null; return(temp); }
protected override TestExecutionResults ExecuteImpl(string testModule, IEnumerable <IMemberDefinition> testsToRun, string[] arguments) { logger.Info(string.Format("Running NUnit2 executor in App domain: {0}", AppDomain.CurrentDomain.FriendlyName)); result = new TestExecutionResults(); var appDomain = AppDomain.CreateDomain("nunit2-test", null, testModule, null, false); NUnit2FrameworkDriver driver = new NUnit2FrameworkDriver(appDomain) { ID = "1" }; // Not sure why "1" but necessary var settings = new Dictionary <string, object>(); string x = driver.Load(testModule, settings); TestFilterBuilder filterBuilder = new TestFilterBuilder(); foreach (var t in testsToRun) { filterBuilder.AddTest(t.GetFullName()); } string filter = filterBuilder.GetFilter().Text; string discoveryXmlString = driver.Explore(filter); // TODO: Check if discovery information can be otained from // execution results. Not sure if execution results contain all // test methods and classes or only executed ones ParseDiscoveryResults(discoveryXmlString.ToXmlDocument()); if (testsToRun.Count() > 0) { string xmlString = driver.Run(this, filter); ParseExecutionResults(xmlString.ToXmlDocument()); } else { logger.Info("No tests selected for run"); } AppDomain.Unload(appDomain); var temp = result; // clean the local field for the next time result = null; return(temp); }
public List <TestExecutionResults> Run() { CleanFiles(); InitializeProject(); List <TestExecutionResults> results = new List <TestExecutionResults>(); do { try { logger.Info("Building project"); if (!BuildProject()) { throw new Exception("Build failed"); } BackupModules(); Dictionary <string, ICollection <IMemberDefinition> > moduleToAffectedTests = InstrumentAndSelect(ProgramModules, TestModules); logger.Info("Executing tests"); TestExecutionResults result = new TestExecutionResults(); foreach (KeyValuePair <string, ICollection <IMemberDefinition> > moduleToTests in moduleToAffectedTests) { result += ExecuteTests(moduleToTests.Key, moduleToTests.Value); } logger.Info("Test execution summary: " + result.ToString()); if (args.Debug) { LogResults(result); } results.Add(result); RestoreModules(); } catch (Exception e) { logger.Error("Exception thrown", e); } }while (MoveToNextRevision()); return(results); }
protected override TestExecutionResults ExecuteImpl(string testModule, IEnumerable <IMemberDefinition> testsToRun, string[] arguments) { // TODO: do discovery of all tests inside the module if (!testsToRun.Any()) { return(new TestExecutionResults()); } EnsureResultsDirExists(); // IEnumerable<string> toRun = FilterRunListBasedOnUserArguments(testsToRun, arguments); List <string> xunitArgs = ConstructXUnitArguments(testsToRun, arguments); TestExecutionResults results = new TestExecutionResults(); for (int i = 0; i < xunitArgs.Count; i++) { string commandLineArgs = xunitArgs[i]; ProcessUtil.ExecuteProcessNoPoppingConsole(CommonPaths.XUnitConsoleExePath, $"{testModule} {commandLineArgs}"); string xmlResults = File.ReadAllText(GetResultsFilePath(i)); results = results + ProcessExecutionResults(xmlResults.ToXmlDocument()); } return(results); }
private void LogResults(TestExecutionResults result) { IOUtil.WriteAllText(CommonPaths.TestResultsFilePath, JsonConvert.SerializeObject(result, Formatting.Indented)); IOUtil.WriteAllText(CommonPaths.TotalNumberOfTestClassesFilePath, result.TotalNumberOfTestClasses.ToString()); IOUtil.WriteAllText(CommonPaths.NumberOfExecutedTestClassesFilePath, result.ExecutedTestClassesCount.ToString()); IOUtil.WriteAllText(CommonPaths.TotalNumberOfTestMethodsFilePath, result.TotalNumberOfTestMethods.ToString()); IOUtil.WriteAllText(CommonPaths.NumberOfExecutedTestMethodsFilePath, result.ExecutedTestMethodsCount.ToString()); IOUtil.WriteAllText(CommonPaths.NumberOfPassedTestMethodsFilePath, result.PassedTestMethodsCount.ToString()); IOUtil.WriteAllText(CommonPaths.NumberOfFailedTestMethodsFilePath, result.FailedTestMethodsCount.ToString()); IOUtil.WriteAllText(CommonPaths.NumberOfSkippedTestMethodsFilePath, result.SkippedTestMethodsCount.ToString()); decimal executionTime = result.ExecutionTime; IOUtil.WriteAllText(CommonPaths.ExecutionTimeFilePath, executionTime.FormattedNumber()); double analysisTime = analysisStopwatch.Elapsed.TotalSeconds; IOUtil.WriteAllText(CommonPaths.AnalysisTimeFilePath, analysisTime.FormattedNumber()); double instrumentationTime = instrumentationStopwatch.Elapsed.TotalSeconds; IOUtil.WriteAllText(CommonPaths.InstrumentationTimeFilePath, instrumentationTime.FormattedNumber()); double totalTime = (double)executionTime + analysisTime + instrumentationTime; IOUtil.WriteAllText(CommonPaths.TotalTimeFilePath, totalTime.FormattedNumber()); }
protected override TestExecutionResults ExecuteImpl(string testModule, IEnumerable <IMemberDefinition> testsToRun, string[] args) { logger.Info(string.Format("Running NUnit3 executor in App domain: {0}", AppDomain.CurrentDomain.FriendlyName)); result = new TestExecutionResults(); TestFilterBuilder filterBuilder = new TestFilterBuilder(); foreach (var t in testsToRun) { filterBuilder.AddTest(t.GetFullName()); } using (var engine = TestEngineActivator.CreateInstance()) { TestPackage package = new TestPackage(testModule); using (var runner = engine.GetRunner(package)) { // explore all tests (default filter used) XmlNode explorationResults = runner.Explore(new TestFilterBuilder().GetFilter()); ParseDiscoveryResults(explorationResults); //foreach (var test in result.TestClasses) //{ // if (!testsToRun.Contains(test)) // { // result.NotExecutedTestClasses.Add(test); // } //} //foreach (var test in testsToRun) //{ // if (!result.TestClasses.Contains(test)) // { // logger.Debug($"Test: {test} found during analyzes phase but not discovered using Runner"); // } //} if (testsToRun.Count() > 0) { XmlNode runResults = runner.Run(this, filterBuilder.GetFilter()); ParseExecutionResults(runResults); try { runner.Unload(); } catch (Exception e) { logger.Warn("Error while unloading runner", e); } } else { logger.Info("No tests selected for run"); } } } var temp = result; // clean the local field for the next time result = null; return(temp); }