private TestLauncherResult RunWithInitializedRunner(ITestRunner runner, TestProject consolidatedTestProject, IReportManager reportManager) { bool wasCanceled = false; // Explore or Run tests. Report report = DoExploreOrRun(runner, consolidatedTestProject.TestPackage, ref wasCanceled); if (report == null) { report = new Report(); } var result = new TestLauncherResult(report); // Generate reports even if the test run is canceled, unless this step // also gets canceled. if (result.Report.TestPackageRun != null || DoNotRun) { GenerateReports(result, reportManager, consolidatedTestProject, ref wasCanceled); } // Done. if (ShowReports) { ShowReportDocuments(result); } // Produce the final result code. if (wasCanceled) { result.SetResultCode(ResultCode.Canceled); } else { if (!IgnoreAnnotations && result.Report.TestModel != null && result.Report.TestModel.GetErrorAnnotationCount() != 0) { result.SetResultCode(ResultCode.Failure); } else if (result.Report.TestPackageRun != null && result.Statistics.FailedCount > 0) { result.SetResultCode(ResultCode.Failure); } else if (result.Report.TestPackageRun != null && result.Statistics.TestCount == 0) { result.SetResultCode(ResultCode.NoTests); } } return(result); }
private TestLauncherResult CreateResult(int resultCode, TestProject consolidatedTestProject) { Report report = new Report(); report.TestPackage = new TestPackageData(consolidatedTestProject.TestPackage); var result = new TestLauncherResult(report); result.SetResultCode(resultCode); return(result); }
public void TaskPassesSpecifiedArgumentsToLauncher() { var task = new StubbedGallioTask(); task.DoNotRun = true; task.EchoResults = false; task.Filter = "Type: SimpleTest"; task.ReportDirectory = "dir"; task.ReportTypes = "XML;Html"; task.ReportNameFormat = "report"; task.ShowReports = true; task.ReportArchive = "zip"; task.RunTimeLimit = 7200; // seconds task.Verbosity = Verbosity.Debug; task.RunnerType = StandardTestRunnerFactoryNames.Local; task.RunnerExtensions.Add(new Argument("DebugExtension,Gallio")); task.PluginDirectories = new DirSet[] { CreateDirSet("plugin") }; task.Files = new FileSet[] { CreateFileSet("assembly1"), CreateFileSet("assembly2") }; task.HintDirectories = new DirSet[] { CreateDirSet("hint1"), CreateDirSet("hint2") }; task.ApplicationBaseDirectory = "baseDir"; task.ShadowCopy = true; task.Debug = true; task.WorkingDirectory = "workingDir"; task.RuntimeVersion = "v4.0.30319"; task.RunnerProperties.AddRange(new[] { new Argument("RunnerOption1=RunnerValue1"), new Argument(" RunnerOption2 "), new Argument("RunnerOption3 = 'RunnerValue3'"), new Argument("RunnerOption4=\"'RunnerValue4'\"") }); task.ReportFormatterProperties.AddRange(new[] { new Argument("FormatterOption1=FormatterValue1"), new Argument(" FormatterOption2 "), new Argument("FormatterOption3 = 'FormatterValue3'"), new Argument("FormatterOption4=\"'FormatterValue4'\"") }); task.SetRunLauncherAction(launcher => { Assert.IsTrue(launcher.DoNotRun); Assert.IsFalse(launcher.EchoResults); Assert.AreEqual("Type: SimpleTest", launcher.TestExecutionOptions.FilterSet.ToFilterSetExpr()); Assert.AreEqual(LogSeverity.Debug, ((FilteredLogger)launcher.Logger).MinSeverity); Assert.IsInstanceOfType(typeof(LogProgressMonitorProvider), launcher.ProgressMonitorProvider); Assert.AreEqual("dir", launcher.TestProject.ReportDirectory); Assert.IsTrue(launcher.TestProject.IsReportDirectorySpecified); Assert.AreElementsEqual(new string[] { "XML", "Html" }, launcher.ReportFormats); Assert.AreEqual("report", launcher.TestProject.ReportNameFormat); Assert.IsTrue(launcher.TestProject.IsReportNameFormatSpecified); Assert.IsTrue(launcher.ShowReports); Assert.AreEqual(TimeSpan.FromMinutes(120), launcher.RunTimeLimit); Assert.AreEqual(ReportArchive.Zip, launcher.TestProject.ReportArchive); Assert.AreEqual(StandardTestRunnerFactoryNames.Local, launcher.TestProject.TestRunnerFactoryName); Assert.IsTrue(launcher.TestProject.IsTestRunnerFactoryNameSpecified); Assert.AreEqual(0, launcher.TestProject.TestRunnerExtensions.Count); Assert.AreElementsEqual(new[] { "DebugExtension,Gallio" }, launcher.TestProject.TestRunnerExtensionSpecifications); Assert.IsNull(launcher.RuntimeSetup.ConfigurationFilePath); Assert.AreEqual(Path.GetDirectoryName(AssemblyUtils.GetAssemblyLocalPath(typeof(GallioTask).Assembly)), launcher.RuntimeSetup.RuntimePath); Assert.AreElementsEqual(new[] { "plugin" }, launcher.RuntimeSetup.PluginDirectories); Assert.AreElementsEqual(new[] { "assembly1", "assembly2" }, from x in launcher.FilePatterns select x.ToString()); Assert.AreElementsEqual(new[] { "hint1", "hint2" }, from x in launcher.TestProject.TestPackage.HintDirectories select x.ToString()); Assert.AreEqual("baseDir", launcher.TestProject.TestPackage.ApplicationBaseDirectory.ToString()); Assert.IsTrue(launcher.TestProject.TestPackage.IsApplicationBaseDirectorySpecified); Assert.AreEqual("workingDir", launcher.TestProject.TestPackage.WorkingDirectory.ToString()); Assert.IsTrue(launcher.TestProject.TestPackage.IsWorkingDirectorySpecified); Assert.IsTrue(launcher.TestProject.TestPackage.ShadowCopy); Assert.IsTrue(launcher.TestProject.TestPackage.IsShadowCopySpecified); Assert.IsNotNull(launcher.TestProject.TestPackage.DebuggerSetup); Assert.IsTrue(launcher.TestProject.TestPackage.IsDebuggerSetupSpecified); Assert.AreEqual("v4.0.30319", launcher.TestProject.TestPackage.RuntimeVersion); Assert.IsTrue(launcher.TestProject.TestPackage.IsRuntimeVersionSpecified); Assert.AreEqual(new PropertySet() { { "RunnerOption1", "RunnerValue1" }, { "RunnerOption2", "" }, { "RunnerOption3", "RunnerValue3" }, { "RunnerOption4", "'RunnerValue4'" } }, launcher.TestRunnerOptions.Properties); Assert.AreEqual(new PropertySet() { { "FormatterOption1", "FormatterValue1" }, { "FormatterOption2", "" }, { "FormatterOption3", "FormatterValue3" }, { "FormatterOption4", "'FormatterValue4'" } }, launcher.ReportFormatterOptions.Properties); var result = new TestLauncherResult(new Report()); result.SetResultCode(ResultCode.NoTests); return result; }); Assert.DoesNotThrow(task.InternalExecute); }
public void TaskPassesDefaultArgumentsToLauncher() { var task = new StubbedGallioTask(); task.SetRunLauncherAction(launcher => { Assert.IsFalse(launcher.DoNotRun); Assert.IsTrue(launcher.EchoResults); Assert.IsTrue(launcher.TestExecutionOptions.FilterSet.IsEmpty); Assert.AreEqual(LogSeverity.Important, ((FilteredLogger)launcher.Logger).MinSeverity); Assert.IsInstanceOfType(typeof(LogProgressMonitorProvider), launcher.ProgressMonitorProvider); Assert.AreEqual("Reports", launcher.TestProject.ReportDirectory); Assert.IsFalse(launcher.TestProject.IsReportDirectorySpecified); Assert.AreElementsEqual(new string[] { }, launcher.ReportFormats); Assert.AreEqual("test-report-{0}-{1}", launcher.TestProject.ReportNameFormat); Assert.IsFalse(launcher.TestProject.IsReportNameFormatSpecified); Assert.IsFalse(launcher.ShowReports); Assert.IsNull(launcher.RunTimeLimit); Assert.AreEqual(ReportArchive.Normal, launcher.TestProject.ReportArchive); Assert.AreEqual(StandardTestRunnerFactoryNames.IsolatedProcess, launcher.TestProject.TestRunnerFactoryName); Assert.IsFalse(launcher.TestProject.IsTestRunnerFactoryNameSpecified); Assert.Count(0, launcher.TestProject.TestRunnerExtensions); Assert.AreElementsEqual(new string[] { }, launcher.TestProject.TestRunnerExtensionSpecifications); Assert.IsNull(launcher.RuntimeSetup.ConfigurationFilePath); Assert.AreEqual(Path.GetDirectoryName(AssemblyUtils.GetAssemblyLocalPath(typeof(GallioTask).Assembly)), launcher.RuntimeSetup.RuntimePath); Assert.AreElementsEqual(new string[] { }, launcher.RuntimeSetup.PluginDirectories); Assert.AreElementsEqual(new string[] { }, from x in launcher.FilePatterns select x.ToString()); Assert.AreElementsEqual(new string[] { }, from x in launcher.TestProject.TestPackage.HintDirectories select x.ToString()); Assert.IsNull(launcher.TestProject.TestPackage.ApplicationBaseDirectory); Assert.IsFalse(launcher.TestProject.TestPackage.IsApplicationBaseDirectorySpecified); Assert.IsNull(launcher.TestProject.TestPackage.WorkingDirectory); Assert.IsFalse(launcher.TestProject.TestPackage.IsWorkingDirectorySpecified); Assert.IsFalse(launcher.TestProject.TestPackage.ShadowCopy); Assert.IsFalse(launcher.TestProject.TestPackage.IsShadowCopySpecified); Assert.IsNull(launcher.TestProject.TestPackage.DebuggerSetup); Assert.IsFalse(launcher.TestProject.TestPackage.IsDebuggerSetupSpecified); Assert.IsNull(launcher.TestProject.TestPackage.RuntimeVersion); Assert.IsFalse(launcher.TestProject.TestPackage.IsRuntimeVersionSpecified); Assert.AreEqual(new PropertySet(), launcher.TestRunnerOptions.Properties); Assert.AreEqual(new PropertySet(), launcher.ReportFormatterOptions.Properties); var result = new TestLauncherResult(new Report()); result.SetResultCode(ResultCode.Success); return result; }); Assert.DoesNotThrow(() => task.InternalExecute()); }
public void FailOnErrorWhenSetToFalseCausesABuildExceptionToNotBeThrownOnFailures() { var task = new StubbedGallioTask(); task.FailOnError = false; task.SetRunLauncherAction(delegate { var result = new TestLauncherResult(new Report()); result.SetResultCode(ResultCode.Failure); return result; }); Assert.DoesNotThrow(task.InternalExecute); }
public void TaskExposesResultsReturnedByLauncher() { var task = new StubbedGallioTask(); task.ResultProperty = "ExitCode"; task.StatisticsPropertiesPrefix = "Stats."; task.SetRunLauncherAction(launcher => { var report = new Report(); report.TestPackageRun = new TestPackageRun(); report.TestPackageRun.Statistics.AssertCount = 42; report.TestPackageRun.Statistics.Duration = 1.5; report.TestPackageRun.Statistics.FailedCount = 5; report.TestPackageRun.Statistics.InconclusiveCount = 11; report.TestPackageRun.Statistics.PassedCount = 21; report.TestPackageRun.Statistics.SkippedCount = 1; report.TestPackageRun.Statistics.StepCount = 30; report.TestPackageRun.Statistics.TestCount = 28; var result = new TestLauncherResult(report); result.SetResultCode(ResultCode.Failure); return result; }); Assert.Throws<BuildException>(task.InternalExecute); Assert.AreEqual(ResultCode.Failure.ToString(), task.Properties["ExitCode"]); Assert.AreEqual("42", task.Properties["Stats.AssertCount"]); Assert.AreEqual("1.5", task.Properties["Stats.Duration"]); Assert.AreEqual("5", task.Properties["Stats.FailedCount"]); Assert.AreEqual("11", task.Properties["Stats.InconclusiveCount"]); Assert.AreEqual("21", task.Properties["Stats.PassedCount"]); Assert.AreEqual("1", task.Properties["Stats.SkippedCount"]); Assert.AreEqual("30", task.Properties["Stats.StepCount"]); Assert.AreEqual("28", task.Properties["Stats.TestCount"]); }
public void RunReturnsCorrectResultCode(int resultCode, FacadeTestRunState expectedRunState) { StubbedLocalTestRunner tr = new StubbedLocalTestRunner(); Assembly assembly = typeof(TDNetAddInTests).Assembly; string assemblyPath = AssemblyUtils.GetAssemblyLocalPath(assembly); tr.SetRunLauncherAction(delegate { TestLauncherResult result = new TestLauncherResult(new Report()); result.SetResultCode(resultCode); return result; }); FacadeOptions facadeOptions = new FacadeOptions(); FacadeTestRunState runResult = tr.Run(MockRepository.GenerateStub<IFacadeTestListener>(), assemblyPath, null, facadeOptions); Assert.AreEqual(expectedRunState, runResult); }
private TestLauncherResult RunWithRuntime() { bool wasCanceled = false; ITestProjectManager testProjectManager = RuntimeAccessor.ServiceLocator.Resolve <ITestProjectManager>(); TestProject consolidatedTestProject = ConsolidateTestProject(testProjectManager, ref wasCanceled); if (wasCanceled) { return(CreateResult(ResultCode.Canceled, testProject)); } if (consolidatedTestProject == null) { return(CreateResult(ResultCode.InvalidArguments, testProject)); } if (consolidatedTestProject.TestPackage.Files.Count == 0) { logger.Log(LogSeverity.Warning, "No test files to execute!"); return(CreateResult(ResultCode.NoTests, consolidatedTestProject)); } IReportManager reportManager = RuntimeAccessor.ServiceLocator.Resolve <IReportManager>(); if (!ValidateReportFormats(reportManager, consolidatedTestProject)) { return(CreateResult(ResultCode.InvalidArguments, consolidatedTestProject)); } ITestRunnerManager testRunnerManager = RuntimeAccessor.ServiceLocator.Resolve <ITestRunnerManager>(); ITestRunnerFactory testRunnerFactory = testRunnerManager.GetFactory(consolidatedTestProject.TestRunnerFactoryName); if (testRunnerFactory == null) { logger.Log(LogSeverity.Error, String.Format("Unrecognized test runner factory name: '{0}'.", consolidatedTestProject.TestRunnerFactoryName)); return(CreateResult(ResultCode.InvalidArguments, consolidatedTestProject)); } ITestRunner runner = testRunnerFactory.CreateTestRunner(); var result = new TestLauncherResult(new Report { TestPackage = new TestPackageData(TestProject.TestPackage) }); try { DoRegisterExtensions(runner, consolidatedTestProject); DoInitialize(runner, ref wasCanceled); if (!wasCanceled) { result = RunWithInitializedRunner(runner, consolidatedTestProject, reportManager); } } finally { DoDispose(runner, ref wasCanceled); } if (wasCanceled) { result.SetResultCode(ResultCode.Canceled); } return(result); }
public void IgnoreFailuresCausesTrueToBeReturnedEvenWhenFailuresOccur() { var task = new StubbedGallioTask(); task.IgnoreFailures = true; task.SetRunLauncherAction(delegate { var result = new TestLauncherResult(new Report()); result.SetResultCode(ResultCode.Failure); return result; }); Assert.IsTrue(task.Execute()); }
public void TaskExposesResultsReturnedByLauncher() { var task = new StubbedGallioTask(); task.SetRunLauncherAction(delegate { var report = new Report(); report.TestPackageRun = new TestPackageRun(); report.TestPackageRun.Statistics.AssertCount = 42; report.TestPackageRun.Statistics.Duration = 1.5; report.TestPackageRun.Statistics.FailedCount = 5; report.TestPackageRun.Statistics.InconclusiveCount = 11; report.TestPackageRun.Statistics.PassedCount = 21; report.TestPackageRun.Statistics.SkippedCount = 1; report.TestPackageRun.Statistics.StepCount = 30; report.TestPackageRun.Statistics.TestCount = 28; var result = new TestLauncherResult(report); result.SetResultCode(ResultCode.Failure); return result; }); Assert.IsFalse(task.InternalExecute()); Assert.AreEqual(ResultCode.Failure, task.ExitCode); Assert.AreEqual(42, task.AssertCount); Assert.AreEqual(1.5, task.Duration); Assert.AreEqual(5, task.FailedCount); Assert.AreEqual(11, task.InconclusiveCount); Assert.AreEqual(21, task.PassedCount); Assert.AreEqual(1, task.SkippedCount); Assert.AreEqual(30, task.StepCount); Assert.AreEqual(28, task.TestCount); }