public void TaskPassesDefaultArgumentsToLauncher() { var task = new StubbedGallioTask(); task.SetRunLauncherAction(launcher => { Assert.IsFalse(launcher.DoNotRun); Assert.IsTrue(launcher.EchoResults); Assert.IsTrue(launcher.TestExecutionOptions.FilterSet.IsEmpty); Assert.AreEqual(LogSeverity.Important, ((FilteredLogger)launcher.Logger).MinSeverity); Assert.IsInstanceOfType(typeof(LogProgressMonitorProvider), launcher.ProgressMonitorProvider); Assert.AreEqual("Reports", launcher.TestProject.ReportDirectory); Assert.IsFalse(launcher.TestProject.IsReportDirectorySpecified); Assert.AreElementsEqual(new string[] { }, launcher.ReportFormats); Assert.AreEqual("test-report-{0}-{1}", launcher.TestProject.ReportNameFormat); Assert.IsFalse(launcher.TestProject.IsReportNameFormatSpecified); Assert.IsFalse(launcher.ShowReports); Assert.IsNull(launcher.RunTimeLimit); Assert.AreEqual(ReportArchive.Normal, launcher.TestProject.ReportArchive); Assert.AreEqual(StandardTestRunnerFactoryNames.IsolatedProcess, launcher.TestProject.TestRunnerFactoryName); Assert.IsFalse(launcher.TestProject.IsTestRunnerFactoryNameSpecified); Assert.Count(0, launcher.TestProject.TestRunnerExtensions); Assert.AreElementsEqual(new string[] { }, launcher.TestProject.TestRunnerExtensionSpecifications); Assert.IsNull(launcher.RuntimeSetup.ConfigurationFilePath); Assert.AreEqual(Path.GetDirectoryName(AssemblyUtils.GetAssemblyLocalPath(typeof(GallioTask).Assembly)), launcher.RuntimeSetup.RuntimePath); Assert.AreElementsEqual(new string[] { }, launcher.RuntimeSetup.PluginDirectories); Assert.AreElementsEqual(new string[] { }, from x in launcher.FilePatterns select x.ToString()); Assert.AreElementsEqual(new string[] { }, from x in launcher.TestProject.TestPackage.HintDirectories select x.ToString()); Assert.IsNull(launcher.TestProject.TestPackage.ApplicationBaseDirectory); Assert.IsFalse(launcher.TestProject.TestPackage.IsApplicationBaseDirectorySpecified); Assert.IsNull(launcher.TestProject.TestPackage.WorkingDirectory); Assert.IsFalse(launcher.TestProject.TestPackage.IsWorkingDirectorySpecified); Assert.IsFalse(launcher.TestProject.TestPackage.ShadowCopy); Assert.IsFalse(launcher.TestProject.TestPackage.IsShadowCopySpecified); Assert.IsNull(launcher.TestProject.TestPackage.DebuggerSetup); Assert.IsFalse(launcher.TestProject.TestPackage.IsDebuggerSetupSpecified); Assert.IsNull(launcher.TestProject.TestPackage.RuntimeVersion); Assert.IsFalse(launcher.TestProject.TestPackage.IsRuntimeVersionSpecified); Assert.AreEqual(new PropertySet(), launcher.TestRunnerOptions.Properties); Assert.AreEqual(new PropertySet(), launcher.ReportFormatterOptions.Properties); var result = new TestLauncherResult(new Report()); result.SetResultCode(ResultCode.Success); return result; }); Assert.DoesNotThrow(() => task.InternalExecute()); }
public void TaskPassesSpecifiedArgumentsToLauncher() { var task = new StubbedGallioTask(); task.DoNotRun = true; task.EchoResults = false; task.Filter = "Type: SimpleTest"; task.ReportDirectory = "dir"; task.ReportTypes = "XML;Html"; task.ReportNameFormat = "report"; task.ShowReports = true; task.ReportArchive = "zip"; task.RunTimeLimit = 7200; // seconds task.Verbosity = Verbosity.Debug; task.RunnerType = StandardTestRunnerFactoryNames.Local; task.RunnerExtensions.Add(new Argument("DebugExtension,Gallio")); task.PluginDirectories = new DirSet[] { CreateDirSet("plugin") }; task.Files = new FileSet[] { CreateFileSet("assembly1"), CreateFileSet("assembly2") }; task.HintDirectories = new DirSet[] { CreateDirSet("hint1"), CreateDirSet("hint2") }; task.ApplicationBaseDirectory = "baseDir"; task.ShadowCopy = true; task.Debug = true; task.WorkingDirectory = "workingDir"; task.RuntimeVersion = "v4.0.30319"; task.RunnerProperties.AddRange(new[] { new Argument("RunnerOption1=RunnerValue1"), new Argument(" RunnerOption2 "), new Argument("RunnerOption3 = 'RunnerValue3'"), new Argument("RunnerOption4=\"'RunnerValue4'\"") }); task.ReportFormatterProperties.AddRange(new[] { new Argument("FormatterOption1=FormatterValue1"), new Argument(" FormatterOption2 "), new Argument("FormatterOption3 = 'FormatterValue3'"), new Argument("FormatterOption4=\"'FormatterValue4'\"") }); task.SetRunLauncherAction(launcher => { Assert.IsTrue(launcher.DoNotRun); Assert.IsFalse(launcher.EchoResults); Assert.AreEqual("Type: SimpleTest", launcher.TestExecutionOptions.FilterSet.ToFilterSetExpr()); Assert.AreEqual(LogSeverity.Debug, ((FilteredLogger)launcher.Logger).MinSeverity); Assert.IsInstanceOfType(typeof(LogProgressMonitorProvider), launcher.ProgressMonitorProvider); Assert.AreEqual("dir", launcher.TestProject.ReportDirectory); Assert.IsTrue(launcher.TestProject.IsReportDirectorySpecified); Assert.AreElementsEqual(new string[] { "XML", "Html" }, launcher.ReportFormats); Assert.AreEqual("report", launcher.TestProject.ReportNameFormat); Assert.IsTrue(launcher.TestProject.IsReportNameFormatSpecified); Assert.IsTrue(launcher.ShowReports); Assert.AreEqual(TimeSpan.FromMinutes(120), launcher.RunTimeLimit); Assert.AreEqual(ReportArchive.Zip, launcher.TestProject.ReportArchive); Assert.AreEqual(StandardTestRunnerFactoryNames.Local, launcher.TestProject.TestRunnerFactoryName); Assert.IsTrue(launcher.TestProject.IsTestRunnerFactoryNameSpecified); Assert.AreEqual(0, launcher.TestProject.TestRunnerExtensions.Count); Assert.AreElementsEqual(new[] { "DebugExtension,Gallio" }, launcher.TestProject.TestRunnerExtensionSpecifications); Assert.IsNull(launcher.RuntimeSetup.ConfigurationFilePath); Assert.AreEqual(Path.GetDirectoryName(AssemblyUtils.GetAssemblyLocalPath(typeof(GallioTask).Assembly)), launcher.RuntimeSetup.RuntimePath); Assert.AreElementsEqual(new[] { "plugin" }, launcher.RuntimeSetup.PluginDirectories); Assert.AreElementsEqual(new[] { "assembly1", "assembly2" }, from x in launcher.FilePatterns select x.ToString()); Assert.AreElementsEqual(new[] { "hint1", "hint2" }, from x in launcher.TestProject.TestPackage.HintDirectories select x.ToString()); Assert.AreEqual("baseDir", launcher.TestProject.TestPackage.ApplicationBaseDirectory.ToString()); Assert.IsTrue(launcher.TestProject.TestPackage.IsApplicationBaseDirectorySpecified); Assert.AreEqual("workingDir", launcher.TestProject.TestPackage.WorkingDirectory.ToString()); Assert.IsTrue(launcher.TestProject.TestPackage.IsWorkingDirectorySpecified); Assert.IsTrue(launcher.TestProject.TestPackage.ShadowCopy); Assert.IsTrue(launcher.TestProject.TestPackage.IsShadowCopySpecified); Assert.IsNotNull(launcher.TestProject.TestPackage.DebuggerSetup); Assert.IsTrue(launcher.TestProject.TestPackage.IsDebuggerSetupSpecified); Assert.AreEqual("v4.0.30319", launcher.TestProject.TestPackage.RuntimeVersion); Assert.IsTrue(launcher.TestProject.TestPackage.IsRuntimeVersionSpecified); Assert.AreEqual(new PropertySet() { { "RunnerOption1", "RunnerValue1" }, { "RunnerOption2", "" }, { "RunnerOption3", "RunnerValue3" }, { "RunnerOption4", "'RunnerValue4'" } }, launcher.TestRunnerOptions.Properties); Assert.AreEqual(new PropertySet() { { "FormatterOption1", "FormatterValue1" }, { "FormatterOption2", "" }, { "FormatterOption3", "FormatterValue3" }, { "FormatterOption4", "'FormatterValue4'" } }, launcher.ReportFormatterOptions.Properties); var result = new TestLauncherResult(new Report()); result.SetResultCode(ResultCode.NoTests); return result; }); Assert.DoesNotThrow(task.InternalExecute); }
public void TaskExposesResultsReturnedByLauncher() { var task = new StubbedGallioTask(); task.ResultProperty = "ExitCode"; task.StatisticsPropertiesPrefix = "Stats."; task.SetRunLauncherAction(launcher => { var report = new Report(); report.TestPackageRun = new TestPackageRun(); report.TestPackageRun.Statistics.AssertCount = 42; report.TestPackageRun.Statistics.Duration = 1.5; report.TestPackageRun.Statistics.FailedCount = 5; report.TestPackageRun.Statistics.InconclusiveCount = 11; report.TestPackageRun.Statistics.PassedCount = 21; report.TestPackageRun.Statistics.SkippedCount = 1; report.TestPackageRun.Statistics.StepCount = 30; report.TestPackageRun.Statistics.TestCount = 28; var result = new TestLauncherResult(report); result.SetResultCode(ResultCode.Failure); return result; }); Assert.Throws<BuildException>(task.InternalExecute); Assert.AreEqual(ResultCode.Failure.ToString(), task.Properties["ExitCode"]); Assert.AreEqual("42", task.Properties["Stats.AssertCount"]); Assert.AreEqual("1.5", task.Properties["Stats.Duration"]); Assert.AreEqual("5", task.Properties["Stats.FailedCount"]); Assert.AreEqual("11", task.Properties["Stats.InconclusiveCount"]); Assert.AreEqual("21", task.Properties["Stats.PassedCount"]); Assert.AreEqual("1", task.Properties["Stats.SkippedCount"]); Assert.AreEqual("30", task.Properties["Stats.StepCount"]); Assert.AreEqual("28", task.Properties["Stats.TestCount"]); }
public void FailOnErrorWhenSetToFalseCausesABuildExceptionToNotBeThrownOnFailures() { var task = new StubbedGallioTask(); task.FailOnError = false; task.SetRunLauncherAction(delegate { var result = new TestLauncherResult(new Report()); result.SetResultCode(ResultCode.Failure); return result; }); Assert.DoesNotThrow(task.InternalExecute); }
private void PopulateStatistics(TestLauncherResult result) { Statistics stats = result.Statistics; AssertCount = stats.AssertCount; Duration = stats.Duration; FailedCount = stats.FailedCount; InconclusiveCount = stats.InconclusiveCount; PassedCount = stats.PassedCount; SkippedCount = stats.SkippedCount; RunCount = stats.RunCount; StepCount = stats.StepCount; TestCount = stats.TestCount; }
private static void LogResultSummary(ILogger logger, TestLauncherResult result) { switch (result.ResultCode) { case ResultCode.Success: logger.Log(LogSeverity.Info, result.ResultSummary); break; case ResultCode.Failure: logger.Log(LogSeverity.Error, result.ResultSummary); break; } }
public void TaskExposesResultsReturnedByLauncher() { var task = new StubbedRunGallioCommand(); var expectedResult = new TestLauncherResult(new Report()); task.SetRunLauncherAction(o => expectedResult); Assert.AreSame(expectedResult, task.ExecuteWithMessagePump()); }
public void RunReturnsCorrectResultCode(int resultCode, FacadeTestRunState expectedRunState) { StubbedLocalTestRunner tr = new StubbedLocalTestRunner(); Assembly assembly = typeof(TDNetAddInTests).Assembly; string assemblyPath = AssemblyUtils.GetAssemblyLocalPath(assembly); tr.SetRunLauncherAction(delegate { TestLauncherResult result = new TestLauncherResult(new Report()); result.SetResultCode(resultCode); return result; }); FacadeOptions facadeOptions = new FacadeOptions(); FacadeTestRunState runResult = tr.Run(MockRepository.GenerateStub<IFacadeTestListener>(), assemblyPath, null, facadeOptions); Assert.AreEqual(expectedRunState, runResult); }
public void IgnoreFailuresCausesTrueToBeReturnedEvenWhenFailuresOccur() { var task = new StubbedGallioTask(); task.IgnoreFailures = true; task.SetRunLauncherAction(delegate { var result = new TestLauncherResult(new Report()); result.SetResultCode(ResultCode.Failure); return result; }); Assert.IsTrue(task.Execute()); }
public void TaskExposesResultsReturnedByLauncher() { var task = new StubbedGallioTask(); task.SetRunLauncherAction(delegate { var report = new Report(); report.TestPackageRun = new TestPackageRun(); report.TestPackageRun.Statistics.AssertCount = 42; report.TestPackageRun.Statistics.Duration = 1.5; report.TestPackageRun.Statistics.FailedCount = 5; report.TestPackageRun.Statistics.InconclusiveCount = 11; report.TestPackageRun.Statistics.PassedCount = 21; report.TestPackageRun.Statistics.SkippedCount = 1; report.TestPackageRun.Statistics.StepCount = 30; report.TestPackageRun.Statistics.TestCount = 28; var result = new TestLauncherResult(report); result.SetResultCode(ResultCode.Failure); return result; }); Assert.IsFalse(task.InternalExecute()); Assert.AreEqual(ResultCode.Failure, task.ExitCode); Assert.AreEqual(42, task.AssertCount); Assert.AreEqual(1.5, task.Duration); Assert.AreEqual(5, task.FailedCount); Assert.AreEqual(11, task.InconclusiveCount); Assert.AreEqual(21, task.PassedCount); Assert.AreEqual(1, task.SkippedCount); Assert.AreEqual(30, task.StepCount); Assert.AreEqual(28, task.TestCount); }
private void Launch(bool doNoRun) { MarkupStreamWriter logStreamWriter = TestLog.Default; var launcher = new TestLauncher(); launcher.TestProject.TestPackage = testPackage; launcher.Logger = new MarkupStreamLogger(logStreamWriter); launcher.TestExecutionOptions.FilterSet = new FilterSet<ITestDescriptor>(new OrFilter<ITestDescriptor>(filters)); launcher.TestProject.TestRunnerFactoryName = TestRunnerFactoryName; string reportDirectory = SpecialPathPolicy.For<SampleRunner>().GetTempDirectory().FullName; launcher.TestProject.ReportDirectory = reportDirectory; launcher.TestProject.ReportNameFormat = "SampleRunnerReport"; launcher.ReportFormatterOptions.AddProperty(@"SaveAttachmentContents", @"false"); launcher.AddReportFormat(@"Text"); // Disabled because the Xml can get really big and causes problems if the sample runner is used frequently. //launcher.AddReportFormat("Xml"); launcher.DoNotRun = doNoRun; GenericCollectionUtils.ForEach(testRunnerOptions.Properties, x => launcher.TestRunnerOptions.AddProperty(x.Key, x.Value)); using (logStreamWriter.BeginSection("Log Output")) result = launcher.Run(); using (logStreamWriter.BeginSection("Text Report")) { foreach (string reportPath in result.ReportDocumentPaths) { string extension = Path.GetExtension(reportPath); if (extension == ".txt") { logStreamWriter.WriteLine(File.ReadAllText(reportPath)); } else if (extension == ".xml") { logStreamWriter.Container.AttachXml(null, File.ReadAllText(reportPath)); } File.Delete(reportPath); } } }
private static FacadeTestRunState GetTestRunState(TestLauncherResult result) { switch (result.ResultCode) { default: return FacadeTestRunState.Error; case ResultCode.Failure: case ResultCode.Canceled: return FacadeTestRunState.Failure; case ResultCode.NoTests: case ResultCode.Success: return FacadeTestRunState.Success; } }
private void PopulateStatistics(TestLauncherResult result) { Statistics stats = result.Statistics; Properties[StatisticsPropertiesPrefix + @"TestCount"] = stats.TestCount.ToString(); Properties[StatisticsPropertiesPrefix + @"StepCount"] = stats.StepCount.ToString(); Properties[StatisticsPropertiesPrefix + @"PassedCount"] = stats.PassedCount.ToString(); Properties[StatisticsPropertiesPrefix + @"FailedCount"] = stats.FailedCount.ToString(); Properties[StatisticsPropertiesPrefix + @"InconclusiveCount"] = stats.InconclusiveCount.ToString(); Properties[StatisticsPropertiesPrefix + @"RunCount"] = stats.RunCount.ToString(); Properties[StatisticsPropertiesPrefix + @"SkippedCount"] = stats.SkippedCount.ToString(); Properties[StatisticsPropertiesPrefix + @"Duration"] = stats.Duration.ToString(); Properties[StatisticsPropertiesPrefix + @"AssertCount"] = stats.AssertCount.ToString(); }