private TestLauncherResult RunWithInitializedRunner(ITestRunner runner, TestProject consolidatedTestProject, IReportManager reportManager)
{
bool wasCanceled = false;
// Explore or Run tests.
Report report = DoExploreOrRun(runner, consolidatedTestProject.TestPackage, ref wasCanceled);
if (report == null)
{
report = new Report();
}
var result = new TestLauncherResult(report);
// Generate reports even if the test run is canceled, unless this step
// also gets canceled.
if (result.Report.TestPackageRun != null || DoNotRun)
{
GenerateReports(result, reportManager, consolidatedTestProject, ref wasCanceled);
}
// Done.
if (ShowReports)
{
ShowReportDocuments(result);
}
// Produce the final result code.
if (wasCanceled)
{
result.SetResultCode(ResultCode.Canceled);
}
else
{
if (!IgnoreAnnotations &&
result.Report.TestModel != null &&
result.Report.TestModel.GetErrorAnnotationCount() != 0)
{
result.SetResultCode(ResultCode.Failure);
}
else if (result.Report.TestPackageRun != null &&
result.Statistics.FailedCount > 0)
{
result.SetResultCode(ResultCode.Failure);
}
else if (result.Report.TestPackageRun != null &&
result.Statistics.TestCount == 0)
{
result.SetResultCode(ResultCode.NoTests);
}
}
return(result);
}
private TestLauncherResult CreateResult(int resultCode, TestProject consolidatedTestProject)
{
Report report = new Report();
report.TestPackage = new TestPackageData(consolidatedTestProject.TestPackage);
var result = new TestLauncherResult(report);
result.SetResultCode(resultCode);
return(result);
}
public void TaskPassesSpecifiedArgumentsToLauncher()
{
var task = new StubbedGallioTask();
task.DoNotRun = true;
task.EchoResults = false;
task.Filter = "Type: SimpleTest";
task.ReportDirectory = "dir";
task.ReportTypes = "XML;Html";
task.ReportNameFormat = "report";
task.ShowReports = true;
task.ReportArchive = "zip";
task.RunTimeLimit = 7200; // seconds
task.Verbosity = Verbosity.Debug;
task.RunnerType = StandardTestRunnerFactoryNames.Local;
task.RunnerExtensions.Add(new Argument("DebugExtension,Gallio"));
task.PluginDirectories = new DirSet[] { CreateDirSet("plugin") };
task.Files = new FileSet[] { CreateFileSet("assembly1"), CreateFileSet("assembly2") };
task.HintDirectories = new DirSet[] { CreateDirSet("hint1"), CreateDirSet("hint2") };
task.ApplicationBaseDirectory = "baseDir";
task.ShadowCopy = true;
task.Debug = true;
task.WorkingDirectory = "workingDir";
task.RuntimeVersion = "v4.0.30319";
task.RunnerProperties.AddRange(new[] { new Argument("RunnerOption1=RunnerValue1"), new Argument(" RunnerOption2 "), new Argument("RunnerOption3 = 'RunnerValue3'"), new Argument("RunnerOption4=\"'RunnerValue4'\"") });
task.ReportFormatterProperties.AddRange(new[] { new Argument("FormatterOption1=FormatterValue1"), new Argument(" FormatterOption2 "), new Argument("FormatterOption3 = 'FormatterValue3'"), new Argument("FormatterOption4=\"'FormatterValue4'\"") });
task.SetRunLauncherAction(launcher =>
{
Assert.IsTrue(launcher.DoNotRun);
Assert.IsFalse(launcher.EchoResults);
Assert.AreEqual("Type: SimpleTest", launcher.TestExecutionOptions.FilterSet.ToFilterSetExpr());
Assert.AreEqual(LogSeverity.Debug, ((FilteredLogger)launcher.Logger).MinSeverity);
Assert.IsInstanceOfType(typeof(LogProgressMonitorProvider), launcher.ProgressMonitorProvider);
Assert.AreEqual("dir", launcher.TestProject.ReportDirectory);
Assert.IsTrue(launcher.TestProject.IsReportDirectorySpecified);
Assert.AreElementsEqual(new string[] { "XML", "Html" }, launcher.ReportFormats);
Assert.AreEqual("report", launcher.TestProject.ReportNameFormat);
Assert.IsTrue(launcher.TestProject.IsReportNameFormatSpecified);
Assert.IsTrue(launcher.ShowReports);
Assert.AreEqual(TimeSpan.FromMinutes(120), launcher.RunTimeLimit);
Assert.AreEqual(ReportArchive.Zip, launcher.TestProject.ReportArchive);
Assert.AreEqual(StandardTestRunnerFactoryNames.Local, launcher.TestProject.TestRunnerFactoryName);
Assert.IsTrue(launcher.TestProject.IsTestRunnerFactoryNameSpecified);
Assert.AreEqual(0, launcher.TestProject.TestRunnerExtensions.Count);
Assert.AreElementsEqual(new[] { "DebugExtension,Gallio" }, launcher.TestProject.TestRunnerExtensionSpecifications);
Assert.IsNull(launcher.RuntimeSetup.ConfigurationFilePath);
Assert.AreEqual(Path.GetDirectoryName(AssemblyUtils.GetAssemblyLocalPath(typeof(GallioTask).Assembly)), launcher.RuntimeSetup.RuntimePath);
Assert.AreElementsEqual(new[] { "plugin" }, launcher.RuntimeSetup.PluginDirectories);
Assert.AreElementsEqual(new[] { "assembly1", "assembly2" }, from x in launcher.FilePatterns select x.ToString());
Assert.AreElementsEqual(new[] { "hint1", "hint2" }, from x in launcher.TestProject.TestPackage.HintDirectories select x.ToString());
Assert.AreEqual("baseDir", launcher.TestProject.TestPackage.ApplicationBaseDirectory.ToString());
Assert.IsTrue(launcher.TestProject.TestPackage.IsApplicationBaseDirectorySpecified);
Assert.AreEqual("workingDir", launcher.TestProject.TestPackage.WorkingDirectory.ToString());
Assert.IsTrue(launcher.TestProject.TestPackage.IsWorkingDirectorySpecified);
Assert.IsTrue(launcher.TestProject.TestPackage.ShadowCopy);
Assert.IsTrue(launcher.TestProject.TestPackage.IsShadowCopySpecified);
Assert.IsNotNull(launcher.TestProject.TestPackage.DebuggerSetup);
Assert.IsTrue(launcher.TestProject.TestPackage.IsDebuggerSetupSpecified);
Assert.AreEqual("v4.0.30319", launcher.TestProject.TestPackage.RuntimeVersion);
Assert.IsTrue(launcher.TestProject.TestPackage.IsRuntimeVersionSpecified);
Assert.AreEqual(new PropertySet()
{
{ "RunnerOption1", "RunnerValue1" },
{ "RunnerOption2", "" },
{ "RunnerOption3", "RunnerValue3" },
{ "RunnerOption4", "'RunnerValue4'" }
}, launcher.TestRunnerOptions.Properties);
Assert.AreEqual(new PropertySet()
{
{ "FormatterOption1", "FormatterValue1" },
{ "FormatterOption2", "" },
{ "FormatterOption3", "FormatterValue3" },
{ "FormatterOption4", "'FormatterValue4'" }
}, launcher.ReportFormatterOptions.Properties);
var result = new TestLauncherResult(new Report());
result.SetResultCode(ResultCode.NoTests);
return result;
});
Assert.DoesNotThrow(task.InternalExecute);
}
public void TaskPassesDefaultArgumentsToLauncher()
{
var task = new StubbedGallioTask();
task.SetRunLauncherAction(launcher =>
{
Assert.IsFalse(launcher.DoNotRun);
Assert.IsTrue(launcher.EchoResults);
Assert.IsTrue(launcher.TestExecutionOptions.FilterSet.IsEmpty);
Assert.AreEqual(LogSeverity.Important, ((FilteredLogger)launcher.Logger).MinSeverity);
Assert.IsInstanceOfType(typeof(LogProgressMonitorProvider), launcher.ProgressMonitorProvider);
Assert.AreEqual("Reports", launcher.TestProject.ReportDirectory);
Assert.IsFalse(launcher.TestProject.IsReportDirectorySpecified);
Assert.AreElementsEqual(new string[] { }, launcher.ReportFormats);
Assert.AreEqual("test-report-{0}-{1}", launcher.TestProject.ReportNameFormat);
Assert.IsFalse(launcher.TestProject.IsReportNameFormatSpecified);
Assert.IsFalse(launcher.ShowReports);
Assert.IsNull(launcher.RunTimeLimit);
Assert.AreEqual(ReportArchive.Normal, launcher.TestProject.ReportArchive);
Assert.AreEqual(StandardTestRunnerFactoryNames.IsolatedProcess, launcher.TestProject.TestRunnerFactoryName);
Assert.IsFalse(launcher.TestProject.IsTestRunnerFactoryNameSpecified);
Assert.Count(0, launcher.TestProject.TestRunnerExtensions);
Assert.AreElementsEqual(new string[] { }, launcher.TestProject.TestRunnerExtensionSpecifications);
Assert.IsNull(launcher.RuntimeSetup.ConfigurationFilePath);
Assert.AreEqual(Path.GetDirectoryName(AssemblyUtils.GetAssemblyLocalPath(typeof(GallioTask).Assembly)), launcher.RuntimeSetup.RuntimePath);
Assert.AreElementsEqual(new string[] { }, launcher.RuntimeSetup.PluginDirectories);
Assert.AreElementsEqual(new string[] { }, from x in launcher.FilePatterns select x.ToString());
Assert.AreElementsEqual(new string[] { }, from x in launcher.TestProject.TestPackage.HintDirectories select x.ToString());
Assert.IsNull(launcher.TestProject.TestPackage.ApplicationBaseDirectory);
Assert.IsFalse(launcher.TestProject.TestPackage.IsApplicationBaseDirectorySpecified);
Assert.IsNull(launcher.TestProject.TestPackage.WorkingDirectory);
Assert.IsFalse(launcher.TestProject.TestPackage.IsWorkingDirectorySpecified);
Assert.IsFalse(launcher.TestProject.TestPackage.ShadowCopy);
Assert.IsFalse(launcher.TestProject.TestPackage.IsShadowCopySpecified);
Assert.IsNull(launcher.TestProject.TestPackage.DebuggerSetup);
Assert.IsFalse(launcher.TestProject.TestPackage.IsDebuggerSetupSpecified);
Assert.IsNull(launcher.TestProject.TestPackage.RuntimeVersion);
Assert.IsFalse(launcher.TestProject.TestPackage.IsRuntimeVersionSpecified);
Assert.AreEqual(new PropertySet(), launcher.TestRunnerOptions.Properties);
Assert.AreEqual(new PropertySet(), launcher.ReportFormatterOptions.Properties);
var result = new TestLauncherResult(new Report());
result.SetResultCode(ResultCode.Success);
return result;
});
Assert.DoesNotThrow(() => task.InternalExecute());
}
public void FailOnErrorWhenSetToFalseCausesABuildExceptionToNotBeThrownOnFailures()
{
var task = new StubbedGallioTask();
task.FailOnError = false;
task.SetRunLauncherAction(delegate
{
var result = new TestLauncherResult(new Report());
result.SetResultCode(ResultCode.Failure);
return result;
});
Assert.DoesNotThrow(task.InternalExecute);
}
public void TaskExposesResultsReturnedByLauncher()
{
var task = new StubbedGallioTask();
task.ResultProperty = "ExitCode";
task.StatisticsPropertiesPrefix = "Stats.";
task.SetRunLauncherAction(launcher =>
{
var report = new Report();
report.TestPackageRun = new TestPackageRun();
report.TestPackageRun.Statistics.AssertCount = 42;
report.TestPackageRun.Statistics.Duration = 1.5;
report.TestPackageRun.Statistics.FailedCount = 5;
report.TestPackageRun.Statistics.InconclusiveCount = 11;
report.TestPackageRun.Statistics.PassedCount = 21;
report.TestPackageRun.Statistics.SkippedCount = 1;
report.TestPackageRun.Statistics.StepCount = 30;
report.TestPackageRun.Statistics.TestCount = 28;
var result = new TestLauncherResult(report);
result.SetResultCode(ResultCode.Failure);
return result;
});
Assert.Throws<BuildException>(task.InternalExecute);
Assert.AreEqual(ResultCode.Failure.ToString(), task.Properties["ExitCode"]);
Assert.AreEqual("42", task.Properties["Stats.AssertCount"]);
Assert.AreEqual("1.5", task.Properties["Stats.Duration"]);
Assert.AreEqual("5", task.Properties["Stats.FailedCount"]);
Assert.AreEqual("11", task.Properties["Stats.InconclusiveCount"]);
Assert.AreEqual("21", task.Properties["Stats.PassedCount"]);
Assert.AreEqual("1", task.Properties["Stats.SkippedCount"]);
Assert.AreEqual("30", task.Properties["Stats.StepCount"]);
Assert.AreEqual("28", task.Properties["Stats.TestCount"]);
}
public void RunReturnsCorrectResultCode(int resultCode, FacadeTestRunState expectedRunState)
{
StubbedLocalTestRunner tr = new StubbedLocalTestRunner();
Assembly assembly = typeof(TDNetAddInTests).Assembly;
string assemblyPath = AssemblyUtils.GetAssemblyLocalPath(assembly);
tr.SetRunLauncherAction(delegate
{
TestLauncherResult result = new TestLauncherResult(new Report());
result.SetResultCode(resultCode);
return result;
});
FacadeOptions facadeOptions = new FacadeOptions();
FacadeTestRunState runResult = tr.Run(MockRepository.GenerateStub<IFacadeTestListener>(), assemblyPath, null, facadeOptions);
Assert.AreEqual(expectedRunState, runResult);
}
private TestLauncherResult RunWithRuntime()
{
bool wasCanceled = false;
ITestProjectManager testProjectManager = RuntimeAccessor.ServiceLocator.Resolve <ITestProjectManager>();
TestProject consolidatedTestProject = ConsolidateTestProject(testProjectManager, ref wasCanceled);
if (wasCanceled)
{
return(CreateResult(ResultCode.Canceled, testProject));
}
if (consolidatedTestProject == null)
{
return(CreateResult(ResultCode.InvalidArguments, testProject));
}
if (consolidatedTestProject.TestPackage.Files.Count == 0)
{
logger.Log(LogSeverity.Warning, "No test files to execute!");
return(CreateResult(ResultCode.NoTests, consolidatedTestProject));
}
IReportManager reportManager = RuntimeAccessor.ServiceLocator.Resolve <IReportManager>();
if (!ValidateReportFormats(reportManager, consolidatedTestProject))
{
return(CreateResult(ResultCode.InvalidArguments, consolidatedTestProject));
}
ITestRunnerManager testRunnerManager = RuntimeAccessor.ServiceLocator.Resolve <ITestRunnerManager>();
ITestRunnerFactory testRunnerFactory = testRunnerManager.GetFactory(consolidatedTestProject.TestRunnerFactoryName);
if (testRunnerFactory == null)
{
logger.Log(LogSeverity.Error, String.Format("Unrecognized test runner factory name: '{0}'.", consolidatedTestProject.TestRunnerFactoryName));
return(CreateResult(ResultCode.InvalidArguments, consolidatedTestProject));
}
ITestRunner runner = testRunnerFactory.CreateTestRunner();
var result = new TestLauncherResult(new Report {
TestPackage = new TestPackageData(TestProject.TestPackage)
});
try
{
DoRegisterExtensions(runner, consolidatedTestProject);
DoInitialize(runner, ref wasCanceled);
if (!wasCanceled)
{
result = RunWithInitializedRunner(runner, consolidatedTestProject, reportManager);
}
}
finally
{
DoDispose(runner, ref wasCanceled);
}
if (wasCanceled)
{
result.SetResultCode(ResultCode.Canceled);
}
return(result);
}
public void IgnoreFailuresCausesTrueToBeReturnedEvenWhenFailuresOccur()
{
var task = new StubbedGallioTask();
task.IgnoreFailures = true;
task.SetRunLauncherAction(delegate
{
var result = new TestLauncherResult(new Report());
result.SetResultCode(ResultCode.Failure);
return result;
});
Assert.IsTrue(task.Execute());
}
public void TaskExposesResultsReturnedByLauncher()
{
var task = new StubbedGallioTask();
task.SetRunLauncherAction(delegate
{
var report = new Report();
report.TestPackageRun = new TestPackageRun();
report.TestPackageRun.Statistics.AssertCount = 42;
report.TestPackageRun.Statistics.Duration = 1.5;
report.TestPackageRun.Statistics.FailedCount = 5;
report.TestPackageRun.Statistics.InconclusiveCount = 11;
report.TestPackageRun.Statistics.PassedCount = 21;
report.TestPackageRun.Statistics.SkippedCount = 1;
report.TestPackageRun.Statistics.StepCount = 30;
report.TestPackageRun.Statistics.TestCount = 28;
var result = new TestLauncherResult(report);
result.SetResultCode(ResultCode.Failure);
return result;
});
Assert.IsFalse(task.InternalExecute());
Assert.AreEqual(ResultCode.Failure, task.ExitCode);
Assert.AreEqual(42, task.AssertCount);
Assert.AreEqual(1.5, task.Duration);
Assert.AreEqual(5, task.FailedCount);
Assert.AreEqual(11, task.InconclusiveCount);
Assert.AreEqual(21, task.PassedCount);
Assert.AreEqual(1, task.SkippedCount);
Assert.AreEqual(30, task.StepCount);
Assert.AreEqual(28, task.TestCount);
}
