public void TaskPassesDefaultArgumentsToLauncher()
{
var task = new StubbedGallioTask();
task.SetRunLauncherAction(launcher =>
{
Assert.IsFalse(launcher.DoNotRun);
Assert.IsTrue(launcher.EchoResults);
Assert.IsTrue(launcher.TestExecutionOptions.FilterSet.IsEmpty);
Assert.AreEqual(LogSeverity.Important, ((FilteredLogger)launcher.Logger).MinSeverity);
Assert.IsInstanceOfType(typeof(LogProgressMonitorProvider), launcher.ProgressMonitorProvider);
Assert.AreEqual("Reports", launcher.TestProject.ReportDirectory);
Assert.IsFalse(launcher.TestProject.IsReportDirectorySpecified);
Assert.AreElementsEqual(new string[] { }, launcher.ReportFormats);
Assert.AreEqual("test-report-{0}-{1}", launcher.TestProject.ReportNameFormat);
Assert.IsFalse(launcher.TestProject.IsReportNameFormatSpecified);
Assert.IsFalse(launcher.ShowReports);
Assert.IsNull(launcher.RunTimeLimit);
Assert.AreEqual(ReportArchive.Normal, launcher.TestProject.ReportArchive);
Assert.AreEqual(StandardTestRunnerFactoryNames.IsolatedProcess, launcher.TestProject.TestRunnerFactoryName);
Assert.IsFalse(launcher.TestProject.IsTestRunnerFactoryNameSpecified);
Assert.Count(0, launcher.TestProject.TestRunnerExtensions);
Assert.AreElementsEqual(new string[] { }, launcher.TestProject.TestRunnerExtensionSpecifications);
Assert.IsNull(launcher.RuntimeSetup.ConfigurationFilePath);
Assert.AreEqual(Path.GetDirectoryName(AssemblyUtils.GetAssemblyLocalPath(typeof(GallioTask).Assembly)), launcher.RuntimeSetup.RuntimePath);
Assert.AreElementsEqual(new string[] { }, launcher.RuntimeSetup.PluginDirectories);
Assert.AreElementsEqual(new string[] { }, from x in launcher.FilePatterns select x.ToString());
Assert.AreElementsEqual(new string[] { }, from x in launcher.TestProject.TestPackage.HintDirectories select x.ToString());
Assert.IsNull(launcher.TestProject.TestPackage.ApplicationBaseDirectory);
Assert.IsFalse(launcher.TestProject.TestPackage.IsApplicationBaseDirectorySpecified);
Assert.IsNull(launcher.TestProject.TestPackage.WorkingDirectory);
Assert.IsFalse(launcher.TestProject.TestPackage.IsWorkingDirectorySpecified);
Assert.IsFalse(launcher.TestProject.TestPackage.ShadowCopy);
Assert.IsFalse(launcher.TestProject.TestPackage.IsShadowCopySpecified);
Assert.IsNull(launcher.TestProject.TestPackage.DebuggerSetup);
Assert.IsFalse(launcher.TestProject.TestPackage.IsDebuggerSetupSpecified);
Assert.IsNull(launcher.TestProject.TestPackage.RuntimeVersion);
Assert.IsFalse(launcher.TestProject.TestPackage.IsRuntimeVersionSpecified);
Assert.AreEqual(new PropertySet(), launcher.TestRunnerOptions.Properties);
Assert.AreEqual(new PropertySet(), launcher.ReportFormatterOptions.Properties);
var result = new TestLauncherResult(new Report());
result.SetResultCode(ResultCode.Success);
return result;
});
Assert.DoesNotThrow(() => task.InternalExecute());
}
public void TaskPassesSpecifiedArgumentsToLauncher()
{
var task = new StubbedGallioTask();
task.DoNotRun = true;
task.EchoResults = false;
task.Filter = "Type: SimpleTest";
task.ReportDirectory = "dir";
task.ReportTypes = "XML;Html";
task.ReportNameFormat = "report";
task.ShowReports = true;
task.ReportArchive = "zip";
task.RunTimeLimit = 7200; // seconds
task.Verbosity = Verbosity.Debug;
task.RunnerType = StandardTestRunnerFactoryNames.Local;
task.RunnerExtensions.Add(new Argument("DebugExtension,Gallio"));
task.PluginDirectories = new DirSet[] { CreateDirSet("plugin") };
task.Files = new FileSet[] { CreateFileSet("assembly1"), CreateFileSet("assembly2") };
task.HintDirectories = new DirSet[] { CreateDirSet("hint1"), CreateDirSet("hint2") };
task.ApplicationBaseDirectory = "baseDir";
task.ShadowCopy = true;
task.Debug = true;
task.WorkingDirectory = "workingDir";
task.RuntimeVersion = "v4.0.30319";
task.RunnerProperties.AddRange(new[] { new Argument("RunnerOption1=RunnerValue1"), new Argument(" RunnerOption2 "), new Argument("RunnerOption3 = 'RunnerValue3'"), new Argument("RunnerOption4=\"'RunnerValue4'\"") });
task.ReportFormatterProperties.AddRange(new[] { new Argument("FormatterOption1=FormatterValue1"), new Argument(" FormatterOption2 "), new Argument("FormatterOption3 = 'FormatterValue3'"), new Argument("FormatterOption4=\"'FormatterValue4'\"") });
task.SetRunLauncherAction(launcher =>
{
Assert.IsTrue(launcher.DoNotRun);
Assert.IsFalse(launcher.EchoResults);
Assert.AreEqual("Type: SimpleTest", launcher.TestExecutionOptions.FilterSet.ToFilterSetExpr());
Assert.AreEqual(LogSeverity.Debug, ((FilteredLogger)launcher.Logger).MinSeverity);
Assert.IsInstanceOfType(typeof(LogProgressMonitorProvider), launcher.ProgressMonitorProvider);
Assert.AreEqual("dir", launcher.TestProject.ReportDirectory);
Assert.IsTrue(launcher.TestProject.IsReportDirectorySpecified);
Assert.AreElementsEqual(new string[] { "XML", "Html" }, launcher.ReportFormats);
Assert.AreEqual("report", launcher.TestProject.ReportNameFormat);
Assert.IsTrue(launcher.TestProject.IsReportNameFormatSpecified);
Assert.IsTrue(launcher.ShowReports);
Assert.AreEqual(TimeSpan.FromMinutes(120), launcher.RunTimeLimit);
Assert.AreEqual(ReportArchive.Zip, launcher.TestProject.ReportArchive);
Assert.AreEqual(StandardTestRunnerFactoryNames.Local, launcher.TestProject.TestRunnerFactoryName);
Assert.IsTrue(launcher.TestProject.IsTestRunnerFactoryNameSpecified);
Assert.AreEqual(0, launcher.TestProject.TestRunnerExtensions.Count);
Assert.AreElementsEqual(new[] { "DebugExtension,Gallio" }, launcher.TestProject.TestRunnerExtensionSpecifications);
Assert.IsNull(launcher.RuntimeSetup.ConfigurationFilePath);
Assert.AreEqual(Path.GetDirectoryName(AssemblyUtils.GetAssemblyLocalPath(typeof(GallioTask).Assembly)), launcher.RuntimeSetup.RuntimePath);
Assert.AreElementsEqual(new[] { "plugin" }, launcher.RuntimeSetup.PluginDirectories);
Assert.AreElementsEqual(new[] { "assembly1", "assembly2" }, from x in launcher.FilePatterns select x.ToString());
Assert.AreElementsEqual(new[] { "hint1", "hint2" }, from x in launcher.TestProject.TestPackage.HintDirectories select x.ToString());
Assert.AreEqual("baseDir", launcher.TestProject.TestPackage.ApplicationBaseDirectory.ToString());
Assert.IsTrue(launcher.TestProject.TestPackage.IsApplicationBaseDirectorySpecified);
Assert.AreEqual("workingDir", launcher.TestProject.TestPackage.WorkingDirectory.ToString());
Assert.IsTrue(launcher.TestProject.TestPackage.IsWorkingDirectorySpecified);
Assert.IsTrue(launcher.TestProject.TestPackage.ShadowCopy);
Assert.IsTrue(launcher.TestProject.TestPackage.IsShadowCopySpecified);
Assert.IsNotNull(launcher.TestProject.TestPackage.DebuggerSetup);
Assert.IsTrue(launcher.TestProject.TestPackage.IsDebuggerSetupSpecified);
Assert.AreEqual("v4.0.30319", launcher.TestProject.TestPackage.RuntimeVersion);
Assert.IsTrue(launcher.TestProject.TestPackage.IsRuntimeVersionSpecified);
Assert.AreEqual(new PropertySet()
{
{ "RunnerOption1", "RunnerValue1" },
{ "RunnerOption2", "" },
{ "RunnerOption3", "RunnerValue3" },
{ "RunnerOption4", "'RunnerValue4'" }
}, launcher.TestRunnerOptions.Properties);
Assert.AreEqual(new PropertySet()
{
{ "FormatterOption1", "FormatterValue1" },
{ "FormatterOption2", "" },
{ "FormatterOption3", "FormatterValue3" },
{ "FormatterOption4", "'FormatterValue4'" }
}, launcher.ReportFormatterOptions.Properties);
var result = new TestLauncherResult(new Report());
result.SetResultCode(ResultCode.NoTests);
return result;
});
Assert.DoesNotThrow(task.InternalExecute);
}
public void TaskExposesResultsReturnedByLauncher()
{
var task = new StubbedGallioTask();
task.ResultProperty = "ExitCode";
task.StatisticsPropertiesPrefix = "Stats.";
task.SetRunLauncherAction(launcher =>
{
var report = new Report();
report.TestPackageRun = new TestPackageRun();
report.TestPackageRun.Statistics.AssertCount = 42;
report.TestPackageRun.Statistics.Duration = 1.5;
report.TestPackageRun.Statistics.FailedCount = 5;
report.TestPackageRun.Statistics.InconclusiveCount = 11;
report.TestPackageRun.Statistics.PassedCount = 21;
report.TestPackageRun.Statistics.SkippedCount = 1;
report.TestPackageRun.Statistics.StepCount = 30;
report.TestPackageRun.Statistics.TestCount = 28;
var result = new TestLauncherResult(report);
result.SetResultCode(ResultCode.Failure);
return result;
});
Assert.Throws<BuildException>(task.InternalExecute);
Assert.AreEqual(ResultCode.Failure.ToString(), task.Properties["ExitCode"]);
Assert.AreEqual("42", task.Properties["Stats.AssertCount"]);
Assert.AreEqual("1.5", task.Properties["Stats.Duration"]);
Assert.AreEqual("5", task.Properties["Stats.FailedCount"]);
Assert.AreEqual("11", task.Properties["Stats.InconclusiveCount"]);
Assert.AreEqual("21", task.Properties["Stats.PassedCount"]);
Assert.AreEqual("1", task.Properties["Stats.SkippedCount"]);
Assert.AreEqual("30", task.Properties["Stats.StepCount"]);
Assert.AreEqual("28", task.Properties["Stats.TestCount"]);
}
public void FailOnErrorWhenSetToFalseCausesABuildExceptionToNotBeThrownOnFailures()
{
var task = new StubbedGallioTask();
task.FailOnError = false;
task.SetRunLauncherAction(delegate
{
var result = new TestLauncherResult(new Report());
result.SetResultCode(ResultCode.Failure);
return result;
});
Assert.DoesNotThrow(task.InternalExecute);
}
private void PopulateStatistics(TestLauncherResult result)
{
Statistics stats = result.Statistics;
AssertCount = stats.AssertCount;
Duration = stats.Duration;
FailedCount = stats.FailedCount;
InconclusiveCount = stats.InconclusiveCount;
PassedCount = stats.PassedCount;
SkippedCount = stats.SkippedCount;
RunCount = stats.RunCount;
StepCount = stats.StepCount;
TestCount = stats.TestCount;
}
private static void LogResultSummary(ILogger logger, TestLauncherResult result)
{
switch (result.ResultCode)
{
case ResultCode.Success:
logger.Log(LogSeverity.Info, result.ResultSummary);
break;
case ResultCode.Failure:
logger.Log(LogSeverity.Error, result.ResultSummary);
break;
}
}
public void TaskExposesResultsReturnedByLauncher()
{
var task = new StubbedRunGallioCommand();
var expectedResult = new TestLauncherResult(new Report());
task.SetRunLauncherAction(o => expectedResult);
Assert.AreSame(expectedResult, task.ExecuteWithMessagePump());
}
public void RunReturnsCorrectResultCode(int resultCode, FacadeTestRunState expectedRunState)
{
StubbedLocalTestRunner tr = new StubbedLocalTestRunner();
Assembly assembly = typeof(TDNetAddInTests).Assembly;
string assemblyPath = AssemblyUtils.GetAssemblyLocalPath(assembly);
tr.SetRunLauncherAction(delegate
{
TestLauncherResult result = new TestLauncherResult(new Report());
result.SetResultCode(resultCode);
return result;
});
FacadeOptions facadeOptions = new FacadeOptions();
FacadeTestRunState runResult = tr.Run(MockRepository.GenerateStub<IFacadeTestListener>(), assemblyPath, null, facadeOptions);
Assert.AreEqual(expectedRunState, runResult);
}
public void IgnoreFailuresCausesTrueToBeReturnedEvenWhenFailuresOccur()
{
var task = new StubbedGallioTask();
task.IgnoreFailures = true;
task.SetRunLauncherAction(delegate
{
var result = new TestLauncherResult(new Report());
result.SetResultCode(ResultCode.Failure);
return result;
});
Assert.IsTrue(task.Execute());
}
public void TaskExposesResultsReturnedByLauncher()
{
var task = new StubbedGallioTask();
task.SetRunLauncherAction(delegate
{
var report = new Report();
report.TestPackageRun = new TestPackageRun();
report.TestPackageRun.Statistics.AssertCount = 42;
report.TestPackageRun.Statistics.Duration = 1.5;
report.TestPackageRun.Statistics.FailedCount = 5;
report.TestPackageRun.Statistics.InconclusiveCount = 11;
report.TestPackageRun.Statistics.PassedCount = 21;
report.TestPackageRun.Statistics.SkippedCount = 1;
report.TestPackageRun.Statistics.StepCount = 30;
report.TestPackageRun.Statistics.TestCount = 28;
var result = new TestLauncherResult(report);
result.SetResultCode(ResultCode.Failure);
return result;
});
Assert.IsFalse(task.InternalExecute());
Assert.AreEqual(ResultCode.Failure, task.ExitCode);
Assert.AreEqual(42, task.AssertCount);
Assert.AreEqual(1.5, task.Duration);
Assert.AreEqual(5, task.FailedCount);
Assert.AreEqual(11, task.InconclusiveCount);
Assert.AreEqual(21, task.PassedCount);
Assert.AreEqual(1, task.SkippedCount);
Assert.AreEqual(30, task.StepCount);
Assert.AreEqual(28, task.TestCount);
}
private void Launch(bool doNoRun)
{
MarkupStreamWriter logStreamWriter = TestLog.Default;
var launcher = new TestLauncher();
launcher.TestProject.TestPackage = testPackage;
launcher.Logger = new MarkupStreamLogger(logStreamWriter);
launcher.TestExecutionOptions.FilterSet = new FilterSet<ITestDescriptor>(new OrFilter<ITestDescriptor>(filters));
launcher.TestProject.TestRunnerFactoryName = TestRunnerFactoryName;
string reportDirectory = SpecialPathPolicy.For<SampleRunner>().GetTempDirectory().FullName;
launcher.TestProject.ReportDirectory = reportDirectory;
launcher.TestProject.ReportNameFormat = "SampleRunnerReport";
launcher.ReportFormatterOptions.AddProperty(@"SaveAttachmentContents", @"false");
launcher.AddReportFormat(@"Text");
// Disabled because the Xml can get really big and causes problems if the sample runner is used frequently.
//launcher.AddReportFormat("Xml");
launcher.DoNotRun = doNoRun;
GenericCollectionUtils.ForEach(testRunnerOptions.Properties, x => launcher.TestRunnerOptions.AddProperty(x.Key, x.Value));
using (logStreamWriter.BeginSection("Log Output"))
result = launcher.Run();
using (logStreamWriter.BeginSection("Text Report"))
{
foreach (string reportPath in result.ReportDocumentPaths)
{
string extension = Path.GetExtension(reportPath);
if (extension == ".txt")
{
logStreamWriter.WriteLine(File.ReadAllText(reportPath));
}
else if (extension == ".xml")
{
logStreamWriter.Container.AttachXml(null, File.ReadAllText(reportPath));
}
File.Delete(reportPath);
}
}
}
private static FacadeTestRunState GetTestRunState(TestLauncherResult result)
{
switch (result.ResultCode)
{
default:
return FacadeTestRunState.Error;
case ResultCode.Failure:
case ResultCode.Canceled:
return FacadeTestRunState.Failure;
case ResultCode.NoTests:
case ResultCode.Success:
return FacadeTestRunState.Success;
}
}
private void PopulateStatistics(TestLauncherResult result)
{
Statistics stats = result.Statistics;
Properties[StatisticsPropertiesPrefix + @"TestCount"] = stats.TestCount.ToString();
Properties[StatisticsPropertiesPrefix + @"StepCount"] = stats.StepCount.ToString();
Properties[StatisticsPropertiesPrefix + @"PassedCount"] = stats.PassedCount.ToString();
Properties[StatisticsPropertiesPrefix + @"FailedCount"] = stats.FailedCount.ToString();
Properties[StatisticsPropertiesPrefix + @"InconclusiveCount"] = stats.InconclusiveCount.ToString();
Properties[StatisticsPropertiesPrefix + @"RunCount"] = stats.RunCount.ToString();
Properties[StatisticsPropertiesPrefix + @"SkippedCount"] = stats.SkippedCount.ToString();
Properties[StatisticsPropertiesPrefix + @"Duration"] = stats.Duration.ToString();
Properties[StatisticsPropertiesPrefix + @"AssertCount"] = stats.AssertCount.ToString();
}
