public void TaskExposesResultsReturnedByLauncher()
{
var task = new StubbedGallioTask();
task.ResultProperty = "ExitCode";
task.StatisticsPropertiesPrefix = "Stats.";
task.SetRunLauncherAction(launcher =>
{
var report = new Report();
report.TestPackageRun = new TestPackageRun();
report.TestPackageRun.Statistics.AssertCount = 42;
report.TestPackageRun.Statistics.Duration = 1.5;
report.TestPackageRun.Statistics.FailedCount = 5;
report.TestPackageRun.Statistics.InconclusiveCount = 11;
report.TestPackageRun.Statistics.PassedCount = 21;
report.TestPackageRun.Statistics.SkippedCount = 1;
report.TestPackageRun.Statistics.StepCount = 30;
report.TestPackageRun.Statistics.TestCount = 28;
var result = new TestLauncherResult(report);
result.SetResultCode(ResultCode.Failure);
return(result);
});
Assert.Throws <BuildException>(task.InternalExecute);
Assert.AreEqual(ResultCode.Failure.ToString(), task.Properties["ExitCode"]);
Assert.AreEqual("42", task.Properties["Stats.AssertCount"]);
Assert.AreEqual("1.5", task.Properties["Stats.Duration"]);
Assert.AreEqual("5", task.Properties["Stats.FailedCount"]);
Assert.AreEqual("11", task.Properties["Stats.InconclusiveCount"]);
Assert.AreEqual("21", task.Properties["Stats.PassedCount"]);
Assert.AreEqual("1", task.Properties["Stats.SkippedCount"]);
Assert.AreEqual("30", task.Properties["Stats.StepCount"]);
Assert.AreEqual("28", task.Properties["Stats.TestCount"]);
}
public void ExceptionsCauseTheTaskToFailRegardlessOfFailOnError()
{
var task = new StubbedGallioTask();
task.FailOnError = false;
task.SetRunLauncherAction(delegate
{
throw new Exception("Simulated error.");
});
Assert.Throws <Exception>(task.InternalExecute);
}
public void FailOnErrorWhenSetToFalseCausesABuildExceptionToNotBeThrownOnFailures()
{
var task = new StubbedGallioTask();
task.FailOnError = false;
task.SetRunLauncherAction(delegate
{
var result = new TestLauncherResult(new Report());
result.SetResultCode(ResultCode.Failure);
return(result);
});
Assert.DoesNotThrow(task.InternalExecute);
}
public void TaskPassesDefaultArgumentsToLauncher()
{
var task = new StubbedGallioTask();
task.SetRunLauncherAction(launcher =>
{
Assert.IsFalse(launcher.DoNotRun);
Assert.IsTrue(launcher.EchoResults);
Assert.IsTrue(launcher.TestExecutionOptions.FilterSet.IsEmpty);
Assert.AreEqual(LogSeverity.Important, ((FilteredLogger)launcher.Logger).MinSeverity);
Assert.IsInstanceOfType(typeof(LogProgressMonitorProvider), launcher.ProgressMonitorProvider);
Assert.AreEqual("Reports", launcher.TestProject.ReportDirectory);
Assert.IsFalse(launcher.TestProject.IsReportDirectorySpecified);
Assert.AreElementsEqual(new string[] { }, launcher.ReportFormats);
Assert.AreEqual("test-report-{0}-{1}", launcher.TestProject.ReportNameFormat);
Assert.IsFalse(launcher.TestProject.IsReportNameFormatSpecified);
Assert.IsFalse(launcher.ShowReports);
Assert.IsNull(launcher.RunTimeLimit);
Assert.AreEqual(ReportArchive.Normal, launcher.TestProject.ReportArchive);
Assert.AreEqual(StandardTestRunnerFactoryNames.IsolatedProcess, launcher.TestProject.TestRunnerFactoryName);
Assert.IsFalse(launcher.TestProject.IsTestRunnerFactoryNameSpecified);
Assert.Count(0, launcher.TestProject.TestRunnerExtensions);
Assert.AreElementsEqual(new string[] { }, launcher.TestProject.TestRunnerExtensionSpecifications);
Assert.IsNull(launcher.RuntimeSetup.ConfigurationFilePath);
Assert.AreEqual(Path.GetDirectoryName(AssemblyUtils.GetAssemblyLocalPath(typeof(GallioTask).Assembly)), launcher.RuntimeSetup.RuntimePath);
Assert.AreElementsEqual(new string[] { }, launcher.RuntimeSetup.PluginDirectories);
Assert.AreElementsEqual(new string[] { }, from x in launcher.FilePatterns select x.ToString());
Assert.AreElementsEqual(new string[] { }, from x in launcher.TestProject.TestPackage.HintDirectories select x.ToString());
Assert.IsNull(launcher.TestProject.TestPackage.ApplicationBaseDirectory);
Assert.IsFalse(launcher.TestProject.TestPackage.IsApplicationBaseDirectorySpecified);
Assert.IsNull(launcher.TestProject.TestPackage.WorkingDirectory);
Assert.IsFalse(launcher.TestProject.TestPackage.IsWorkingDirectorySpecified);
Assert.IsFalse(launcher.TestProject.TestPackage.ShadowCopy);
Assert.IsFalse(launcher.TestProject.TestPackage.IsShadowCopySpecified);
Assert.IsNull(launcher.TestProject.TestPackage.DebuggerSetup);
Assert.IsFalse(launcher.TestProject.TestPackage.IsDebuggerSetupSpecified);
Assert.IsNull(launcher.TestProject.TestPackage.RuntimeVersion);
Assert.IsFalse(launcher.TestProject.TestPackage.IsRuntimeVersionSpecified);
Assert.AreEqual(new PropertySet(), launcher.TestRunnerOptions.Properties);
Assert.AreEqual(new PropertySet(), launcher.ReportFormatterOptions.Properties);
var result = new TestLauncherResult(new Report());
result.SetResultCode(ResultCode.Success);
return result;
});
Assert.DoesNotThrow(() => task.InternalExecute());
}
public void TaskPassesDefaultArgumentsToLauncher()
{
var task = new StubbedGallioTask();
task.SetRunLauncherAction(launcher =>
{
Assert.IsFalse(launcher.DoNotRun);
Assert.IsTrue(launcher.EchoResults);
Assert.IsTrue(launcher.TestExecutionOptions.FilterSet.IsEmpty);
Assert.AreEqual(LogSeverity.Important, ((FilteredLogger)launcher.Logger).MinSeverity);
Assert.IsInstanceOfType(typeof(LogProgressMonitorProvider), launcher.ProgressMonitorProvider);
Assert.AreEqual("Reports", launcher.TestProject.ReportDirectory);
Assert.IsFalse(launcher.TestProject.IsReportDirectorySpecified);
Assert.AreElementsEqual(new string[] { }, launcher.ReportFormats);
Assert.AreEqual("test-report-{0}-{1}", launcher.TestProject.ReportNameFormat);
Assert.IsFalse(launcher.TestProject.IsReportNameFormatSpecified);
Assert.IsFalse(launcher.ShowReports);
Assert.IsNull(launcher.RunTimeLimit);
Assert.AreEqual(ReportArchive.Normal, launcher.TestProject.ReportArchive);
Assert.AreEqual(StandardTestRunnerFactoryNames.IsolatedProcess, launcher.TestProject.TestRunnerFactoryName);
Assert.IsFalse(launcher.TestProject.IsTestRunnerFactoryNameSpecified);
Assert.Count(0, launcher.TestProject.TestRunnerExtensions);
Assert.AreElementsEqual(new string[] { }, launcher.TestProject.TestRunnerExtensionSpecifications);
Assert.IsNull(launcher.RuntimeSetup.ConfigurationFilePath);
Assert.AreEqual(Path.GetDirectoryName(AssemblyUtils.GetAssemblyLocalPath(typeof(GallioTask).Assembly)), launcher.RuntimeSetup.RuntimePath);
Assert.AreElementsEqual(new string[] { }, launcher.RuntimeSetup.PluginDirectories);
Assert.AreElementsEqual(new string[] { }, from x in launcher.FilePatterns select x.ToString());
Assert.AreElementsEqual(new string[] { }, from x in launcher.TestProject.TestPackage.HintDirectories select x.ToString());
Assert.IsNull(launcher.TestProject.TestPackage.ApplicationBaseDirectory);
Assert.IsFalse(launcher.TestProject.TestPackage.IsApplicationBaseDirectorySpecified);
Assert.IsNull(launcher.TestProject.TestPackage.WorkingDirectory);
Assert.IsFalse(launcher.TestProject.TestPackage.IsWorkingDirectorySpecified);
Assert.IsFalse(launcher.TestProject.TestPackage.ShadowCopy);
Assert.IsFalse(launcher.TestProject.TestPackage.IsShadowCopySpecified);
Assert.IsNull(launcher.TestProject.TestPackage.DebuggerSetup);
Assert.IsFalse(launcher.TestProject.TestPackage.IsDebuggerSetupSpecified);
Assert.IsNull(launcher.TestProject.TestPackage.RuntimeVersion);
Assert.IsFalse(launcher.TestProject.TestPackage.IsRuntimeVersionSpecified);
Assert.AreEqual(new PropertySet(), launcher.TestRunnerOptions.Properties);
Assert.AreEqual(new PropertySet(), launcher.ReportFormatterOptions.Properties);
var result = new TestLauncherResult(new Report());
result.SetResultCode(ResultCode.Success);
return(result);
});
Assert.DoesNotThrow(() => task.InternalExecute());
}
public void TaskPassesSpecifiedArgumentsToLauncher()
{
var task = new StubbedGallioTask();
task.DoNotRun = true;
task.EchoResults = false;
task.Filter = "Type: SimpleTest";
task.ReportDirectory = "dir";
task.ReportTypes = "XML;Html";
task.ReportNameFormat = "report";
task.ShowReports = true;
task.ReportArchive = "zip";
task.RunTimeLimit = 7200; // seconds
task.Verbosity = Verbosity.Debug;
task.RunnerType = StandardTestRunnerFactoryNames.Local;
task.RunnerExtensions.Add(new Argument("DebugExtension,Gallio"));
task.PluginDirectories = new DirSet[] { CreateDirSet("plugin") };
task.Files = new FileSet[] { CreateFileSet("assembly1"), CreateFileSet("assembly2") };
task.HintDirectories = new DirSet[] { CreateDirSet("hint1"), CreateDirSet("hint2") };
task.ApplicationBaseDirectory = "baseDir";
task.ShadowCopy = true;
task.Debug = true;
task.WorkingDirectory = "workingDir";
task.RuntimeVersion = "v4.0.30319";
task.RunnerProperties.AddRange(new[] { new Argument("RunnerOption1=RunnerValue1"), new Argument(" RunnerOption2 "), new Argument("RunnerOption3 = 'RunnerValue3'"), new Argument("RunnerOption4=\"'RunnerValue4'\"") });
task.ReportFormatterProperties.AddRange(new[] { new Argument("FormatterOption1=FormatterValue1"), new Argument(" FormatterOption2 "), new Argument("FormatterOption3 = 'FormatterValue3'"), new Argument("FormatterOption4=\"'FormatterValue4'\"") });
task.SetRunLauncherAction(launcher =>
{
Assert.IsTrue(launcher.DoNotRun);
Assert.IsFalse(launcher.EchoResults);
Assert.AreEqual("Type: SimpleTest", launcher.TestExecutionOptions.FilterSet.ToFilterSetExpr());
Assert.AreEqual(LogSeverity.Debug, ((FilteredLogger)launcher.Logger).MinSeverity);
Assert.IsInstanceOfType(typeof(LogProgressMonitorProvider), launcher.ProgressMonitorProvider);
Assert.AreEqual("dir", launcher.TestProject.ReportDirectory);
Assert.IsTrue(launcher.TestProject.IsReportDirectorySpecified);
Assert.AreElementsEqual(new string[] { "XML", "Html" }, launcher.ReportFormats);
Assert.AreEqual("report", launcher.TestProject.ReportNameFormat);
Assert.IsTrue(launcher.TestProject.IsReportNameFormatSpecified);
Assert.IsTrue(launcher.ShowReports);
Assert.AreEqual(TimeSpan.FromMinutes(120), launcher.RunTimeLimit);
Assert.AreEqual(ReportArchive.Zip, launcher.TestProject.ReportArchive);
Assert.AreEqual(StandardTestRunnerFactoryNames.Local, launcher.TestProject.TestRunnerFactoryName);
Assert.IsTrue(launcher.TestProject.IsTestRunnerFactoryNameSpecified);
Assert.AreEqual(0, launcher.TestProject.TestRunnerExtensions.Count);
Assert.AreElementsEqual(new[] { "DebugExtension,Gallio" }, launcher.TestProject.TestRunnerExtensionSpecifications);
Assert.IsNull(launcher.RuntimeSetup.ConfigurationFilePath);
Assert.AreEqual(Path.GetDirectoryName(AssemblyUtils.GetAssemblyLocalPath(typeof(GallioTask).Assembly)), launcher.RuntimeSetup.RuntimePath);
Assert.AreElementsEqual(new[] { "plugin" }, launcher.RuntimeSetup.PluginDirectories);
Assert.AreElementsEqual(new[] { "assembly1", "assembly2" }, from x in launcher.FilePatterns select x.ToString());
Assert.AreElementsEqual(new[] { "hint1", "hint2" }, from x in launcher.TestProject.TestPackage.HintDirectories select x.ToString());
Assert.AreEqual("baseDir", launcher.TestProject.TestPackage.ApplicationBaseDirectory.ToString());
Assert.IsTrue(launcher.TestProject.TestPackage.IsApplicationBaseDirectorySpecified);
Assert.AreEqual("workingDir", launcher.TestProject.TestPackage.WorkingDirectory.ToString());
Assert.IsTrue(launcher.TestProject.TestPackage.IsWorkingDirectorySpecified);
Assert.IsTrue(launcher.TestProject.TestPackage.ShadowCopy);
Assert.IsTrue(launcher.TestProject.TestPackage.IsShadowCopySpecified);
Assert.IsNotNull(launcher.TestProject.TestPackage.DebuggerSetup);
Assert.IsTrue(launcher.TestProject.TestPackage.IsDebuggerSetupSpecified);
Assert.AreEqual("v4.0.30319", launcher.TestProject.TestPackage.RuntimeVersion);
Assert.IsTrue(launcher.TestProject.TestPackage.IsRuntimeVersionSpecified);
Assert.AreEqual(new PropertySet()
{
{ "RunnerOption1", "RunnerValue1" },
{ "RunnerOption2", "" },
{ "RunnerOption3", "RunnerValue3" },
{ "RunnerOption4", "'RunnerValue4'" }
}, launcher.TestRunnerOptions.Properties);
Assert.AreEqual(new PropertySet()
{
{ "FormatterOption1", "FormatterValue1" },
{ "FormatterOption2", "" },
{ "FormatterOption3", "FormatterValue3" },
{ "FormatterOption4", "'FormatterValue4'" }
}, launcher.ReportFormatterOptions.Properties);
var result = new TestLauncherResult(new Report());
result.SetResultCode(ResultCode.NoTests);
return result;
});
Assert.DoesNotThrow(task.InternalExecute);
}
public void ExceptionsCauseTheTaskToFailRegardlessOfFailOnError()
{
var task = new StubbedGallioTask();
task.FailOnError = false;
task.SetRunLauncherAction(delegate
{
throw new Exception("Simulated error.");
});
Assert.Throws<Exception>(task.InternalExecute);
}
public void FailOnErrorWhenSetToFalseCausesABuildExceptionToNotBeThrownOnFailures()
{
var task = new StubbedGallioTask();
task.FailOnError = false;
task.SetRunLauncherAction(delegate
{
var result = new TestLauncherResult(new Report());
result.SetResultCode(ResultCode.Failure);
return result;
});
Assert.DoesNotThrow(task.InternalExecute);
}
public void TaskExposesResultsReturnedByLauncher()
{
var task = new StubbedGallioTask();
task.ResultProperty = "ExitCode";
task.StatisticsPropertiesPrefix = "Stats.";
task.SetRunLauncherAction(launcher =>
{
var report = new Report();
report.TestPackageRun = new TestPackageRun();
report.TestPackageRun.Statistics.AssertCount = 42;
report.TestPackageRun.Statistics.Duration = 1.5;
report.TestPackageRun.Statistics.FailedCount = 5;
report.TestPackageRun.Statistics.InconclusiveCount = 11;
report.TestPackageRun.Statistics.PassedCount = 21;
report.TestPackageRun.Statistics.SkippedCount = 1;
report.TestPackageRun.Statistics.StepCount = 30;
report.TestPackageRun.Statistics.TestCount = 28;
var result = new TestLauncherResult(report);
result.SetResultCode(ResultCode.Failure);
return result;
});
Assert.Throws<BuildException>(task.InternalExecute);
Assert.AreEqual(ResultCode.Failure.ToString(), task.Properties["ExitCode"]);
Assert.AreEqual("42", task.Properties["Stats.AssertCount"]);
Assert.AreEqual("1.5", task.Properties["Stats.Duration"]);
Assert.AreEqual("5", task.Properties["Stats.FailedCount"]);
Assert.AreEqual("11", task.Properties["Stats.InconclusiveCount"]);
Assert.AreEqual("21", task.Properties["Stats.PassedCount"]);
Assert.AreEqual("1", task.Properties["Stats.SkippedCount"]);
Assert.AreEqual("30", task.Properties["Stats.StepCount"]);
Assert.AreEqual("28", task.Properties["Stats.TestCount"]);
}
public void TaskPassesSpecifiedArgumentsToLauncher()
{
var task = new StubbedGallioTask();
task.DoNotRun = true;
task.EchoResults = false;
task.Filter = "Type: SimpleTest";
task.ReportDirectory = "dir";
task.ReportTypes = "XML;Html";
task.ReportNameFormat = "report";
task.ShowReports = true;
task.ReportArchive = "zip";
task.RunTimeLimit = 7200; // seconds
task.Verbosity = Verbosity.Debug;
task.RunnerType = StandardTestRunnerFactoryNames.Local;
task.RunnerExtensions.Add(new Argument("DebugExtension,Gallio"));
task.PluginDirectories = new DirSet[] { CreateDirSet("plugin") };
task.Files = new FileSet[] { CreateFileSet("assembly1"), CreateFileSet("assembly2") };
task.HintDirectories = new DirSet[] { CreateDirSet("hint1"), CreateDirSet("hint2") };
task.ApplicationBaseDirectory = "baseDir";
task.ShadowCopy = true;
task.Debug = true;
task.WorkingDirectory = "workingDir";
task.RuntimeVersion = "v4.0.30319";
task.RunnerProperties.AddRange(new[] { new Argument("RunnerOption1=RunnerValue1"), new Argument(" RunnerOption2 "), new Argument("RunnerOption3 = 'RunnerValue3'"), new Argument("RunnerOption4=\"'RunnerValue4'\"") });
task.ReportFormatterProperties.AddRange(new[] { new Argument("FormatterOption1=FormatterValue1"), new Argument(" FormatterOption2 "), new Argument("FormatterOption3 = 'FormatterValue3'"), new Argument("FormatterOption4=\"'FormatterValue4'\"") });
task.SetRunLauncherAction(launcher =>
{
Assert.IsTrue(launcher.DoNotRun);
Assert.IsFalse(launcher.EchoResults);
Assert.AreEqual("Type: SimpleTest", launcher.TestExecutionOptions.FilterSet.ToFilterSetExpr());
Assert.AreEqual(LogSeverity.Debug, ((FilteredLogger)launcher.Logger).MinSeverity);
Assert.IsInstanceOfType(typeof(LogProgressMonitorProvider), launcher.ProgressMonitorProvider);
Assert.AreEqual("dir", launcher.TestProject.ReportDirectory);
Assert.IsTrue(launcher.TestProject.IsReportDirectorySpecified);
Assert.AreElementsEqual(new string[] { "XML", "Html" }, launcher.ReportFormats);
Assert.AreEqual("report", launcher.TestProject.ReportNameFormat);
Assert.IsTrue(launcher.TestProject.IsReportNameFormatSpecified);
Assert.IsTrue(launcher.ShowReports);
Assert.AreEqual(TimeSpan.FromMinutes(120), launcher.RunTimeLimit);
Assert.AreEqual(ReportArchive.Zip, launcher.TestProject.ReportArchive);
Assert.AreEqual(StandardTestRunnerFactoryNames.Local, launcher.TestProject.TestRunnerFactoryName);
Assert.IsTrue(launcher.TestProject.IsTestRunnerFactoryNameSpecified);
Assert.AreEqual(0, launcher.TestProject.TestRunnerExtensions.Count);
Assert.AreElementsEqual(new[] { "DebugExtension,Gallio" }, launcher.TestProject.TestRunnerExtensionSpecifications);
Assert.IsNull(launcher.RuntimeSetup.ConfigurationFilePath);
Assert.AreEqual(Path.GetDirectoryName(AssemblyUtils.GetAssemblyLocalPath(typeof(GallioTask).Assembly)), launcher.RuntimeSetup.RuntimePath);
Assert.AreElementsEqual(new[] { "plugin" }, launcher.RuntimeSetup.PluginDirectories);
Assert.AreElementsEqual(new[] { "assembly1", "assembly2" }, from x in launcher.FilePatterns select x.ToString());
Assert.AreElementsEqual(new[] { "hint1", "hint2" }, from x in launcher.TestProject.TestPackage.HintDirectories select x.ToString());
Assert.AreEqual("baseDir", launcher.TestProject.TestPackage.ApplicationBaseDirectory.ToString());
Assert.IsTrue(launcher.TestProject.TestPackage.IsApplicationBaseDirectorySpecified);
Assert.AreEqual("workingDir", launcher.TestProject.TestPackage.WorkingDirectory.ToString());
Assert.IsTrue(launcher.TestProject.TestPackage.IsWorkingDirectorySpecified);
Assert.IsTrue(launcher.TestProject.TestPackage.ShadowCopy);
Assert.IsTrue(launcher.TestProject.TestPackage.IsShadowCopySpecified);
Assert.IsNotNull(launcher.TestProject.TestPackage.DebuggerSetup);
Assert.IsTrue(launcher.TestProject.TestPackage.IsDebuggerSetupSpecified);
Assert.AreEqual("v4.0.30319", launcher.TestProject.TestPackage.RuntimeVersion);
Assert.IsTrue(launcher.TestProject.TestPackage.IsRuntimeVersionSpecified);
Assert.AreEqual(new PropertySet()
{
{ "RunnerOption1", "RunnerValue1" },
{ "RunnerOption2", "" },
{ "RunnerOption3", "RunnerValue3" },
{ "RunnerOption4", "'RunnerValue4'" }
}, launcher.TestRunnerOptions.Properties);
Assert.AreEqual(new PropertySet()
{
{ "FormatterOption1", "FormatterValue1" },
{ "FormatterOption2", "" },
{ "FormatterOption3", "FormatterValue3" },
{ "FormatterOption4", "'FormatterValue4'" }
}, launcher.ReportFormatterOptions.Properties);
var result = new TestLauncherResult(new Report());
result.SetResultCode(ResultCode.NoTests);
return(result);
});
Assert.DoesNotThrow(task.InternalExecute);
}