private TestPackage MakeTestPackage(string testName)
{
TestPackage package = TestProject.ActiveConfig.MakeTestPackage();
package.TestName = testName;
ISettings settings = Services.UserSettings;
package.Settings["MergeAssemblies"] = settings.GetSetting("Options.TestLoader.MergeAssemblies", false);
package.Settings["AutoNamespaceSuites"] = settings.GetSetting("Options.TestLoader.AutoNamespaceSuites", true);
package.Settings["ShadowCopyFiles"] = settings.GetSetting("Options.TestLoader.ShadowCopyFiles", true);
ProcessModel processModel = (ProcessModel)settings.GetSetting("Options.TestLoader.ProcessModel", ProcessModel.Default);
DomainUsage domainUsage = (DomainUsage)settings.GetSetting("Options.TestLoader.DomainUsage", DomainUsage.Default);
if (processModel != ProcessModel.Default && !package.Settings.Contains("ProcessModel"))
{
package.Settings["ProcessModel"] = processModel;
}
if (processModel != ProcessModel.Multiple && domainUsage == DomainUsage.Multiple &&
!package.Settings.Contains("DomainUsage"))
{
package.Settings["DomainUsage"] = domainUsage;
}
return(package);
}
private static RunnerResult GetProcessModelInProcessResult(DomainUsage domainUsage)
{
switch (domainUsage)
{
case DomainUsage.Default:
case DomainUsage.Multiple:
return(new RunnerResult
{
TestRunner = typeof(MultipleTestDomainRunner),
SubRunners = new[]
{
RunnerResult.TestDomainRunner,
RunnerResult.TestDomainRunner
}
});
case DomainUsage.None:
return(RunnerResult.LocalTestRunner);
case DomainUsage.Single:
return(RunnerResult.TestDomainRunner);
default:
throw new ArgumentOutOfRangeException(nameof(domainUsage), domainUsage, ExceptionMessage);
}
}
/// <summary>
/// Returns a test runner based on the settings in a TestPackage.
/// Any setting that is "consumed" by the factory is removed, so
/// that downstream runners using the factory will not repeatedly
/// create the same type of runner.
/// </summary>
/// <param name="package">The TestPackage to be loaded and run</param>
/// <returns>An ITestEngineRunner</returns>
public virtual ITestEngineRunner MakeTestRunner(TestPackage package)
{
DomainUsage domainUsage = (DomainUsage)System.Enum.Parse(
typeof(DomainUsage),
package.GetSetting(EnginePackageSettings.DomainUsage, "Default"));
switch (domainUsage)
{
default:
case DomainUsage.Default:
if (package.SubPackages.Count > 1)
{
return(new MultipleTestDomainRunner(this.ServiceContext, package));
}
else
{
return(new TestDomainRunner(this.ServiceContext, package));
}
case DomainUsage.Multiple:
return(new MultipleTestDomainRunner(ServiceContext, package));
case DomainUsage.None:
return(new LocalTestRunner(ServiceContext, package));
case DomainUsage.Single:
return(new TestDomainRunner(ServiceContext, package));
}
}
private static RunnerResult GetProcessModelSeparateResult(DomainUsage domainUsage)
{
switch (domainUsage)
{
case DomainUsage.Default:
return(new RunnerResult
{
TestRunner = typeof(ProcessRunner)
});
case DomainUsage.None:
return(new RunnerResult
{
TestRunner = typeof(ProcessRunner)
});
case DomainUsage.Single:
return(new RunnerResult
{
TestRunner = typeof(ProcessRunner)
});
case DomainUsage.Multiple:
return(new RunnerResult
{
TestRunner = typeof(ProcessRunner)
});
default:
ThrowOutOfRange(domainUsage);
break;
}
return(null);
}
private TestPackage createPackage(Options options)
{
TestPackage package;
DomainUsage domainUsage = DomainUsage.Default;
ProcessModel processModel = ProcessModel.Default;
package = new TestPackage(options.Assembly);
domainUsage = DomainUsage.Single;
package.TestName = null;
package.Settings["ProcessModel"] = processModel;
package.Settings["DomainUsage"] = domainUsage;
//if (framework != null)
//package.Settings["RuntimeFramework"] = Environment.Version.ToString();
if (domainUsage == DomainUsage.None)
{
CoreExtensions.Host.AddinRegistry = Services.AddinRegistry;
}
package.Settings["ShadowCopyFiles"] = false;
package.Settings["UseThreadedRunner"] = false;
package.Settings["DefaultTimeout"] = 0;
return(package);
}
private static RunnerResult GetProcessModelSeparateResult(DomainUsage domainUsage)
{
switch (domainUsage)
{
case DomainUsage.Default:
case DomainUsage.None:
case DomainUsage.Single:
case DomainUsage.Multiple:
return(new RunnerResult
{
TestRunner = typeof(AggregatingTestRunner),
SubRunners = new[]
{
new RunnerResult {
TestRunner = typeof(ProcessRunner)
},
new RunnerResult {
TestRunner = typeof(ProcessRunner)
}
}
});
default:
throw new ArgumentOutOfRangeException(nameof(domainUsage), domainUsage, ExceptionMessage);
}
}
private int CheckDomain([NotNull] DomainUsage domainUsage)
{
Assert.ArgumentNotNull(domainUsage, nameof(domainUsage));
string domainName = domainUsage.DomainName;
const int noError = 0;
if (_matchIsError)
{
if (!_regex.Match(domainName).Success)
{
// no match -> correct
return(noError);
}
// the pattern matches, and a match is considered an error
return(ReportSchemaPropertyError(
Codes[Code.MatchingErrorPattern], domainName,
GetMatchErrorDescription(domainName)));
}
if (_regex.Match(domainName).Success)
{
// the pattern matches, and a match is considered correct
return(noError);
}
// no match -> error
return(ReportSchemaPropertyError(
Codes[Code.NotMatchingExpectedPattern], domainName,
GetNoMatchErrorDescription(domainName)));
}
/// <summary>
/// Returns a test runner based on the settings in a TestPackage.
/// Any setting that is "consumed" by the factory is removed, so
/// that downstream runners using the factory will not repeatedly
/// create the same type of runner.
/// </summary>
/// <param name="package">The TestPackage to be loaded and run</param>
/// <returns>An ITestEngineRunner</returns>
public virtual ITestEngineRunner MakeTestRunner(TestPackage package)
{
#if NETSTANDARD1_6 || NETSTANDARD2_0
return new LocalTestRunner(ServiceContext, package);
#else
DomainUsage domainUsage = (DomainUsage)System.Enum.Parse(
typeof(DomainUsage),
package.GetSetting(EnginePackageSettings.DomainUsage, "Default"));
switch (domainUsage)
{
default:
case DomainUsage.Default:
case DomainUsage.Multiple:
if (package.AssemblyPackages().Count > 1)
return new MultipleTestDomainRunner(this.ServiceContext, package);
else
return new TestDomainRunner(this.ServiceContext, package);
case DomainUsage.None:
return new LocalTestRunner(ServiceContext, package);
case DomainUsage.Single:
return new TestDomainRunner(ServiceContext, package);
}
#endif
}
public static RunnerResult ResultFor(ProcessModel processModel, DomainUsage domainUsage, int numAssemblies)
{
switch (processModel)
{
case ProcessModel.Default:
case ProcessModel.Multiple:
switch (domainUsage)
{
case DomainUsage.Default:
case DomainUsage.None:
case DomainUsage.Single:
case DomainUsage.Multiple:
return(numAssemblies > 1
? new RunnerResult()
{
TestRunner = typeof(MultipleTestProcessRunner),
SubRunners = GetSubRunners(RunnerResult.ProcessRunner, numAssemblies)
}
: RunnerResult.ProcessRunner);
}
break;
case ProcessModel.InProcess:
switch (domainUsage)
{
case DomainUsage.Default:
case DomainUsage.Multiple:
return(numAssemblies > 1
? new RunnerResult
{
TestRunner = typeof(MultipleTestDomainRunner),
SubRunners = GetSubRunners(RunnerResult.TestDomainRunner, numAssemblies)
}
: RunnerResult.TestDomainRunner);
case DomainUsage.None:
return(RunnerResult.LocalTestRunner);
case DomainUsage.Single:
return(RunnerResult.TestDomainRunner);
}
break;
case ProcessModel.Separate:
switch (domainUsage)
{
case DomainUsage.Default:
case DomainUsage.None:
case DomainUsage.Single:
case DomainUsage.Multiple:
return(RunnerResult.ProcessRunner);
}
break;
}
throw new ArgumentOutOfRangeException(nameof(domainUsage), domainUsage, ExceptionMessage);
}
private TestResults RunTests(TestPackage package, TestFilter filter, Messages.OnMessage[] subscribers)
{
NUnit.Core.TestResult result;
ProcessModel processModel = package.Settings.Contains("ProcessModel") ? (ProcessModel)package.Settings["ProcessModel"] : ProcessModel.Default;
DomainUsage domainUsage = package.Settings.Contains("DomainUsage") ? (DomainUsage)package.Settings[(object)"DomainUsage"] : DomainUsage.Default;
RuntimeFramework runtimeFramework = package.Settings.Contains("RuntimeFramework") ? (RuntimeFramework)package.Settings["RuntimeFramework"] : RuntimeFramework.CurrentFramework;
using (new SaveConsoleOutput())
using (TestRunner testRunner = new DefaultTestRunnerFactory().MakeTestRunner(package))
using (var ms = new MemoryStream())
using (var output = CreateOutputWriter(ms))
{
try
{
TestEventHandler eventCollector = new TestEventHandler(_options, output, subscribers);
testRunner.Load(package);
if (testRunner.Test == null)
{
testRunner.Unload();
return(new TestResults(TestResults.Code.FixtureNotFound, "Unable to locate fixture"));
}
result = testRunner.Run(eventCollector, filter, false, LoggingThreshold.All);
var summary = eventCollector.GetSummary();
output.Flush();
if (summary.UnexpectedError)
{
return(new TestResults(TestResults.Code.UnexpectedError, GetResultText(ms), summary));
}
return(new TestResults(summary.InvalidAssemblies > 0
? TestResults.Code.InvalidAssembly
: GetCode(summary.FailureCount + summary.ErrorCount + summary.InvalidCount),
GetResultText(ms), summary));
}
catch (FileNotFoundException ex)
{
output.WriteLine(ex.Message);
output.Flush();
return(new TestResults(TestResults.Code.InvalidAssembly, GetResultText(ms)));
}
catch (DirectoryNotFoundException ex)
{
output.WriteLine(ex.Message);
output.Flush();
return(new TestResults(TestResults.Code.InvalidAssembly, GetResultText(ms)));
}
catch (Exception ex)
{
output.WriteLine(ex.ToString());
output.Flush();
return(new TestResults(TestResults.Code.UnexpectedError, GetResultText(ms)));
}
}
}
private static RunnerResult GetProcessModelInProcessResult(DomainUsage domainUsage)
{
switch (domainUsage)
{
case DomainUsage.Default:
return(new RunnerResult
{
TestRunner = typeof(MultipleTestDomainRunner),
SubRunners = new[]
{
new RunnerResult {
TestRunner = typeof(TestDomainRunner)
},
new RunnerResult {
TestRunner = typeof(TestDomainRunner)
}
}
});
case DomainUsage.None:
return(new RunnerResult
{
TestRunner = typeof(LocalTestRunner)
});
case DomainUsage.Single:
return(new RunnerResult
{
TestRunner = typeof(TestDomainRunner)
});
case DomainUsage.Multiple:
return(new RunnerResult
{
TestRunner = typeof(MultipleTestDomainRunner),
SubRunners = new[]
{
new RunnerResult {
TestRunner = typeof(TestDomainRunner)
},
new RunnerResult {
TestRunner = typeof(TestDomainRunner)
}
}
});
default:
ThrowOutOfRange(domainUsage);
break;
}
return(null);
}
/// <summary>
/// Returns a test runner based on the settings in a TestPackage.
/// Any setting that is "consumed" by the factory is removed, so
/// that downstream runners using the factory will not repeatedly
/// create the same type of runner.
/// </summary>
/// <param name="package">The TestPackage to be loaded and run</param>
/// <returns>A TestRunner</returns>
public static TestRunner MakeTestRunner(TestPackage package)
{
RuntimeFramework currentFramework = RuntimeFramework.CurrentFramework;
RuntimeFramework targetFramework = package.Settings["RuntimeFramework"] as RuntimeFramework;
if (targetFramework == null)
{
targetFramework = currentFramework;
}
else if (targetFramework.Runtime == RuntimeType.Any)
{
targetFramework = new RuntimeFramework(currentFramework.Runtime, targetFramework.Version);
package.Settings["RuntimeFramework"] = targetFramework;
}
log.Debug("Test requires {0} framework", targetFramework);
ProcessModel processModel = (ProcessModel)package.GetSetting("ProcessModel", ProcessModel.Default);
if (processModel == ProcessModel.Default)
{
if (!targetFramework.Matches(currentFramework))
{
processModel = ProcessModel.Separate;
}
}
switch (processModel)
{
case ProcessModel.Multiple:
package.Settings.Remove("ProcessModel");
return(new MultipleTestProcessRunner());
case ProcessModel.Separate:
package.Settings.Remove("ProcessModel");
return(new ProcessRunner());
default:
DomainUsage domainUsage =
(DomainUsage)package.GetSetting("DomainUsage", DomainUsage.Default);
if (domainUsage == DomainUsage.Multiple)
{
package.Settings.Remove("DomainUsage");
return(new MultipleTestDomainRunner());
}
else
{
return(new TestDomain());
}
}
}
private static RunnerResult GetProcessModelSeparateResult(DomainUsage domainUsage)
{
switch (domainUsage)
{
case DomainUsage.Default:
case DomainUsage.None:
case DomainUsage.Single:
case DomainUsage.Multiple:
return(RunnerResult.ProcessRunner);
default:
throw new ArgumentOutOfRangeException(nameof(domainUsage), domainUsage, ExceptionMessage);
}
}
private int CheckUniqueNames([NotNull] DomainUsage domainUsage,
[NotNull] ICollection <CodedValue> codedValues)
{
SimpleSet <string> uniqueNames = CreateUniqueNamesSet(_uniqueNamesConstraint);
if (uniqueNames == null)
{
return(NoError);
}
var nonUniqueNames = new List <string>();
foreach (CodedValue codedValue in codedValues)
{
// gather non-unique names, uniqueness check will be done at end
string name = codedValue.Name.Trim();
if (uniqueNames.Contains(name))
{
nonUniqueNames.Add(name);
}
else
{
uniqueNames.Add(name);
}
}
bool caseSensitive = _uniqueNamesConstraint ==
UniqueStringsConstraint.UniqueExactCase;
int errorCount = 0;
foreach (string nonUniqueName in nonUniqueNames)
{
string description =
string.Format(
"Name '{0}' in coded value domain '{1}' is non-unique. The following values have the same name: {2}",
nonUniqueName, domainUsage.DomainName,
StringUtils.Concatenate(
GetValuesForName(nonUniqueName, codedValues, caseSensitive), ", "));
errorCount += ReportSchemaPropertyError(Codes[Code.NamesNotUnique],
domainUsage.DomainName,
new object[] { nonUniqueName },
description);
}
return(errorCount);
}
private static RunnerResult GetProcessModelDefaultResult(DomainUsage domainUsage)
{
switch (domainUsage)
{
case DomainUsage.Default:
case DomainUsage.None:
case DomainUsage.Single:
case DomainUsage.Multiple:
return(new RunnerResult(typeof(AggregatingTestRunner),
RunnerResult.MultiRunnerWithTwoSubRunners));
default:
throw new ArgumentOutOfRangeException(nameof(domainUsage), domainUsage, ExceptionMessage);
}
}
private static TestCaseData TwoUnknownsTest(ProcessModel processModel, DomainUsage domainUsage)
{
var testName = "Two unknown extensions - " +
$"{nameof(EnginePackageSettings.ProcessModel)}:{processModel} " +
$"{nameof(EnginePackageSettings.DomainUsage)}:{domainUsage}";
var package = TestPackageFactory.TwoUnknownExtension();
package.AddSetting(EnginePackageSettings.ProcessModel, processModel.ToString());
package.AddSetting(EnginePackageSettings.DomainUsage, domainUsage.ToString());
var expected = Net20TwoAssemblyExpectedRunnerResults.ResultFor(processModel, domainUsage);
return(new TestCaseData(package, expected).SetName($"{{m}}({testName})"));
}
private TestPackage MakeTestPackage(string testName)
{
TestPackage package = TestProject.ActiveConfig.MakeTestPackage();
package.TestName = testName;
ISettings userSettings = Services.UserSettings;
package.Settings["MergeAssemblies"] = userSettings.GetSetting("Options.TestLoader.MergeAssemblies", false);
package.Settings["AutoNamespaceSuites"] = userSettings.GetSetting("Options.TestLoader.AutoNamespaceSuites", true);
package.Settings["ShadowCopyFiles"] = userSettings.GetSetting("Options.TestLoader.ShadowCopyFiles", true);
ProcessModel processModel = (ProcessModel)userSettings.GetSetting("Options.TestLoader.ProcessModel", ProcessModel.Default);
DomainUsage domainUsage = (DomainUsage)userSettings.GetSetting("Options.TestLoader.DomainUsage", DomainUsage.Default);
if (processModel != ProcessModel.Default && // Ignore default setting
!package.Settings.Contains("ProcessModel")) // Ignore global setting if package has a setting
{
package.Settings["ProcessModel"] = processModel;
}
// NOTE: This code ignores DomainUsage.None because TestLoader
// is only called from the GUI and the GUI can't support that setting.
// TODO: Move this logic to the GUI if TestLoader is used more widely
if (domainUsage != DomainUsage.Default && // Ignore default setting
domainUsage != DomainUsage.None && // Ignore DomainUsage.None in Gui
(processModel != ProcessModel.Multiple ||
domainUsage != DomainUsage.Multiple) && // Both process and domain may not be multiple
!package.Settings.Contains("DomainUsage")) // Ignore global setting if package has a setting
{
package.Settings["DomainUsage"] = domainUsage;
}
if (!package.Settings.Contains("WorkDirectory"))
{
package.Settings["WorkDirectory"] = Environment.CurrentDirectory;
}
//if (NUnitConfiguration.ApartmentState != System.Threading.ApartmentState.Unknown)
// package.Settings["ApartmentState"] = NUnitConfiguration.ApartmentState;
return(package);
}
/// <summary>
/// Returns a test runner based on the settings in a TestPackage.
/// Any setting that is "consumed" by the factory is removed, so
/// that downstream runners using the factory will not repeatedly
/// create the same type of runner.
/// </summary>
/// <param name="package">The TestPackage to be loaded and run</param>
/// <returns>A TestRunner</returns>
public virtual TestRunner MakeTestRunner(TestPackage package)
{
DomainUsage domainUsage =
(DomainUsage)package.GetSetting("DomainUsage", DomainUsage.Default);
switch (domainUsage)
{
case DomainUsage.Multiple:
package.Settings.Remove("DomainUsage");
return(new MultipleTestDomainRunner());
case DomainUsage.None:
return(new RemoteTestRunner());
case DomainUsage.Single:
default:
return(new TestDomain());
}
}
/// <summary>
/// Returns a test runner based on the settings in a TestPackage.
/// Any setting that is "consumed" by the factory is removed, so
/// that downstream runners using the factory will not repeatedly
/// create the same type of runner.
/// </summary>
/// <param name="package">The TestPackage to be loaded and run</param>
/// <returns>An ITestEngineRunner</returns>
public virtual ITestEngineRunner MakeTestRunner(TestPackage package)
{
DomainUsage domainUsage = (DomainUsage)System.Enum.Parse(
typeof(DomainUsage),
package.GetSetting(PackageSettings.DomainUsage, "Default"));
switch (domainUsage)
{
case DomainUsage.Multiple:
package.Settings.Remove("DomainUsage");
return(new MultipleTestDomainRunner(ServiceContext, package));
case DomainUsage.None:
return(new LocalTestRunner(ServiceContext, package));
case DomainUsage.Single:
default:
return(new TestDomainRunner(ServiceContext, package));
}
}
public static RunnerResult ResultFor(ProcessModel processModel, DomainUsage domainUsage)
{
switch (processModel)
{
case ProcessModel.Default:
return(GetProcessModelDefaultResult(domainUsage));
case ProcessModel.InProcess:
return(GetProcessModelInProcessResult(domainUsage));
case ProcessModel.Separate:
return(GetProcessModelSeparateResult(domainUsage));
case ProcessModel.Multiple:
return(GetProcessModelMultipleResult(domainUsage));
default:
throw new ArgumentOutOfRangeException(nameof(processModel), processModel, ExceptionMessage);
}
}
public static RunnerResult ResultFor(ProcessModel processModel, DomainUsage domainUsage)
{
switch (processModel)
{
case ProcessModel.Default:
return(GetProcessModelDefaultResult(domainUsage));
case ProcessModel.InProcess:
return(GetProcessModelInProcessResult(domainUsage));
case ProcessModel.Separate:
return(GetProcessModelSeparateResult(domainUsage));
case ProcessModel.Multiple:
return(GetProcessModelMultipleResult(domainUsage));
default:
ThrowOutOfRange(processModel);
break;
}
throw new ArgumentOutOfRangeException(
$"No expected Test result provided for this {nameof(ProcessModel)}/{nameof(DomainUsage)} combination.");
}
public int Execute(ConsoleOptions options)
{
this.workDir = options.work;
if (workDir == null || workDir == string.Empty)
{
workDir = Environment.CurrentDirectory;
}
else
{
workDir = Path.GetFullPath(workDir);
if (!Directory.Exists(workDir))
{
Directory.CreateDirectory(workDir);
}
}
TextWriter outWriter = Console.Out;
bool redirectOutput = options.output != null && options.output != string.Empty;
if (redirectOutput)
{
StreamWriter outStreamWriter = new StreamWriter(Path.Combine(workDir, options.output));
outStreamWriter.AutoFlush = true;
outWriter = outStreamWriter;
}
TextWriter errorWriter = Console.Error;
bool redirectError = options.err != null && options.err != string.Empty;
if (redirectError)
{
StreamWriter errorStreamWriter = new StreamWriter(Path.Combine(workDir, options.err));
errorStreamWriter.AutoFlush = true;
errorWriter = errorStreamWriter;
}
TestPackage package = MakeTestPackage(options);
ProcessModel processModel = package.Settings.Contains("ProcessModel")
? (ProcessModel)package.Settings["ProcessModel"]
: ProcessModel.Default;
DomainUsage domainUsage = package.Settings.Contains("DomainUsage")
? (DomainUsage)package.Settings["DomainUsage"]
: DomainUsage.Default;
RuntimeFramework framework = package.Settings.Contains("RuntimeFramework")
? (RuntimeFramework)package.Settings["RuntimeFramework"]
: RuntimeFramework.CurrentFramework;
#if CLR_2_0 || CLR_4_0
Console.WriteLine("ProcessModel: {0} DomainUsage: {1}", processModel, domainUsage);
Console.WriteLine("Execution Runtime: {0}", framework);
#else
Console.WriteLine("DomainUsage: {0}", domainUsage);
if (processModel != ProcessModel.Default && processModel != ProcessModel.Single)
{
Console.WriteLine("Warning: Ignoring project setting 'processModel={0}'", processModel);
}
if (!RuntimeFramework.CurrentFramework.Supports(framework))
{
Console.WriteLine("Warning: Ignoring project setting 'runtimeFramework={0}'", framework);
}
#endif
using (TestRunner testRunner = new DefaultTestRunnerFactory().MakeTestRunner(package))
{
testRunner.Load(package);
if (testRunner.Test == null)
{
testRunner.Unload();
Console.Error.WriteLine("Unable to locate fixture {0}", options.fixture);
return(FIXTURE_NOT_FOUND);
}
EventCollector collector = new EventCollector(options, outWriter, errorWriter);
TestFilter testFilter = TestFilter.Empty;
SimpleNameFilter nameFilter = new SimpleNameFilter();
if (options.run != null && options.run != string.Empty)
{
Console.WriteLine("Selected test(s): " + options.run);
foreach (string name in TestNameParser.Parse(options.run))
{
nameFilter.Add(name);
}
testFilter = nameFilter;
}
if (options.runlist != null && options.runlist != string.Empty)
{
Console.WriteLine("Run list: " + options.runlist);
using (StreamReader rdr = new StreamReader(options.runlist))
{
// NOTE: We can't use rdr.EndOfStream because it's
// not present in .NET 1.x.
string line = rdr.ReadLine();
while (line != null)
{
if (line[0] != '#')
{
nameFilter.Add(line);
}
line = rdr.ReadLine();
}
}
testFilter = nameFilter;
}
if (options.include != null && options.include != string.Empty)
{
TestFilter includeFilter = new CategoryExpression(options.include).Filter;
Console.WriteLine("Included categories: " + includeFilter.ToString());
if (testFilter.IsEmpty)
{
testFilter = includeFilter;
}
else
{
testFilter = new AndFilter(testFilter, includeFilter);
}
}
if (options.exclude != null && options.exclude != string.Empty)
{
TestFilter excludeFilter = new NotFilter(new CategoryExpression(options.exclude).Filter);
Console.WriteLine("Excluded categories: " + excludeFilter.ToString());
if (testFilter.IsEmpty)
{
testFilter = excludeFilter;
}
else if (testFilter is AndFilter)
{
((AndFilter)testFilter).Add(excludeFilter);
}
else
{
testFilter = new AndFilter(testFilter, excludeFilter);
}
}
if (testFilter is NotFilter)
{
((NotFilter)testFilter).TopLevel = true;
}
TestResult result = null;
string savedDirectory = Environment.CurrentDirectory;
TextWriter savedOut = Console.Out;
TextWriter savedError = Console.Error;
try
{
result = testRunner.Run(collector, testFilter, false, LoggingThreshold.Off);
}
finally
{
outWriter.Flush();
errorWriter.Flush();
if (redirectOutput)
{
outWriter.Close();
}
if (redirectError)
{
errorWriter.Close();
}
Environment.CurrentDirectory = savedDirectory;
Console.SetOut(savedOut);
Console.SetError(savedError);
}
Console.WriteLine();
int returnCode = UNEXPECTED_ERROR;
if (result != null)
{
string xmlOutput = CreateXmlOutput(result);
ResultSummarizer summary = new ResultSummarizer(result);
if (options.xmlConsole)
{
Console.WriteLine(xmlOutput);
}
else
{
WriteSummaryReport(summary);
if (summary.ErrorsAndFailures > 0 || result.IsError || result.IsFailure)
{
if (options.stoponerror)
{
Console.WriteLine("Test run was stopped after first error, as requested.");
Console.WriteLine();
}
WriteErrorsAndFailuresReport(result);
}
if (summary.TestsNotRun > 0)
{
WriteNotRunReport(result);
}
if (!options.noresult)
{
// Write xml output here
string xmlResultFile = options.result == null || options.result == string.Empty
? "TestResult.xml" : options.result;
using (StreamWriter writer = new StreamWriter(Path.Combine(workDir, xmlResultFile)))
{
writer.Write(xmlOutput);
}
}
}
returnCode = summary.ErrorsAndFailures;
}
if (collector.HasExceptions)
{
collector.WriteExceptions();
returnCode = UNEXPECTED_ERROR;
}
return(returnCode);
}
}
// TODO: See if this can be unified with the Gui's MakeTestPackage
private TestPackage MakeTestPackage(ConsoleOptions options)
{
TestPackage package;
DomainUsage domainUsage = DomainUsage.Default;
ProcessModel processModel = ProcessModel.Default;
RuntimeFramework framework = null;
string[] parameters = new string[options.ParameterCount];
for (int i = 0; i < options.ParameterCount; i++)
{
parameters[i] = Path.GetFullPath((string)options.Parameters[i]);
}
if (options.IsTestProject)
{
NUnitProject project =
Services.ProjectService.LoadProject(parameters[0]);
string configName = options.config;
if (configName != null)
{
project.SetActiveConfig(configName);
}
package = project.ActiveConfig.MakeTestPackage();
processModel = project.ProcessModel;
domainUsage = project.DomainUsage;
framework = project.ActiveConfig.RuntimeFramework;
}
else if (parameters.Length == 1)
{
package = new TestPackage(parameters[0]);
domainUsage = DomainUsage.Single;
}
else
{
// TODO: Figure out a better way to handle "anonymous" packages
package = new TestPackage(null, parameters);
package.AutoBinPath = true;
domainUsage = DomainUsage.Multiple;
}
#if CLR_2_0 || CLR_4_0
if (options.framework != null)
{
framework = RuntimeFramework.Parse(options.framework);
}
if (options.process != ProcessModel.Default)
{
processModel = options.process;
}
#endif
if (options.domain != DomainUsage.Default)
{
domainUsage = options.domain;
}
package.TestName = options.fixture;
package.Settings["ProcessModel"] = processModel;
package.Settings["DomainUsage"] = domainUsage;
if (framework != null)
{
package.Settings["RuntimeFramework"] = framework;
}
if (domainUsage == DomainUsage.None)
{
// Make sure that addins are available
CoreExtensions.Host.AddinRegistry = Services.AddinRegistry;
}
package.Settings["ShadowCopyFiles"] = !options.noshadow;
package.Settings["UseThreadedRunner"] = !options.nothread;
package.Settings["DefaultTimeout"] = options.timeout;
package.Settings["WorkDirectory"] = this.workDir;
package.Settings["StopOnError"] = options.stoponerror;
if (options.apartment != System.Threading.ApartmentState.Unknown)
{
package.Settings["ApartmentState"] = options.apartment;
}
return(package);
}
private int CheckDomain([NotNull] DomainUsage domainUsage)
{
Assert.ArgumentNotNull(domainUsage, nameof(domainUsage));
string domainName = domainUsage.DomainName;
int errorCount = 0;
string caseMessage;
if (!SchemaTestUtils.HasExpectedCase(domainName, _expectedCase, "name",
out caseMessage))
{
errorCount += ReportSchemaPropertyError(
GetIssueCode(_expectedCase), domainName,
"Domain name '{0}' has unexpected case: {1}",
domainName, caseMessage);
}
if (StringUtils.IsNotEmpty(_expectedPrefix))
{
if (!domainName.StartsWith(_expectedPrefix))
{
errorCount +=
ReportSchemaPropertyError(
Codes[Code.DoesStartWithPrefix], domainName,
"Domain name '{0}' does not start with prefix '{1}'",
domainName, _expectedPrefix);
}
}
if (_mustContainFieldName)
{
foreach (IField field in domainUsage.ReferencingFields)
{
string fieldName = field.Name;
if (domainName.Contains(fieldName))
{
continue;
}
errorCount +=
ReportSchemaPropertyError(
Codes[Code.DoesNotContainFieldName], domainName,
"Domain name '{0}' does not contain the field name '{1}'",
domainName, fieldName);
}
}
if (_maximumLength > 0)
{
string message;
TextLengthIssue?lengthIssue = SchemaTestUtils.HasValidLength(
domainName, _maximumLength, "name", out message);
if (lengthIssue != null)
{
errorCount += ReportSchemaPropertyError(
GetIssueCode(lengthIssue.Value), domainName,
"Domain '{0}': '{1}'",
domainName, message);
}
}
return(errorCount);
}
// TODO: See if this can be unified with the Gui's MakeTestPackage
private static TestPackage MakeTestPackage(ConsoleOptions options)
{
TestPackage package;
DomainUsage domainUsage = DomainUsage.Default;
ProcessModel processModel = ProcessModel.Default;
RuntimeFramework framework = null;
if (options.IsTestProject)
{
NUnitProject project =
Services.ProjectService.LoadProject((string)options.Parameters[0]);
string configName = options.config;
if (configName != null)
{
project.SetActiveConfig(configName);
}
package = project.ActiveConfig.MakeTestPackage();
processModel = project.ProcessModel;
domainUsage = project.DomainUsage;
framework = project.ActiveConfig.RuntimeFramework;
}
else if (options.Parameters.Count == 1)
{
package = new TestPackage((string)options.Parameters[0]);
domainUsage = DomainUsage.Single;
}
else
{
// TODO: Figure out a better way to handle "anonymous" packages
package = new TestPackage(null, options.Parameters);
package.AutoBinPath = true;
domainUsage = DomainUsage.Multiple;
}
if (options.process != ProcessModel.Default)
{
processModel = options.process;
}
if (options.domain != DomainUsage.Default)
{
domainUsage = options.domain;
}
if (options.framework != null)
{
framework = RuntimeFramework.Parse(options.framework);
}
package.TestName = options.fixture;
package.Settings["ProcessModel"] = processModel;
package.Settings["DomainUsage"] = domainUsage;
if (framework != null)
{
package.Settings["RuntimeFramework"] = framework;
}
if (domainUsage == DomainUsage.None)
{
// Make sure that addins are available
CoreExtensions.Host.AddinRegistry = Services.AddinRegistry;
}
package.Settings["ShadowCopyFiles"] = !options.noshadow;
package.Settings["UseThreadedRunner"] = !options.nothread;
package.Settings["DefaultTimeout"] = options.timeout;
return(package);
}
public static RunnerResult ResultFor(ProcessModel processModel, DomainUsage domainUsage)
{
switch (processModel)
{
case ProcessModel.Default:
switch (domainUsage)
{
case DomainUsage.Default:
case DomainUsage.None:
case DomainUsage.Single:
case DomainUsage.Multiple:
return(RunnerResult.ProcessRunner);
default:
throw new ArgumentOutOfRangeException(nameof(domainUsage), domainUsage, ExceptionMessage);
}
case ProcessModel.InProcess:
switch (domainUsage)
{
case DomainUsage.Default:
return(RunnerResult.TestDomainRunner);
case DomainUsage.None:
return(RunnerResult.LocalTestRunner);
case DomainUsage.Single:
return(RunnerResult.TestDomainRunner);
case DomainUsage.Multiple:
return(RunnerResult.TestDomainRunner);
default:
throw new ArgumentOutOfRangeException(nameof(domainUsage), domainUsage, ExceptionMessage);
}
case ProcessModel.Separate:
switch (domainUsage)
{
case DomainUsage.Default:
case DomainUsage.None:
case DomainUsage.Single:
case DomainUsage.Multiple:
return(RunnerResult.ProcessRunner);
default:
throw new ArgumentOutOfRangeException(nameof(domainUsage), domainUsage, ExceptionMessage);
}
case ProcessModel.Multiple:
switch (domainUsage)
{
case DomainUsage.Default:
case DomainUsage.None:
case DomainUsage.Single:
case DomainUsage.Multiple:
return(new RunnerResult {
TestRunner = typeof(MultipleTestProcessRunner)
});
default:
throw new ArgumentOutOfRangeException(nameof(domainUsage), domainUsage, ExceptionMessage);
}
default:
throw new ArgumentOutOfRangeException(nameof(processModel), processModel, ExceptionMessage);
}
}
public TestResult Execute(ConsoleOptions options)
{
_workDir = options.work;
if (string.IsNullOrEmpty(_workDir))
{
_workDir = Environment.CurrentDirectory;
}
else
{
_workDir = Path.GetFullPath(_workDir);
if (!Directory.Exists(_workDir))
{
Directory.CreateDirectory(_workDir);
}
}
TextWriter outWriter = Console.Out;
bool redirectOutput = !string.IsNullOrEmpty(options.output);
if (redirectOutput)
{
StreamWriter outStreamWriter = new StreamWriter(Path.Combine(_workDir, options.output));
outStreamWriter.AutoFlush = true;
outWriter = outStreamWriter;
}
TextWriter errorWriter = Console.Error;
bool redirectError = !string.IsNullOrEmpty(options.err);
if (redirectError)
{
StreamWriter errorStreamWriter = new StreamWriter(Path.Combine(_workDir, options.err));
errorStreamWriter.AutoFlush = true;
errorWriter = errorStreamWriter;
}
TestPackage package = MakeTestPackage(options);
ProcessModel processModel = package.Settings.Contains("ProcessModel")
? (ProcessModel)package.Settings["ProcessModel"]
: ProcessModel.Default;
DomainUsage domainUsage = package.Settings.Contains("DomainUsage")
? (DomainUsage)package.Settings["DomainUsage"]
: DomainUsage.Default;
RuntimeFramework framework = package.Settings.Contains("RuntimeFramework")
? (RuntimeFramework)package.Settings["RuntimeFramework"]
: RuntimeFramework.CurrentFramework;
#if CLR_2_0 || CLR_4_0
Console.WriteLine("ProcessModel: {0} DomainUsage: {1}", processModel, domainUsage);
Console.WriteLine("Execution Runtime: {0}", framework);
#else
Console.WriteLine("DomainUsage: {0}", domainUsage);
if (processModel != ProcessModel.Default && processModel != ProcessModel.Single)
{
Console.WriteLine("Warning: Ignoring project setting 'processModel={0}'", processModel);
}
if (!RuntimeFramework.CurrentFramework.Supports(framework))
{
Console.WriteLine("Warning: Ignoring project setting 'runtimeFramework={0}'", framework);
}
#endif
using (NUnit.Core.TestRunner testRunner = new DefaultTestRunnerFactory().MakeTestRunner(package))
{
testRunner.Load(package);
if (testRunner.Test == null)
{
testRunner.Unload();
Console.Error.WriteLine("Unable to locate fixture {0}", options.fixture);
return(null);
}
EventCollector collector = new EventCollector(options, outWriter, errorWriter);
TestFilter testFilter;
if (!CreateTestFilter(options, out testFilter))
{
return(null);
}
string savedDirectory = Environment.CurrentDirectory;
TextWriter savedOut = Console.Out;
TextWriter savedError = Console.Error;
try
{
return(testRunner.Run(collector, testFilter, false, LoggingThreshold.Off));
}
finally
{
outWriter.Flush();
errorWriter.Flush();
if (redirectOutput)
{
outWriter.Close();
}
if (redirectError)
{
errorWriter.Close();
}
Environment.CurrentDirectory = savedDirectory;
Console.SetOut(savedOut);
Console.SetError(savedError);
}
}
}
public int Execute(ExtendedConsoleOptions options)
{
this.workDir = options.work;
if (workDir == null || workDir == string.Empty)
{
workDir = Environment.CurrentDirectory;
}
else
{
workDir = Path.GetFullPath(workDir);
if (!Directory.Exists(workDir))
{
Directory.CreateDirectory(workDir);
}
}
TextWriter outWriter = Console.Out;
bool redirectOutput = options.output != null && options.output != string.Empty;
if (redirectOutput)
{
StreamWriter outStreamWriter = new StreamWriter(Path.Combine(workDir, options.output));
outStreamWriter.AutoFlush = true;
outWriter = outStreamWriter;
}
TextWriter errorWriter = Console.Error;
bool redirectError = options.err != null && options.err != string.Empty;
if (redirectError)
{
StreamWriter errorStreamWriter = new StreamWriter(Path.Combine(workDir, options.err));
errorStreamWriter.AutoFlush = true;
errorWriter = errorStreamWriter;
}
TextWriter testResultWriter = null;
if (options.IsResults)
{
testResultWriter = new StreamWriter(options.results, false, Encoding.UTF8);
((StreamWriter)testResultWriter).AutoFlush = true;
}
TestPackage package = MakeTestPackage(options);
ProcessModel processModel = package.Settings.Contains("ProcessModel")
? (ProcessModel)package.Settings["ProcessModel"]
: ProcessModel.Default;
DomainUsage domainUsage = package.Settings.Contains("DomainUsage")
? (DomainUsage)package.Settings["DomainUsage"]
: DomainUsage.Default;
RuntimeFramework framework = package.Settings.Contains("RuntimeFramework")
? (RuntimeFramework)package.Settings["RuntimeFramework"]
: RuntimeFramework.CurrentFramework;
#if CLR_2_0 || CLR_4_0
Console.WriteLine("ProcessModel: {0} DomainUsage: {1}", processModel, domainUsage);
Console.WriteLine("Execution Runtime: {0}", framework);
#else
Console.WriteLine("DomainUsage: {0}", domainUsage);
if (processModel != ProcessModel.Default && processModel != ProcessModel.Single)
{
Console.WriteLine("Warning: Ignoring project setting 'processModel={0}'", processModel);
}
if (!RuntimeFramework.CurrentFramework.Supports(framework))
{
Console.WriteLine("Warning: Ignoring project setting 'runtimeFramework={0}'", framework);
}
#endif
using (TestRunner testRunner = new DefaultTestRunnerFactory().MakeTestRunner(package))
{
testRunner.Load(package);
if (testRunner.Test == null)
{
testRunner.Unload();
Console.Error.WriteLine("Unable to locate fixture {0}", options.fixture);
return(FIXTURE_NOT_FOUND);
}
ExtendedEventCollector collector = new ExtendedEventCollector(options, outWriter, errorWriter, testResultWriter);
TestFilter testFilter;
if (!CreateTestFilter(options, out testFilter))
{
return(INVALID_ARG);
}
TestResult result = null;
string savedDirectory = Environment.CurrentDirectory;
TextWriter savedOut = Console.Out;
TextWriter savedError = Console.Error;
try
{
result = testRunner.Run(collector, testFilter, true, LoggingThreshold.Off);
}
finally
{
outWriter.Flush();
errorWriter.Flush();
if (redirectOutput)
{
outWriter.Close();
}
if (redirectError)
{
errorWriter.Close();
}
if (options.IsResults)
{
testResultWriter.Close();
}
Environment.CurrentDirectory = savedDirectory;
Console.SetOut(savedOut);
Console.SetError(savedError);
}
Console.WriteLine();
int returnCode = UNEXPECTED_ERROR;
if (result != null)
{
string xmlOutput = CreateXmlOutput(result);
ResultSummarizer summary = new ResultSummarizer(result);
if (options.xmlConsole)
{
Console.WriteLine(xmlOutput);
}
else
{
WriteSummaryReport(summary);
bool hasErrors = summary.Errors > 0 || summary.Failures > 0 || result.IsError || result.IsFailure;
if (options.stoponerror && (hasErrors || summary.NotRunnable > 0))
{
Console.WriteLine("Test run was stopped after first error, as requested.");
Console.WriteLine();
}
if (hasErrors)
{
WriteErrorsAndFailuresReport(result);
}
if (summary.TestsNotRun > 0)
{
WriteNotRunReport(result);
}
if (!options.noresult)
{
// Write xml output here
string xmlResultFile = options.result == null || options.result == string.Empty
? "TestResult.xml" : options.result;
using (StreamWriter writer = new StreamWriter(Path.Combine(workDir, xmlResultFile)))
{
writer.Write(xmlOutput);
}
}
}
returnCode = summary.Errors + summary.Failures + summary.NotRunnable;
}
if (collector.HasExceptions)
{
collector.WriteExceptions();
returnCode = UNEXPECTED_ERROR;
}
return(returnCode);
}
}
private int CheckDomain([NotNull] DomainUsage domainUsage)
{
IDomain domain = domainUsage.Domain;
var codedValueDomain = domain as ICodedValueDomain;
if (codedValueDomain == null)
{
return(NoError);
}
List <CodedValue> codedValues = DomainUtils.GetCodedValueList(codedValueDomain);
int nonEqualNameValueCount = 0;
int errorCount = 0;
foreach (CodedValue codedValue in codedValues)
{
if (!Equals(codedValue.Value.ToString().Trim(), codedValue.Name.Trim()))
{
nonEqualNameValueCount++;
}
if (!StringUtils.IsNotEmpty(codedValue.Name))
{
if (!_allowEmptyName)
{
errorCount +=
ReportSchemaPropertyError(
Codes[Code.NoName], domain.Name,
new[] { codedValue.Value },
"Value [{0}] in coded value domain '{1}' does not have an associated name",
codedValue.Value, domain.Name);
}
continue;
}
string message;
TextLengthIssue?lengthIssue = SchemaTestUtils.HasValidLength(
codedValue.Name, _maximumNameLength, "name", out message);
if (lengthIssue != null)
{
errorCount +=
ReportSchemaPropertyError(GetIssueCode(lengthIssue.Value), domain.Name,
new[] { codedValue.Value },
"Value [{0}] in coded value domain '{1}': {2} ('{3}')",
codedValue.Value, domain.Name, message,
codedValue.Name);
}
}
// report non-unique names
errorCount += CheckUniqueNames(domainUsage, codedValues);
// report error if the number of coded values is less than the minimum count
if (codedValues.Count < _minimumValueCount)
{
string format = codedValues.Count == 1
? "Domain '{0}' has {1} coded value. Minimum: {2}"
: "Domain '{0}' has {1} coded values. Minimum: {2}";
string description = string.Format(format,
domainUsage.DomainName, codedValues.Count,
_minimumValueCount);
errorCount += ReportSchemaPropertyError(Codes[Code.TooFewCodedValues],
domainUsage.DomainName, description);
}
// report error if the number of coded values with non-equal name/value pair is
// less than the minimum count for non-equal values (and the total coded value
// count exceeds that minimum value; otherwise an error would always be reported)
if (nonEqualNameValueCount < _minimumNonEqualNameValueCount &&
codedValues.Count >= _minimumNonEqualNameValueCount)
{
string description = string.Format(
"Domain '{0}' has {1} coded values with a name that is different from the value. Minimum: {2}",
domainUsage.DomainName, nonEqualNameValueCount,
_minimumNonEqualNameValueCount);
errorCount +=
ReportSchemaPropertyError(Codes[Code.NotEnoughNamesDifferentFromValue],
domainUsage.DomainName, description);
}
return(errorCount);
}
private static TestCaseData SingleAssemblyListCtorTest(ProcessModel processModel, DomainUsage domainUsage)
{
var testName = "Single assembly (list ctor) - " +
$"{nameof(EnginePackageSettings.ProcessModel)}:{processModel} " +
$"{nameof(EnginePackageSettings.DomainUsage)}:{domainUsage}";
var package = TestPackageFactory.OneAssemblyListCtor();
package.AddSetting(EnginePackageSettings.ProcessModel, processModel.ToString());
package.AddSetting(EnginePackageSettings.DomainUsage, domainUsage.ToString());
var expected = Net20SingleAssemblyListCtorExpectedRunnerResults.ResultFor(processModel, domainUsage);
return(new TestCaseData(package, expected).SetName($"{{m}}({testName})"));
}
