public void Normalize_should_set_default_value()
{
var tro = new TestRunnerOptions();
var clone = tro.Normalize();
Assert.Equal(500, clone.SlowTestThreshold.Value.TotalMilliseconds);
}
private void DoWithTestRunner(Action <ITestRunner> action, IProgressMonitor progressMonitor,
double initializationAndDisposalWorkUnits, IEnumerable <string> testRunnerExtensions)
{
if (testRunnerFactory == null)
{
throw new InvalidOperationException("A test runner factory must be set first.");
}
var testRunner = testRunnerFactory.CreateTestRunner();
try
{
var testRunnerOptions = new TestRunnerOptions();
var logger = RuntimeAccessor.Logger;
RegisterTestRunnerExtensions(testRunnerExtensions, testRunner);
var subProgressMonitor = progressMonitor.CreateSubProgressMonitor(initializationAndDisposalWorkUnits / 2);
testRunner.Initialize(testRunnerOptions, logger, subProgressMonitor);
WireUpTestRunnerEvents(testRunner);
action(testRunner);
}
finally
{
var subProgressMonitor = progressMonitor.CreateSubProgressMonitor(initializationAndDisposalWorkUnits / 2);
testRunner.Dispose(subProgressMonitor);
}
}
/// <summary>
/// Creates a sample runner.
/// </summary>
public SampleRunner()
{
testPackage = new TestPackage();
filters = new List <Filter <ITestDescriptor> >();
TestRunnerFactoryName = StandardTestRunnerFactoryNames.Local;
testRunnerOptions = new TestRunnerOptions();
}
/// <summary>
/// Run the database runner
/// </summary>
/// <returns></returns>
public async Task <TestResult> Run(Test databaseTest,
TestRunnerOptions databaseTestOptions)
{
using (var dbConnection = new MySqlConnection(databaseTestOptions.ConnectionString))
{
dbConnection.Open();
var sw = new Stopwatch();
sw.Start();
var databaseResult = await dbConnection.ExecuteScalarAsync <int>(databaseTest.Script);
sw.Stop();
return(new TestResult(
Convert.ToBoolean(databaseResult),
databaseTest.Variable,
databaseTest.Script,
databaseTest.DatabaseTestType,
DatabaseKind.MySql,
0,
0,
sw.ElapsedMilliseconds,
null));
}
}
private static Browser Init()
{
log.Info("Initializing browser...");
string browserType = TestRunnerOptions.getBrowserType();
IWebDriver driver = null;
switch (browserType)
{
case "ff":
driver = new FirefoxDriver();
break;
case "gc":
driver = new ChromeDriver();
break;
case "ie":
driver = new InternetExplorerDriver();
break;
}
driver.Manage().Timeouts().ImplicitlyWait(COMMAND_DEFAULT_TIMEOUT_SECONDS);
driver.Manage().Timeouts().SetPageLoadTimeout(PAGE_LOAD_DEFAULT_TIMEOUT_SECONDS);
driver.Manage().Window.Maximize();
return(new Browser(driver));
}
/// <summary>
/// Initializes the event arguments.
/// </summary>
/// <param name="options">The test runner options.</param>
/// <exception cref="ArgumentNullException">Thrown if <paramref name="options"/> is null.</exception>
public InitializeStartedEventArgs(TestRunnerOptions options)
{
if (options == null)
throw new ArgumentNullException("options");
this.options = options;
}
private void RunAllTests(IRunContext runContext)
{
ITestRunnerManager runnerManager = RuntimeAccessor.ServiceLocator.Resolve <ITestRunnerManager>();
var runner = runnerManager.CreateTestRunner(StandardTestRunnerFactoryNames.IsolatedAppDomain);
runner.RegisterExtension(new RunContextExtension(runContext));
ILogger logger = new RunContextLogger(runContext);
TestRunnerOptions testRunnerOptions = new TestRunnerOptions();
try
{
RunWithProgressMonitor(delegate(IProgressMonitor progressMonitor)
{
runner.Initialize(testRunnerOptions, logger, progressMonitor);
});
if (isCanceled)
{
return;
}
TestPackage testPackage = new TestPackage();
testPackage.AddExcludedTestFrameworkId("MSTestAdapter.TestFramework");
foreach (ITestElement testElement in runContext.RunConfig.TestElements)
{
GallioTestElement gallioTestElement = testElement as GallioTestElement;
if (gallioTestElement != null)
{
testPackage.AddFile(new FileInfo(gallioTestElement.AssemblyPath));
}
}
TestExplorationOptions testExplorationOptions = new TestExplorationOptions();
TestExecutionOptions testExecutionOptions = new TestExecutionOptions();
List <Filter <string> > idFilters = new List <Filter <string> >();
foreach (ITestElement includedTestElement in runContext.RunConfig.TestElements)
{
GallioTestElement gallioTestElement = includedTestElement as GallioTestElement;
if (gallioTestElement != null)
{
idFilters.Add(new EqualityFilter <string>(gallioTestElement.GallioTestId));
}
}
testExecutionOptions.FilterSet = new FilterSet <ITestDescriptor>(new IdFilter <ITestDescriptor>(new OrFilter <string>(idFilters)));
RunWithProgressMonitor(delegate(IProgressMonitor progressMonitor)
{
runner.Run(testPackage, testExplorationOptions, testExecutionOptions, progressMonitor);
});
}
finally
{
runner.Dispose(NullProgressMonitor.CreateInstance());
}
}
public void AddSQLServerConnectionTest_Successful()
{
var options = new TestRunnerOptions();
options.AddSQLServerConnection("server=localhost;user=user;password=mypassword;Initial Catalog=myDatabase");
Assert.NotNull(options);
}
/// <summary>
/// Initializes the event arguments.
/// </summary>
/// <param name="options">The test runner options.</param>
/// <exception cref="ArgumentNullException">Thrown if <paramref name="options"/> is null.</exception>
public InitializeStartedEventArgs(TestRunnerOptions options)
{
if (options == null)
{
throw new ArgumentNullException("options");
}
this.options = options;
}
private static DefaultTestRunner.TestPlan CreatePlan(params string[] names)
{
var opts = new TestRunnerOptions();
var testRun = new TestRun();
testRun.Children.AddAll(TestUnits(names));
var runner = new DefaultTestRunner(opts);
return((DefaultTestRunner.TestPlan)runner.CreatePlan(testRun));
}
public void Constructor_does_not_clone_ReadOnly_property()
{
var tro = new TestRunnerOptions();
tro.MakeReadOnly();
var clone = new TestRunnerOptions(tro);
Assert.False(clone.IsReadOnly);
}
public ConsoleOutputParts(TestRunnerOptions opts)
{
onTestCaseFinished = ConsoleOutputPart.Compose(
new ConsoleTestCaseStatus()
);
onTestTheoryFinished = ConsoleOutputPart.Compose(
new ConsoleTestTheoryStatus()
);
onTestRunFinished = ConsoleOutputPart.Compose(
new ConsoleTestRunExplicitlyPassed {
Show = opts.ShowPassExplicitly,
},
new ConsoleTestRunProblems {
Show = !opts.SuppressSummary,
},
new ConsoleTestRunResults()
);
forExceptionInfo = ConsoleOutputPart.Compose(
new ConsoleExceptionInfo {
ShowNoisyStackFrames = opts.AssertionMessageFormatMode.HasFlag(
AssertionMessageFormatModes.FullStackTraces
),
}
);
forUserData = ConsoleOutputPart.Compose(
new ConsoleUserData {
AssertionMessageFormat = opts.AssertionMessageFormatMode
}
);
forMessage = ConsoleOutputPart.Compose(
new ConsoleMessage()
);
forPatch = ConsoleOutputPart.Compose(
new ConsolePatch {
ContextLines = opts.ContextLines,
ShowWhitespace = opts.AssertionMessageFormatMode.HasFlag(
AssertionMessageFormatModes.PrintWhitespace
),
}
);
forStatus = ConsoleOutputPart.Compose(
new ConsoleTestUnitStatusBullet {
ShowSkipped = false,
}
);
forResultDetails = ConsoleOutputPart.Compose(
new ConsoleResultDetails()
);
forName = ConsoleOutputPart.Compose(
new ConsoleTestName()
);
}
public void Constructor_should_copy_other_property()
{
var tro = new TestRunnerOptions {
ContextLines = 20,
RandomSeed = 100,
AssertionMessageFormatMode = AssertionMessageFormatModes.UseUnifiedDiff,
};
var clone = new TestRunnerOptions(tro);
Assert.Equal(20, clone.ContextLines);
Assert.Equal(100, clone.RandomSeed);
Assert.Equal(AssertionMessageFormatModes.UseUnifiedDiff, clone.AssertionMessageFormatMode);
}
public void WillRunTestCases_should_contain_all_when_ignoring_focus()
{
var opts = new TestRunnerOptions {
IgnoreFocus = true,
};
var testRun = new TestRun();
testRun.Children.AddAll(TestUnits("A", "B"));
var runner = new DefaultTestRunner(opts);
var plan = (DefaultTestRunner.TestPlan)runner.CreatePlan(testRun);
var names = WillRun(plan);
Assert.Equal(new [] { "TestA1", "TestB1", "TestB2", "TestB3" }, names);
}
public void WillRunTestCases_should_contain_focus_match_names()
{
var opts = new TestRunnerOptions();
opts.PlanFilter.FocusPatterns.AddNew("regex:(2|3)$");
var testRun = new TestRun();
testRun.Children.AddAll(TestUnits("C"));
var runner = new DefaultTestRunner(opts);
var plan = (DefaultTestRunner.TestPlan)runner.CreatePlan(testRun);
var names = WillRun(plan);
Assert.Equal(new [] { "TestC2", "TestC3" }, names);
}
public void WillRunTestCases_should_prefer_focus_specified_by_options()
{
var opts = new TestRunnerOptions();
opts.PlanFilter.FocusPatterns.AddNew("regex:2$");
var testRun = new TestRun();
testRun.Children.AddAll(TestUnits("A", "B", "C", "D"));
var runner = new DefaultTestRunner(opts);
var plan = (DefaultTestRunner.TestPlan)runner.CreatePlan(testRun);
var names = WillRun(plan);
Assert.Equal(new [] { "TestB2", "TestC2", "TestD2" }, names);
}
public void Constructor_should_copy_TimeSpan_property()
{
var two = TimeSpan.FromSeconds(2);
var three = TimeSpan.FromSeconds(3);
var four = TimeSpan.FromSeconds(4);
var tro = new TestRunnerOptions {
PlanTimeout = two,
TestTimeout = three,
SlowTestThreshold = four,
};
var clone = new TestRunnerOptions(tro);
Assert.Equal(two, clone.PlanTimeout);
Assert.Equal(three, clone.TestTimeout);
Assert.Equal(four, clone.SlowTestThreshold);
}
public ConsoleTestRunner(ITestEngine engine, TestRunnerOptions options, ExtendedTextWriter writer, IFileSystem fileSystem)
{
_engine = engine;
_options = options;
_outWriter = writer;
_fileSystem = fileSystem;
var workDirectory = options.WorkDirectory ?? _fileSystem.Directory.GetCurrentDirectory();
if (!_fileSystem.Directory.Exists(workDirectory))
{
_fileSystem.Directory.CreateDirectory(workDirectory);
}
_engine.Services.GetService <IResultService>();
_engine.Services.GetService <IExtensionService>();
}
public ResultReporter(XmlNode resultNode, ExtendedTextWriter writer, TestRunnerOptions options)
{
ResultNode = resultNode;
Writer = writer;
Options = options;
OverallResult = resultNode.GetAttribute("result");
if (OverallResult == "Skipped")
{
OverallResult = "Warning";
}
if (OverallResult == null)
{
OverallResult = "Unknown";
}
Summary = new ResultSummary(resultNode);
}
public void Constructor_should_copy_Boolean_property()
{
var tro = new TestRunnerOptions {
FailFast = true,
IgnoreFocus = true,
RandomizeSpecs = true,
ShowPassExplicitly = true,
ShowTestNames = true,
SuppressSummary = true,
};
var clone = new TestRunnerOptions(tro);
Assert.True(clone.FailFast);
Assert.True(clone.IgnoreFocus);
Assert.True(clone.RandomizeSpecs);
Assert.True(clone.ShowPassExplicitly);
Assert.True(clone.ShowTestNames);
Assert.True(clone.SuppressSummary);
}
public ConsoleLogger(TestRunnerOptions opts)
{
if (!opts.SuppressSummary)
{
_flags = DisplayFlags.ShowSummary;
}
if (opts.ShowTestNames)
{
_flags |= DisplayFlags.ShowCaseStart;
}
if (opts.ShowPassExplicitly)
{
_flags |= DisplayFlags.ShowExplicitPasses;
}
_parts = new ConsoleOutputParts(opts);
_renderContext = new RenderContext {
Console = console,
Parts = _parts,
};
}
static string[] WillRun(Action <TestPlanFilter> planner)
{
var testRun = new TestRun();
var opts = new TestRunnerOptions();
planner(opts.PlanFilter);
testRun.Children.AddAll(TestUnits("A", "B", "C", "D"));
var runner = new DefaultTestRunner(opts);
var plan = (DefaultTestRunner.TestPlan)runner.CreatePlan(testRun);
Func <string, string> removePrefix = s => {
const string prefix = "Carbonfrost.SelfTest.Spec.ExecutionModel.TestPlanFilterTests+";
if (s.StartsWith(prefix, StringComparison.Ordinal))
{
return(s.Substring(prefix.Length + 2));
}
return(s);
};
return(plan.WillRunTestCases.Select(t => removePrefix(t.TestName.Method)).ToArray());
}
internal TestLoader(TestRunnerOptions opts,
TestContext parent)
{
_opts = opts;
_context = parent;
}
public FailFastTestPlan(DefaultTestRunner runner, TestRun run, TestRunnerOptions normalized)
: base(runner, run, normalized)
{
}
public void AddSQLServerConnectionTest_Failure()
{
var options = new TestRunnerOptions();
Assert.Throws <ArgumentNullException>(() => options.AddSQLServerConnection(null));
}
/// <summary>
/// SqServer test runner instance
/// </summary>
/// <param name="databaseTests"></param>
/// <param name="options"></param>
public TestRunnerFactory(IEnumerable <Test> databaseTests, Action <TestRunnerOptions> options)
{
_databaseTestOptions = new TestRunnerOptions();
options?.Invoke(_databaseTestOptions);
_databaseTests = databaseTests ?? throw new ArgumentNullException($"{nameof(databaseTests)} is null.");
}
private FacadeTaskResult RunTests()
{
var logger = new FacadeLoggerWrapper(facadeLogger);
var runner = TestRunnerUtils.CreateTestRunnerByName(StandardTestRunnerFactoryNames.IsolatedAppDomain);
// Set parameters.
var testPackage = new TestPackage();
foreach (var assemblyLocation in assemblyLocations)
{
testPackage.AddFile(new FileInfo(assemblyLocation));
}
testPackage.ShadowCopy = facadeTaskExecutorConfiguration.ShadowCopy;
if (facadeTaskExecutorConfiguration.AssemblyFolder != null)
{
testPackage.ApplicationBaseDirectory = new DirectoryInfo(facadeTaskExecutorConfiguration.AssemblyFolder);
testPackage.WorkingDirectory = new DirectoryInfo(facadeTaskExecutorConfiguration.AssemblyFolder);
}
var testRunnerOptions = new TestRunnerOptions();
var testExplorationOptions = new TestExplorationOptions();
var filters = GenericCollectionUtils.ConvertAllToArray <string, Filter <string> >(explicitTestIds,
testId => new EqualityFilter <string>(testId));
var filterSet = new FilterSet <ITestDescriptor>(new IdFilter <ITestDescriptor>(new OrFilter <string>(filters)));
var testExecutionOptions = new TestExecutionOptions {
FilterSet = filterSet
};
// Install the listeners.
runner.Events.TestStepStarted += TestStepStarted;
runner.Events.TestStepFinished += TestStepFinished;
runner.Events.TestStepLifecyclePhaseChanged += TestStepLifecyclePhaseChanged;
// Run the tests.
try
{
try
{
runner.Initialize(testRunnerOptions, logger, CreateProgressMonitor());
Report report = runner.Run(testPackage, testExplorationOptions, testExecutionOptions, CreateProgressMonitor());
if (sessionId != null)
{
SessionCache.SaveSerializedReport(sessionId, report);
}
return(FacadeTaskResult.Success);
}
catch (Exception ex)
{
if (sessionId != null)
{
SessionCache.ClearSerializedReport(sessionId);
}
logger.Log(LogSeverity.Error, "A fatal exception occurred during test execution.", ex);
return(FacadeTaskResult.Exception);
}
finally
{
SubmitFailureForRemainingPendingTasks();
}
}
finally
{
runner.Dispose(CreateProgressMonitor());
}
}
internal NUnitApplication(ILifetimeBroker <ITestEngine> testEngineBroker, TestRunnerOptions options)
{
_testEngineBroker = testEngineBroker;
_options = options;
}
/// <summary>
/// Run the database runner
/// </summary>
/// <returns></returns>
public async Task <TestResult> Run(Test databaseTest,
TestRunnerOptions databaseTestOptions)
{
using (var dbConnection = new SqlConnection(databaseTestOptions.ConnectionString))
{
var serverCpuTime = 0;
var serverElapsedTime = 0;
var statistics = new Dictionary <string, Dictionary <string, long> >();
dbConnection.Open();
if (databaseTest.DatabaseTestType == TestType.QueryStatistics)
{
dbConnection.InfoMessage += (sender, args) =>
{
var message = args.Message;
if (message.Contains("SQL Server Execution Times:"))
{
var cpuTime = Regex.Match(message, "(?<=CPU time = ).*?(?= ms,)");
var elapsedTime = Regex.Match(message, "(?<=elapsed time = ).*?(?= ms.)");
if (Convert.ToInt32(cpuTime.Value) > 0)
{
serverCpuTime = Convert.ToInt32(cpuTime.Value);
}
if (Convert.ToInt32(elapsedTime.Value) > 0)
{
serverElapsedTime = Convert.ToInt32(elapsedTime.Value);
}
}
if (message.StartsWith("Table"))
{
var tableName = Regex.Match(message, "(?<=Table ').*?(?='.)");
var stats = new Dictionary <string, long>();
var list = new List <string>();
list.AddRange(Regex.Replace(message, $"Table '{tableName}'. ", string.Empty).Split(','));
foreach (var item in list)
{
var statName = Regex.Match(item, "[^0-9]+");
var statValue = Regex.Match(item, @"\d+");
stats.Add(statName.Value.ToPascalCase(new[] { "-", "_", " " }),
Convert.ToInt64(statValue.Value));
}
statistics.Add(tableName.Value, stats);
}
};
}
var sw = new Stopwatch();
sw.Start();
bool?databaseResult = null;
if (databaseTest.DatabaseTestType == TestType.QueryStatistics)
{
await dbConnection.QueryAsync <dynamic>(databaseTest.Script);
}
else
{
var result = await dbConnection.ExecuteScalarAsync <int>(databaseTest.Script);
databaseResult = Convert.ToBoolean(result);
}
sw.Stop();
return(new TestResult(
databaseResult,
databaseTest.Variable,
databaseTest.Script,
databaseTest.DatabaseTestType,
DatabaseKind.SqlServer,
serverCpuTime,
serverElapsedTime,
sw.ElapsedMilliseconds,
statistics));
}
}
private TestPackage MakeTestPackage(TestRunnerOptions options)
{
var package = new TestPackage(options.InputFiles);
package.AddSetting(EnginePackageSettings.DisposeRunners, true);
if (options.ShadowCopyFiles)
{
package.AddSetting(EnginePackageSettings.ShadowCopyFiles, true);
}
if (options.SkipNonTestAssemblies)
{
package.AddSetting(EnginePackageSettings.SkipNonTestAssemblies, true);
}
if (options.DefaultTimeout >= 0)
{
package.AddSetting(FrameworkPackageSettings.DefaultTimeout, options.DefaultTimeout);
}
if (options.InternalTraceLevel.HasValue)
{
package.AddSetting(FrameworkPackageSettings.InternalTraceLevel, options.InternalTraceLevel.Value);
}
if (options.ActiveConfig.HasValue)
{
package.AddSetting(EnginePackageSettings.ActiveConfig, options.ActiveConfig.Value);
}
// Always add work directory, in case current directory is changed
var workDirectory = options.WorkDirectory ?? _fileSystem.Directory.GetCurrentDirectory();
package.AddSetting(FrameworkPackageSettings.WorkDirectory, workDirectory);
if (options.StopOnError)
{
package.AddSetting(FrameworkPackageSettings.StopOnError, true);
}
if (options.RandomSeed.HasValue)
{
package.AddSetting(FrameworkPackageSettings.RandomSeed, options.RandomSeed.Value);
}
if (System.Diagnostics.Debugger.IsAttached)
{
package.AddSetting(FrameworkPackageSettings.DebugTests, true);
package.AddSetting(FrameworkPackageSettings.NumberOfTestWorkers, 0);
}
if (options.PauseBeforeRun)
{
package.AddSetting(FrameworkPackageSettings.PauseBeforeRun, true);
}
if (options.TestParameters.Count != 0)
{
AddTestParametersSetting(package, options.TestParameters);
}
if (options.ConfigurationFile != null)
{
package.AddSetting(EnginePackageSettings.ConfigurationFile, options.ConfigurationFile.Value);
}
return(package);
}
public ConsoleTestRunner(ITestEngine engine, TestRunnerOptions options)
: this(engine, options, new ExtendedTextWriter(), new FileSystem())
{
}
