/// <summary>
/// Run the tests in the specified assembly and output the results as xml to
/// the specified file.
/// </summary>
/// <param name="fullAssemblyPath">The full assembly path.</param>
/// <param name="outputXmlPath">The output xml path.</param>
/// <returns>False if any tests failed or any errors occurred, otherwise true</returns>
public bool RunTests(string fullAssemblyPath, string outputXmlPath)
{
var assemblyToTest = Assembly.LoadFrom(fullAssemblyPath);
var results = new ParallelTestRunner().RunTestsInParallel(assemblyToTest);
SaveXmlOutput(results, outputXmlPath);
return(!results.Results.Cast <TestResult>().Any(x => x.IsFailure || x.IsError));
}
private void ComputeTestRunner(ITestFrameworkReporter reporter, bool isBeingDebugged, string solutionDirectory)
{
if (TestEnvironment.Options.ParallelTestExecution && !isBeingDebugged)
{
Runner = new ParallelTestRunner(reporter, TestEnvironment, solutionDirectory);
}
else
{
Runner = new PreparingTestRunner(0, solutionDirectory, reporter, TestEnvironment);
if (TestEnvironment.Options.ParallelTestExecution && isBeingDebugged)
{
TestEnvironment.DebugInfo(
"Parallel execution is selected in options, but tests are executed sequentially because debugger is attached.");
}
}
}
/// <summary>
/// creates a correct type of runner and runs a single test.
/// </summary>
/// <param name="testPath"></param>
/// <param name="errorReason"></param>
/// <returns></returns>
private TestRunResults RunHPToolsTest(TestInfo testinf, ref string errorReason)
{
var testPath = testinf.TestPath;
var type = Helper.GetTestType(testPath);
// if we have at least one environment for parallel runner,
// then it must be enabled
var isParallelRunnerEnabled = _parallelRunnerEnvironments.Count > 0;
if (isParallelRunnerEnabled && type == TestType.QTP)
{
type = TestType.ParallelRunner;
}
// if the current test is an api test ignore the parallel runner flag
// and just continue as usual
else if (isParallelRunnerEnabled && type == TestType.ST)
{
ConsoleWriter.WriteLine("ParallelRunner does not support API tests, treating as normal test.");
}
IFileSysTestRunner runner = null;
switch (type)
{
case TestType.ST:
runner = new ApiTestRunner(this, _timeout - _stopwatch.Elapsed);
break;
case TestType.QTP:
runner = new GuiTestRunner(this, _useUFTLicense, _timeout - _stopwatch.Elapsed, _uftRunMode, _mcConnection, _mobileInfoForAllGuiTests);
break;
case TestType.LoadRunner:
AppDomain.CurrentDomain.AssemblyResolve += Helper.HPToolsAssemblyResolver;
runner = new PerformanceTestRunner(this, _timeout, _pollingInterval, _perScenarioTimeOutMinutes, _ignoreErrorStrings, _displayController, _analysisTemplate, _summaryDataLogger, _scriptRTSSet);
break;
case TestType.ParallelRunner:
runner = new ParallelTestRunner(this, _timeout - _stopwatch.Elapsed, _mcConnection, _mobileInfoForAllGuiTests, _parallelRunnerEnvironments);
break;
}
if (runner != null)
{
if (!_colRunnersForCleanup.ContainsKey(type))
{
_colRunnersForCleanup.Add(type, runner);
}
Stopwatch s = Stopwatch.StartNew();
var results = runner.RunTest(testinf, ref errorReason, RunCancelled);
results.Runtime = s.Elapsed;
if (type == TestType.LoadRunner)
{
AppDomain.CurrentDomain.AssemblyResolve -= Helper.HPToolsAssemblyResolver;
}
return(results);
}
//check for abortion
if (System.IO.File.Exists(_abortFilename))
{
ConsoleWriter.WriteLine(Resources.GeneralStopAborted);
//stop working
Environment.Exit((int)Launcher.ExitCodeEnum.Aborted);
}
return(new TestRunResults {
ErrorDesc = "Unknown TestType", TestState = TestState.Error
});
}
protected override int DoRun(Console.ConsoleOptions options, bool redirectOutput, bool redirectError, TestPackage package, TextWriter outWriter, TextWriter errorWriter, TestFilter testFilter, out TestResult result, EventCollector collector)
{
result = null;
var testRunner1 = new DefaultTestRunnerFactory().MakeTestRunner(package);
try
{
testRunner1.Load(package);
if (testRunner1.Test == null)
{
testRunner1.Unload();
System.Console.Error.WriteLine("Unable to locate fixture {0}", options.fixture);
return FIXTURE_NOT_FOUND;
}
}
finally
{
var disp = testRunner1 as IDisposable;
if (disp != null)
disp.Dispose();
}
result = new TestResult(new TestName { Name = "Global" });
var timer = new Stopwatch();
timer.Start();
var testRunnerId = 0;
{
var syncTestRunner = new DefaultTestRunnerFactory().MakeTestRunner(package);
var syncFilter = new AndFilter(testFilter, new SynchronousFilter());
var logger = new ConsoleLoggingEventListener(collector);
result.AddResult(RunPartition(redirectOutput, redirectError, package, outWriter, errorWriter, syncFilter, syncTestRunner, logger));
testRunnerId = syncTestRunner.ID;
}
var dep = 0;
var consoleOptions = options as ConsoleOptions;
if (options != null)
dep = consoleOptions.degreeofparallelism;
if (dep == 0)
dep = 4;
System.Console.WriteLine("Degree of Parallelism: {0}", dep);
var state = new AsynchronousFilterState(dep);
var asyncTestRunner = new ParallelTestRunner(testRunnerId + 1, dep, state);
result.AddResult(RunPartition(redirectOutput, redirectError, package, outWriter, errorWriter, testFilter, asyncTestRunner, collector));
if (consoleOptions != null && consoleOptions.retestfailures)
{
var failedTests = (from test in Flatten(result)
where test.Result.IsFailure
select test).ToList();
var failedTestFilter = new SimpleNameFilter(failedTests.Select(t => t.Result.FullName));
var retestTestRunner = new DefaultTestRunnerFactory().MakeTestRunner(package);
var retestFilter = new AndFilter(testFilter, failedTestFilter);
var logger = new ConsoleLoggingEventListener(collector);
var retestResults = RunPartition(redirectOutput, redirectError, package, outWriter, errorWriter, retestFilter, retestTestRunner, logger);
var newTests = Flatten(retestResults).ToDictionary(test => test.Result.FullName);
foreach (var failedTest in failedTests)
{
var newTest = newTests[failedTest.Result.FullName];
if (newTest.Result.IsSuccess)
{
failedTest.Parent.Results.Remove(failedTest.Result);
failedTest.Parent.Results.Add(newTest.Result);
}
}
}
result = MergeResults(result.Test, (TestResult)result.Results[0], (TestResult)result.Results[1]);
timer.Stop();
result.Time = timer.ElapsedTicks / (double)TimeSpan.TicksPerSecond;
return 0;
}
