private TestExecutionResults ExecuteTests(string testModule, IEnumerable <IMemberDefinition> testsToRun)
{
if (!args.RunInIsolation && args.InstrumentationStrategy == InstrumentationStrategy.StaticConstructor)
{
throw new Exception("When StaticConstructor instrumentation strategy is used tests should be run in Isolation.");
}
ITestExecutor executor = null;
if (args.RunInIsolation)
{
executor = new IsolationTestExecutor(args.TestingFramework.GetExecutor());
}
else
{
executor = args.TestingFramework.GetExecutor();
}
TestExecutionResults results = executor.Execute(testModule, testsToRun, args.TestingFrameworkArguments);
int numOfExecuted = args.DependencyCollectionGranularity == DependencyCollectionGranularity.Class ?
results.ExecutedTestClassesCount : results.ExecutedTestMethodsCount;
if (testsToRun.Count() != numOfExecuted)
{
logger.Warn("Number of tests scheduled for run does not match number of actually run tests");
}
return(results);
}
private TestExecutionResults ProcessExecutionResults(XmlNode xml)
{
TestExecutionResults result = new TestExecutionResults();
var assemblyNodes = xml.GetChildren("assembly");
if (assemblyNodes.Count != 1)
{
logger.Error("There should be results from exactly one assembly since we are running a single assembly.");
return(null);
}
XmlNode assemblyNode = assemblyNodes[0];
result.PassedTestMethodsCount = assemblyNode.ParseAttribute <int>(PassedTestCaseCountAttribute);
result.FailedTestMethodsCount = assemblyNode.ParseAttribute <int>(FailedTestCaseCountAttribute);
result.SkippedTestMethodsCount = assemblyNode.ParseAttribute <int>(SkippedTestCaseCountAttribute);
result.ExecutionTime = assemblyNode.ParseAttribute <decimal>(TimeAttribute);
var tests = assemblyNode.GetChildren("test");
foreach (var test in tests)
{
string testClassName = test.ParseAttribute <string>("type");
string testMethodName = test.ParseAttribute <string>("name");
if (testMethodName.Contains('('))
{
// convert trait name into test method name
testMethodName = testMethodName.CutTillFirstOccurence('(');
}
result.ExecutedTestClasses.Add(testClassName);
result.ExecutedTestMethods.Add(testMethodName);
}
return(result);
}
// TODO: Check if we can avoid using operator
public static TestExecutionResults operator +(TestExecutionResults one, TestExecutionResults two)
{
TestExecutionResults cumalative = new TestExecutionResults();
cumalative.PassedTestMethodsCount = one.PassedTestMethodsCount + two.PassedTestMethodsCount;
cumalative.FailedTestMethodsCount = one.FailedTestMethodsCount + two.FailedTestMethodsCount;
cumalative.SkippedTestMethodsCount = one.SkippedTestMethodsCount + two.SkippedTestMethodsCount;
cumalative.ExecutionTime = one.ExecutionTime + two.ExecutionTime;
cumalative.TestClasses.UnionWith(one.TestClasses);
cumalative.TestClasses.UnionWith(two.TestClasses);
cumalative.ExecutedTestClasses.UnionWith(one.ExecutedTestClasses);
cumalative.ExecutedTestClasses.UnionWith(two.ExecutedTestClasses);
cumalative.NotExecutedTestClasses.UnionWith(one.NotExecutedTestClasses);
cumalative.NotExecutedTestClasses.UnionWith(two.NotExecutedTestClasses);
cumalative.TestMethods.UnionWith(one.TestMethods);
cumalative.TestMethods.UnionWith(two.TestMethods);
cumalative.ExecutedTestMethods.UnionWith(one.ExecutedTestMethods);
cumalative.ExecutedTestMethods.UnionWith(two.ExecutedTestMethods);
cumalative.NotExecutedTestMethods.UnionWith(one.NotExecutedTestMethods);
cumalative.NotExecutedTestMethods.UnionWith(two.NotExecutedTestMethods);
cumalative.FailedTestMethods.UnionWith(one.FailedTestMethods);
cumalative.FailedTestMethods.UnionWith(two.FailedTestMethods);
return(cumalative);
}
protected override TestExecutionResults ExecuteImpl(string testModule, IEnumerable <IMemberDefinition> testsToRun, string[] args)
{
if (!testsToRun.Any())
{
// discovery cannot be done
// When running with empty list of tests XUnit hangs
return(new TestExecutionResults());
}
executedTests.Clear();
this.testsToRun = testsToRun;
executionCanceled = false;
result = new TestExecutionResults();
runner = AssemblyRunner.WithAppDomain(testModule);
logger.Debug($"Running XUnit executor in App domain: {AppDomain.CurrentDomain.FriendlyName}");
runner.TestCaseFilter = TestCaseFilterHandler;
runner.OnExecutionComplete = ExecutionCompleteHandler;
runner.OnTestFailed = TestFailedHandler;
runner.OnTestFinished += TestFinishedHandler;
runner.OnDiscoveryComplete += DiscoveryCompleteHandler;
runner.OnErrorMessage += ErrorMessageHandler;
logger.Debug("Starting tests");
runner.Start(maxParallelThreads: 1);
finished.WaitOne();
finished.Dispose();
while (runner.Status != AssemblyRunnerStatus.Idle && !executionCanceled)
{
// spin until runner finishes
}
if (executionCanceled)
{
try
{
runner.Dispose();
}
catch (InvalidOperationException e)
{
// for now ignore this exception
// due to XUnit bug of not resetting Status to Idle when cancelling run
}
}
else
{
runner.Dispose();
}
var temp = result;
// reset result variable
result = null;
return(temp);
}
protected override TestExecutionResults ExecuteImpl(string testModule, IEnumerable <IMemberDefinition> testsToRun, string[] arguments)
{
logger.Info(string.Format("Running NUnit2 executor in App domain: {0}", AppDomain.CurrentDomain.FriendlyName));
result = new TestExecutionResults();
var appDomain = AppDomain.CreateDomain("nunit2-test", null, testModule, null, false);
NUnit2FrameworkDriver driver = new NUnit2FrameworkDriver(appDomain)
{
ID = "1"
}; // Not sure why "1" but necessary
var settings = new Dictionary <string, object>();
string x = driver.Load(testModule, settings);
TestFilterBuilder filterBuilder = new TestFilterBuilder();
foreach (var t in testsToRun)
{
filterBuilder.AddTest(t.GetFullName());
}
string filter = filterBuilder.GetFilter().Text;
string discoveryXmlString = driver.Explore(filter);
// TODO: Check if discovery information can be otained from
// execution results. Not sure if execution results contain all
// test methods and classes or only executed ones
ParseDiscoveryResults(discoveryXmlString.ToXmlDocument());
if (testsToRun.Count() > 0)
{
string xmlString = driver.Run(this, filter);
ParseExecutionResults(xmlString.ToXmlDocument());
}
else
{
logger.Info("No tests selected for run");
}
AppDomain.Unload(appDomain);
var temp = result;
// clean the local field for the next time
result = null;
return(temp);
}
public List <TestExecutionResults> Run()
{
CleanFiles();
InitializeProject();
List <TestExecutionResults> results = new List <TestExecutionResults>();
do
{
try
{
logger.Info("Building project");
if (!BuildProject())
{
throw new Exception("Build failed");
}
BackupModules();
Dictionary <string, ICollection <IMemberDefinition> > moduleToAffectedTests = InstrumentAndSelect(ProgramModules, TestModules);
logger.Info("Executing tests");
TestExecutionResults result = new TestExecutionResults();
foreach (KeyValuePair <string, ICollection <IMemberDefinition> > moduleToTests in moduleToAffectedTests)
{
result += ExecuteTests(moduleToTests.Key, moduleToTests.Value);
}
logger.Info("Test execution summary: " + result.ToString());
if (args.Debug)
{
LogResults(result);
}
results.Add(result);
RestoreModules();
}
catch (Exception e)
{
logger.Error("Exception thrown", e);
}
}while (MoveToNextRevision());
return(results);
}
protected override TestExecutionResults ExecuteImpl(string testModule, IEnumerable <IMemberDefinition> testsToRun, string[] arguments)
{
// TODO: do discovery of all tests inside the module
if (!testsToRun.Any())
{
return(new TestExecutionResults());
}
EnsureResultsDirExists();
// IEnumerable<string> toRun = FilterRunListBasedOnUserArguments(testsToRun, arguments);
List <string> xunitArgs = ConstructXUnitArguments(testsToRun, arguments);
TestExecutionResults results = new TestExecutionResults();
for (int i = 0; i < xunitArgs.Count; i++)
{
string commandLineArgs = xunitArgs[i];
ProcessUtil.ExecuteProcessNoPoppingConsole(CommonPaths.XUnitConsoleExePath, $"{testModule} {commandLineArgs}");
string xmlResults = File.ReadAllText(GetResultsFilePath(i));
results = results + ProcessExecutionResults(xmlResults.ToXmlDocument());
}
return(results);
}
private void LogResults(TestExecutionResults result)
{
IOUtil.WriteAllText(CommonPaths.TestResultsFilePath, JsonConvert.SerializeObject(result, Formatting.Indented));
IOUtil.WriteAllText(CommonPaths.TotalNumberOfTestClassesFilePath, result.TotalNumberOfTestClasses.ToString());
IOUtil.WriteAllText(CommonPaths.NumberOfExecutedTestClassesFilePath, result.ExecutedTestClassesCount.ToString());
IOUtil.WriteAllText(CommonPaths.TotalNumberOfTestMethodsFilePath, result.TotalNumberOfTestMethods.ToString());
IOUtil.WriteAllText(CommonPaths.NumberOfExecutedTestMethodsFilePath, result.ExecutedTestMethodsCount.ToString());
IOUtil.WriteAllText(CommonPaths.NumberOfPassedTestMethodsFilePath, result.PassedTestMethodsCount.ToString());
IOUtil.WriteAllText(CommonPaths.NumberOfFailedTestMethodsFilePath, result.FailedTestMethodsCount.ToString());
IOUtil.WriteAllText(CommonPaths.NumberOfSkippedTestMethodsFilePath, result.SkippedTestMethodsCount.ToString());
decimal executionTime = result.ExecutionTime;
IOUtil.WriteAllText(CommonPaths.ExecutionTimeFilePath, executionTime.FormattedNumber());
double analysisTime = analysisStopwatch.Elapsed.TotalSeconds;
IOUtil.WriteAllText(CommonPaths.AnalysisTimeFilePath, analysisTime.FormattedNumber());
double instrumentationTime = instrumentationStopwatch.Elapsed.TotalSeconds;
IOUtil.WriteAllText(CommonPaths.InstrumentationTimeFilePath, instrumentationTime.FormattedNumber());
double totalTime = (double)executionTime + analysisTime + instrumentationTime;
IOUtil.WriteAllText(CommonPaths.TotalTimeFilePath, totalTime.FormattedNumber());
}
protected override TestExecutionResults ExecuteImpl(string testModule, IEnumerable <IMemberDefinition> testsToRun, string[] args)
{
logger.Info(string.Format("Running NUnit3 executor in App domain: {0}", AppDomain.CurrentDomain.FriendlyName));
result = new TestExecutionResults();
TestFilterBuilder filterBuilder = new TestFilterBuilder();
foreach (var t in testsToRun)
{
filterBuilder.AddTest(t.GetFullName());
}
using (var engine = TestEngineActivator.CreateInstance())
{
TestPackage package = new TestPackage(testModule);
using (var runner = engine.GetRunner(package))
{
// explore all tests (default filter used)
XmlNode explorationResults = runner.Explore(new TestFilterBuilder().GetFilter());
ParseDiscoveryResults(explorationResults);
//foreach (var test in result.TestClasses)
//{
// if (!testsToRun.Contains(test))
// {
// result.NotExecutedTestClasses.Add(test);
// }
//}
//foreach (var test in testsToRun)
//{
// if (!result.TestClasses.Contains(test))
// {
// logger.Debug($"Test: {test} found during analyzes phase but not discovered using Runner");
// }
//}
if (testsToRun.Count() > 0)
{
XmlNode runResults = runner.Run(this, filterBuilder.GetFilter());
ParseExecutionResults(runResults);
try
{
runner.Unload();
}
catch (Exception e)
{
logger.Warn("Error while unloading runner", e);
}
}
else
{
logger.Info("No tests selected for run");
}
}
}
var temp = result;
// clean the local field for the next time
result = null;
return(temp);
}
