public async Task <TestResults> ScanAndRunAsync(List <TestOutput> outputs, Func <CancellationTokenSource> GetNewCancellationToken)
{
await outputs.ForEachAsync(output => output.AtStartupAsync());
TestResults results = new TestResults();
Stopwatch testTimeWatch = new Stopwatch();
var options = new EnumerationOptions
{
RecurseSubdirectories = true,
};
TestParser parser = new TestParser();
// sort tests before running to keep test order the same everywhere
var sortedFiles = root
.EnumerateFiles("test.json", options)
.OrderBy(f => f.DirectoryName);
foreach (var file in sortedFiles)
{
testTimeWatch.Reset();
var cancellationTokenSource = GetNewCancellationToken();
var cancellationToken = cancellationTokenSource.Token;
await cleaner.CleanLocalDotNetCacheAsync();
testTimeWatch.Start();
var parsedTest = await parser.TryParseAsync(system, nuGetConfig, file);
if (!parsedTest.Success)
{
Console.WriteLine($"WARNING: Unable to parse {file}");
continue;
}
var test = parsedTest.Test;
await outputs.ForEachAsync(output => output.AfterParsingTestAsync(test.Descriptor.Name, !test.Skip));
if (test.Descriptor.Cleanup)
{
await cleaner.CleanProjectLocalDotNetCruftAsync();
}
var result = await test.RunAsync(cancellationToken);
testTimeWatch.Stop();
results.Total++;
switch (result.Status)
{
case TestStatus.Passed: results.Passed++; break;
case TestStatus.Failed: results.Failed++; break;
case TestStatus.Skipped: results.Skipped++; break;
}
await outputs.ForEachAsync(output => output.AfterRunningTestAsync(test.Descriptor.Name, result, testTimeWatch.Elapsed));
}
await outputs.ForEachAsync(output => output.AfterRunningAllTestsAsync(results));
return(results);
}
public async override Task AfterRunningAllTestsAsync(TestResults results)
{
Console.WriteLine($"Total: {results.Total} Passed: {results.Passed} Failed: {results.Failed}");
}
public async virtual Task AfterRunningAllTestsAsync(TestResults results)
{
}
public async override Task AfterRunningAllTestsAsync(TestResults results)
{
var settings = new XmlWriterSettings();
settings.Indent = true;
using (var writer = XmlWriter.Create(_resultsFile.FullName, settings))
{
writer.WriteStartDocument();
writer.WriteStartElement("testsuite");
writer.WriteAttributeString("name", "dotnet");
writer.WriteAttributeString("tests", _testCases.Count.ToString());
writer.WriteAttributeString("failures", _testCases.Where(t => t.Failed).Count().ToString());
writer.WriteAttributeString("errors", "0");
foreach (var testCase in _testCases)
{
writer.WriteStartElement("testcase");
writer.WriteAttributeString("name", testCase.Name);
writer.WriteAttributeString("classname", testCase.ClassName);
if (testCase.Skipped)
{
writer.WriteStartElement("skipped");
writer.WriteAttributeString("message", testCase.Message);
writer.WriteEndElement();
}
if (testCase.Failed)
{
writer.WriteStartElement("failure");
writer.WriteAttributeString("message", testCase.Message);
writer.WriteAttributeString("type", "AssertionError");
writer.WriteEndElement();
}
if (testCase.StandardOutput != null)
{
writer.WriteStartElement("system-out");
string standardOutput = RemoveInvalidXmlCharacters(testCase.StandardOutput);
writer.WriteString(standardOutput);
writer.WriteEndElement();
}
if (testCase.StandardError != null)
{
writer.WriteStartElement("system-err");
string standardError = RemoveInvalidXmlCharacters(testCase.StandardError);
writer.WriteString(standardError);
writer.WriteEndElement();
}
writer.WriteEndElement();
}
writer.WriteEndElement();
writer.WriteEndDocument();
writer.Close();
}
}