public override async Task ExecuteAsync(IOperationExecutionContext context)
{
var fileOps = await context.Agent.GetServiceAsync <IFileOperationsExecuter>();
var tmpXmlPath = fileOps.CombinePath(context.WorkingDirectory, $"{Guid.NewGuid().ToString()}.xml");
var testStart = DateTimeOffset.Now;
await this.ExecutePHPAsync(context, $"{await this.GetPHPUnitPathAsync(context)} --log-junit {tmpXmlPath} {this.TestToRun}");
if (!await fileOps.FileExistsAsync(tmpXmlPath))
{
this.LogWarning("PHPUnit did not generate an output XML file, therefore no tests were run. This can be caused if there are no test cases in the action's source directory, or the test file names do not end with \"Test\" (case-sensitive).");
return;
}
List <UnitTestOutput.TestSuite> suites;
try
{
using (var stream = await fileOps.OpenFileAsync(tmpXmlPath, FileMode.Open, FileAccess.Read))
{
suites = UnitTestOutput.Load(stream);
}
}
finally
{
await fileOps.DeleteFileAsync(tmpXmlPath);
}
var testRecorder = await context.TryGetServiceAsync <IUnitTestRecorder>();
foreach (var suite in suites)
{
foreach (var test in suite.TestCases)
{
await testRecorder.RecordUnitTestAsync(
groupName : suite.Name,
testName : test.Name,
testStatus : test.Failures.Any()?UnitTestStatus.Failed : UnitTestStatus.Passed,
testResult : string.Empty,
startTime : testStart,
duration : TimeSpan.FromSeconds((double)test.Time)
);
}
}
}
private async Task <string> GetContainerConfigText(IOperationExecutionContext context)
{
var deployer = await context.TryGetServiceAsync <IConfigurationFileDeployer>();
if (deployer == null)
{
throw new NotSupportedException("Configuration files are not supported in this context.");
}
using (var writer = new StringWriter())
{
if (!await deployer.WriteAsync(writer, this.ConfigFileName, this.ConfigInstanceName, this))
{
return(null);
}
return(Regex.Replace(writer.ToString(), @"\r?\n", " "));
}
}
public override async Task ExecuteAsync(IOperationExecutionContext context)
{
var fileOps = await context.Agent.GetServiceAsync <IFileOperationsExecuter>();
var testFilePath = context.ResolvePath(this.TestFile);
this.LogDebug("Test file: " + testFilePath);
if (!await fileOps.FileExistsAsync(testFilePath))
{
this.LogError($"Test file {testFilePath} does not exist.");
return;
}
var exePath = context.ResolvePath(this.ExePath);
if (await fileOps.FileExistsAsync(exePath))
{
this.LogDebug("Exe path: " + exePath);
}
else
{
exePath = this.ExePath;
// different message formatting to assit with debugging
this.LogDebug("Using executable: " + exePath);
}
string outputFilePath;
if (string.IsNullOrEmpty(this.CustomXmlOutputPath))
{
outputFilePath = fileOps.CombinePath(context.WorkingDirectory, Guid.NewGuid().ToString("N") + ".xml");
}
else
{
outputFilePath = context.ResolvePath(this.CustomXmlOutputPath);
}
this.LogDebug("Output file: " + outputFilePath);
var args = this.IsNUnit3
? $"\"{testFilePath}\" --result:\"{outputFilePath}\";format=nunit2"
: $"\"{testFilePath}\" /xml:\"{outputFilePath}\"";
if (!string.IsNullOrEmpty(this.AdditionalArguments))
{
this.LogDebug("Additional arguments: " + this.AdditionalArguments);
args += " " + this.AdditionalArguments;
}
try
{
await this.ExecuteCommandLineAsync(
context,
new RemoteProcessStartInfo
{
FileName = exePath,
Arguments = args,
WorkingDirectory = context.WorkingDirectory
}
);
XDocument xdoc;
using (var stream = await fileOps.OpenFileAsync(outputFilePath, FileMode.Open, FileAccess.Read))
{
xdoc = XDocument.Load(stream);
}
#if DEBUG
this.LogDebug(xdoc.ToString());
#endif
var testResultsElement = xdoc.Element("test-results");
var startTime = this.TryParseStartTime((string)testResultsElement.Attribute("date"), (string)testResultsElement.Attribute("time")) ?? DateTime.UtcNow;
var failures = 0;
var testRecorder = await context.TryGetServiceAsync <IUnitTestRecorder>();
foreach (var testCaseElement in xdoc.Descendants("test-case"))
{
var testName = (string)testCaseElement.Attribute("name");
// skip tests that weren't actually run
if (string.Equals((string)testCaseElement.Attribute("executed"), "False", StringComparison.OrdinalIgnoreCase))
{
this.LogInformation($"NUnit test: {testName} (skipped)");
continue;
}
var result = AH.Switch <string, UnitTestStatus>((string)testCaseElement.Attribute("success"), StringComparer.OrdinalIgnoreCase)
.Case("True", UnitTestStatus.Passed)
.Case("Inconclusive", UnitTestStatus.Inconclusive)
.Default(UnitTestStatus.Failed)
.End();
if (result == UnitTestStatus.Failed)
{
failures++;
}
var testDuration = this.TryParseTestTime((string)testCaseElement.Attribute("time"));
this.LogInformation($"NUnit test: {testName}, Result: {result}, Test length: {testDuration}");
if (testRecorder != null)
{
await testRecorder.RecordUnitTestAsync(
groupName : AH.NullIf(this.GroupName, string.Empty) ?? "NUnit",
testName : testName,
testStatus : result,
testResult : testCaseElement.ToString(),
startTime : startTime,
duration : testDuration
);
}
startTime += testDuration;
}
if (failures > 0)
{
this.LogError($"{failures} test failures were reported.");
}
}
finally
{
if (string.IsNullOrEmpty(this.CustomXmlOutputPath))
{
this.LogDebug($"Deleting temp output file ({outputFilePath})...");
try
{
await fileOps.DeleteFileAsync(outputFilePath);
}
catch
{
this.LogWarning($"Could not delete {outputFilePath}.");
}
}
}
}
public override async Task ExecuteAsync(IOperationExecutionContext context)
{
var recorder = await context.TryGetServiceAsync <IUnitTestRecorder>() ?? throw new ExecutionFailureException("This operation requires a unit test recorder.");
await this.RunTestsAsync(context);
foreach (var test in this.Events.Where(e => e.Test != null).GroupBy(e => e.Test))
{
await recorder.RecordUnitTestAsync(
groupName : test.Key.Group,
testName : test.Key.Name,
testStatus : getStatus(test),
testResult : getTestLog(test),
startTime : getStartTime(test),
duration : getDuration(test)
);
}
UnitTestStatus getStatus(IGrouping <TestCaseID, TestEvent> test)
{
foreach (var e in test)
{
switch (e.Type)
{
case EventType.Success:
case EventType.ExpectedFailure:
return(UnitTestStatus.Passed);
case EventType.Error:
case EventType.Failure:
case EventType.UnexpectedSuccess:
return(UnitTestStatus.Failed);
case EventType.Skip:
return(UnitTestStatus.Inconclusive);
}
}
return(UnitTestStatus.Inconclusive);
}
string getTestLog(IGrouping <TestCaseID, TestEvent> test)
{
var end = test.FirstOrDefault(e => e.Type == EventType.StopCase);
var result = (test.FirstOrDefault(e => e.Type == EventType.Error || e.Type == EventType.Failure || e.Type == EventType.UnexpectedSuccess || e.Type == EventType.Skip)
?? test.FirstOrDefault(e => e.Type == EventType.Success || e.Type == EventType.ExpectedFailure))?.Type;
var stdout = end?.Output;
var stderr = end?.Error;
var skipReason = test.FirstOrDefault(e => e.Type == EventType.Skip)?.Message;
var exceptions = test.Select(e => e.Err).Where(e => e != null).ToArray();
return("Test: " + test.Key.ID + AH.ConcatNE("\n", test.Key.Desc) +
"\n\nResult: " + AH.CoalesceString(result, "Unknown") + AH.ConcatNE(" (", skipReason, ")") +
AH.ConcatNE("\n\nOutput:\n", stdout) + AH.ConcatNE("\n\nError:\n", stderr) +
(exceptions.Any() ? "\n\nExceptions:\n\n" + string.Join("\n\n", exceptions) : string.Empty) +
"\n");
}
DateTimeOffset getStartTime(IGrouping <TestCaseID, TestEvent> test)
{
return((test.FirstOrDefault(e => e.Type == EventType.StartCase) ?? test.First()).NowTime);
}
TimeSpan getDuration(IGrouping <TestCaseID, TestEvent> test)
{
var start = (test.FirstOrDefault(e => e.Type == EventType.StartCase) ?? test.First()).Time;
var end = (test.FirstOrDefault(e => e.Type == EventType.StopCase) ?? test.Last()).Time;
return(TimeSpan.FromSeconds(end - start));
}
}
public override async Task ExecuteAsync(IOperationExecutionContext context)
{
var vsTestPath = await this.GetVsTestPathAsync(context);
if (string.IsNullOrEmpty(vsTestPath))
{
return;
}
var fileOps = await context.Agent.GetServiceAsync <IFileOperationsExecuter>();
var containerPath = context.ResolvePath(this.TestContainer);
var sourceDirectory = PathEx.GetDirectoryName(containerPath);
var resultsPath = PathEx.Combine(sourceDirectory, "TestResults");
if (this.ClearExistingTestResults)
{
this.LogDebug($"Clearing {resultsPath} directory...");
await fileOps.ClearDirectoryAsync(resultsPath);
}
await this.ExecuteCommandLineAsync(
context,
new RemoteProcessStartInfo
{
FileName = vsTestPath,
Arguments = $"\"{containerPath}\" /logger:trx {this.AdditionalArguments}",
WorkingDirectory = sourceDirectory
}
);
if (!await fileOps.DirectoryExistsAsync(resultsPath))
{
this.LogError("Could not find the generated \"TestResults\" directory after running unit tests at: " + sourceDirectory);
return;
}
var trxFiles = (await fileOps.GetFileSystemInfosAsync(resultsPath, new MaskingContext(new[] { "*.trx" }, Enumerable.Empty <string>())))
.OfType <SlimFileInfo>()
.ToList();
if (trxFiles.Count == 0)
{
this.LogError("There are no .trx files in the \"TestResults\" directory.");
return;
}
var trxPath = trxFiles
.Aggregate((latest, next) => next.LastWriteTimeUtc > latest.LastWriteTimeUtc ? next : latest)
.FullName;
XDocument doc;
using (var file = await fileOps.OpenFileAsync(trxPath, FileMode.Open, FileAccess.Read))
using (var reader = new XmlTextReader(file)
{
Namespaces = false
})
{
doc = XDocument.Load(reader);
}
var testRecorder = await context.TryGetServiceAsync <IUnitTestRecorder>();
bool failures = false;
foreach (var result in doc.Element("TestRun").Element("Results").Elements("UnitTestResult"))
{
var testName = (string)result.Attribute("testName");
var outcome = (string)result.Attribute("outcome");
var output = result.Element("Output");
UnitTestStatus status;
string testResult;
if (string.Equals(outcome, "Passed", StringComparison.OrdinalIgnoreCase))
{
status = UnitTestStatus.Passed;
testResult = "Passed";
}
else if (string.Equals(outcome, "NotExecuted", StringComparison.OrdinalIgnoreCase))
{
status = UnitTestStatus.Inconclusive;
if (output == null)
{
testResult = "Ignored";
}
else
{
testResult = GetResultTextFromOutput(output);
}
}
else
{
status = UnitTestStatus.Failed;
testResult = GetResultTextFromOutput(output);
failures = true;
}
if (testRecorder != null)
{
var startDate = (DateTimeOffset)result.Attribute("startTime");
var duration = parseDuration((string)result.Attribute("duration"));
await testRecorder.RecordUnitTestAsync(AH.CoalesceString(this.TestGroup, "Unit Tests"), testName, status, testResult, startDate, duration);
}
}
if (failures)
{
this.LogError("One or more unit tests failed.");
}
else
{
this.LogInformation("Tests completed with no failures.");
}
TimeSpan parseDuration(string s)
{
if (!string.IsNullOrWhiteSpace(s))
{
var m = DurationRegex.Match(s);
if (m.Success)
{
int hours = int.Parse(m.Groups[1].Value);
int minutes = int.Parse(m.Groups[2].Value);
int seconds = int.Parse(m.Groups[3].Value);
var timeSpan = new TimeSpan(hours, minutes, seconds);
if (m.Groups[4].Success)
{
var fractionalSeconds = double.Parse("0." + m.Groups[4].Value, CultureInfo.InvariantCulture);
timeSpan += TimeSpan.FromSeconds(fractionalSeconds);
}
return(timeSpan);
}
}
return(TimeSpan.Zero);
}
}
public override async Task ExecuteAsync(IOperationExecutionContext context)
{
var fileOps = context.Agent.GetService <IFileOperationsExecuter>();
var bmjPath = fileOps.CombinePath(fileOps.GetBaseWorkingDirectory(), "ExtTemp", "Java", "BuildMaster.jar");
var sourceDirectory = context.ResolvePath(this.SourceDirectory);
if (!fileOps.DirectoryExists(sourceDirectory))
{
this.LogError($"Source directory {sourceDirectory} not found.");
return;
}
this.LogDebug("Source directory: " + sourceDirectory);
var testClasses = (from f in fileOps.GetFileSystemInfos(sourceDirectory, new MaskingContext(this.Includes, this.Excludes)).OfType <SlimFileSystemInfo>()
where f.FullName.EndsWith(".class", StringComparison.OrdinalIgnoreCase)
select f.FullName.Substring(sourceDirectory.Length, f.FullName.Length - sourceDirectory.Length - ".class".Length).Trim('/', '\\').Replace('/', '.').Replace('\\', '.')).ToList();
if (testClasses.Count == 0)
{
this.LogWarning($"No test class files found in {sourceDirectory}.");
return;
}
var testClassesFileName = fileOps.CombinePath(sourceDirectory, Guid.NewGuid().ToString("N"));
try
{
fileOps.WriteAllText(testClassesFileName, string.Join(fileOps.NewLine, testClasses));
var testTime = DateTime.UtcNow;
await this.ExecuteCommandLineAsync(
context,
new RemoteProcessStartInfo
{
FileName = this.JavaPath,
Arguments = $"-cp .;{bmjPath} -Djava.exe.dirs=\"{string.Join(";", this.ExtensionDirectories ?? Enumerable.Empty<string>())}\" inedo.buildmasterextensions.java.jUnitAction \"@{testClassesFileName}\"",
WorkingDirectory = sourceDirectory
}
);
XDocument testResults;
try
{
testResults = XDocument.Parse(standardOut.ToString());
}
catch (Exception ex)
{
this.LogError("Unable to load test results: " + ex.Message);
this.LogDebug(standardOut.ToString());
return;
}
var testRecorder = await context.TryGetServiceAsync <IUnitTestRecorder>();
foreach (var tr in testResults.Descendants("TestResult"))
{
var failures = tr.Descendants("Failure").ToList();
int runCount = (int)tr.Attribute("RunCount");
int failCount = failures.Count;
int ignoreCount = (int)tr.Attribute("IgnoreCount");
int runTime = (int)tr.Attribute("RunTime");
var testName = (string)tr.Attribute("Class");
var testResult = new StringBuilder();
if (runCount > 0)
{
testResult.AppendLine("Tests run: " + runCount.ToString());
if (failCount == 0)
{
testResult.AppendLine().AppendLine("No failures.");
}
else if (failCount == 1)
{
testResult.AppendLine().AppendLine("1 failure.");
}
else
{
testResult.AppendLine().AppendLine(failCount.ToString() + " failures.");
}
}
foreach (var fail in failures)
{
testResult.AppendLine();
testResult.AppendLine();
testResult.AppendLine("__" + fail.Attribute("TestHeader").Value + "__");
testResult.AppendLine("Exception: " + fail.Attribute("ExceptionClass").Value);
testResult.AppendLine("Message: " + fail.Attribute("Message").Value);
}
var tout = tr.Element("TestOutput");
if (tout != null)
{
testResult.AppendLine();
testResult.AppendLine();
testResult.AppendLine("__Output__");
foreach (var cdata in tout.Nodes().OfType <XCData>())
{
testResult.AppendLine(cdata.Value);
}
}
if (testRecorder != null)
{
await testRecorder.RecordUnitTestAsync(
groupName : null,
testName : testName,
testStatus : failCount == 0?UnitTestStatus.Passed : UnitTestStatus.Failed,
testResult : testResult.ToString(),
startTime : testTime,
duration : TimeSpan.FromMilliseconds(runTime)
).ConfigureAwait(false);
}
testTime = testTime.AddMilliseconds(runTime);
}
}
finally
{
try
{
fileOps.DeleteFile(testClassesFileName);
}
catch
{
}
}
}
public override async Task ExecuteAsync(IOperationExecutionContext context)
{
var execOps = await context.Agent.GetServiceAsync <IRemoteProcessExecuter>();
var bits = AH.Switch <BuildArchitecture, int>(this.Architecture)
.Case(BuildArchitecture.i386, 32)
.Case(BuildArchitecture.x86_64, 64)
.End();
var testStartInfo = new RemoteProcessStartInfo
{
FileName = "dfhack-test",
Arguments = $"{this.OperatingSystem.ToString().ToLowerInvariant()} {bits}",
WorkingDirectory = context.WorkingDirectory
};
var cidfile = await this.LogAndWrapCommandAsync(context, testStartInfo, false, false);
var testLines = new List <string>();
var recorder = await context.TryGetServiceAsync <IUnitTestRecorder>();
Task recorderTask = InedoLib.NullTask;
using (var test = execOps.CreateProcess(testStartInfo))
{
var lastTime = DateTimeOffset.Now;
test.OutputDataReceived += (s, e) =>
{
string text = e.Data;
if (this.ImageTag == "msvc")
{
text = e.Data.TrimEnd();
if (text == @"wine: cannot find L""C:\\windows\\Microsoft.NET\\Framework\\v4.0.30319\\mscorsvw.exe""")
{
return;
}
}
testLines.Add(text);
if (text.StartsWith("Running ") && text.EndsWith(" tests"))
{
testLines.Clear();
lastTime = DateTimeOffset.Now;
}
if (text.StartsWith("ERROR: "))
{
this.LogError(text.Substring("ERROR: ".Length));
return;
}
if (text.StartsWith("WARN: "))
{
this.LogWarning(text.Substring("WARN: ".Length));
return;
}
if (text.StartsWith("warning: "))
{
this.LogWarning(text.Substring("warning: ".Length));
return;
}
if (text.StartsWith("test passed: "))
{
var testName = text.Substring("test passed: ".Length);
recordUnitTest(testName, UnitTestStatus.Passed, ref lastTime);
}
this.LogInformation(text);
};
test.ErrorDataReceived += (s, e) =>
{
string text = e.Data;
if (this.ImageTag == "msvc")
{
text = e.Data.TrimEnd();
if (text == @"wine: cannot find L""C:\\windows\\Microsoft.NET\\Framework\\v4.0.30319\\mscorsvw.exe""")
{
return;
}
}
testLines.Add(text);
if (text.StartsWith("Plugin ") && text.Contains(" is missing required globals: "))
{
this.LogWarning(text);
return;
}
if (text.StartsWith("test failed: "))
{
var testName = text.Substring("test failed: ".Length);
recordUnitTest(testName, UnitTestStatus.Failed, ref lastTime);
}
if (text.StartsWith("test errored: "))
{
var testName = string.Join(":", text.Substring("test errored: ".Length).Split(new[] { ':' }, 3).Take(2));
recordUnitTest(testName, UnitTestStatus.Failed, ref lastTime);
}
this.LogError(text);
};
test.Start();
using (ManageContainerIDFile(context, cidfile))
{
await test.WaitAsync(context.CancellationToken);
}
await recorderTask;
if (test.ExitCode != 0)
{
this.LogError("Tests failed!");
}
void recordUnitTest(string testName, UnitTestStatus testStatus, ref DateTimeOffset now)
{
var splitName = testName.Split(new[] { ':' }, 2);
var groupName = splitName[0];
if (splitName.Length > 1)
{
testName = splitName[1];
}
var testResult = string.Join("\n", testLines);
testLines.Clear();
var prevRecorderTask = recorderTask;
var endTime = DateTimeOffset.Now;
var startTime = now;
now = endTime;
recorderTask = Task.Run(async() =>
{
await prevRecorderTask;
await recorder.RecordUnitTestAsync(groupName, testName, testStatus, testResult, startTime, endTime);
});
}
}
}
protected async Task DeactivateAttachedAsync(IOperationExecutionContext context, ContainerId containerId)
{
var containerManager = await context.TryGetServiceAsync <IContainerManager>();
await containerManager.DeactivateContainerAsync(containerId.Name, containerId.Tag, containerId.Source);
}
protected async Task AttachToBuildAsync(IOperationExecutionContext context, ContainerId containerId)
{
var containerManager = await context.TryGetServiceAsync <IContainerManager>();
await containerManager.AttachContainerToBuildAsync(containerId, context.CancellationToken);
}
