public ITestReporter Create(ILog mainLog,
ILog runLog,
ILogs logs,
ICrashSnapshotReporter crashReporter,
ISimpleListener simpleListener,
IResultParser parser,
AppBundleInformation appInformation,
RunMode runMode,
XmlResultJargon xmlJargon,
string device,
TimeSpan timeout,
string additionalLogsDirectory = null,
ExceptionLogger exceptionLogger = null)
{
return(new TestReporter(processManager,
mainLog,
runLog,
logs,
crashReporter,
simpleListener,
parser,
appInformation,
runMode,
xmlJargon,
device,
timeout,
additionalLogsDirectory,
exceptionLogger));
}
public ITestReporter Create(IFileBackedLog mainLog,
IReadableLog runLog,
ILogs logs,
ICrashSnapshotReporter crashReporter,
ISimpleListener simpleListener,
IResultParser parser,
AppBundleInformation appInformation,
RunMode runMode,
XmlResultJargon xmlJargon,
string?device,
TimeSpan timeout,
string?additionalLogsDirectory = null,
ExceptionLogger?exceptionLogger = null,
bool generateHtml = false) => new TestReporter(_processManager,
mainLog,
runLog,
logs,
crashReporter,
simpleListener,
parser,
appInformation,
runMode,
xmlJargon,
device,
timeout,
additionalLogsDirectory,
exceptionLogger,
generateHtml);
public override string WriteResultsToFile(XmlResultJargon jargon)
{
if (_assembliesElement == null)
{
return(string.Empty);
}
// remove all the empty nodes
_assembliesElement.Descendants().Where(e => e.Name == "collection" && !e.Descendants().Any()).Remove();
string outputFilePath = GetResultsFilePath();
var settings = new XmlWriterSettings {
Indent = true
};
using (var xmlWriter = XmlWriter.Create(outputFilePath, settings))
{
switch (jargon)
{
case XmlResultJargon.TouchUnit:
case XmlResultJargon.NUnitV2:
Transform_Results("NUnitXml.xslt", _assembliesElement, xmlWriter);
break;
case XmlResultJargon.NUnitV3:
Transform_Results("NUnit3Xml.xslt", _assembliesElement, xmlWriter);
break;
default: // xunit as default, includes when we got Missing
_assembliesElement.Save(xmlWriter);
break;
}
}
return(outputFilePath);
}
private MlaunchArguments GetCommonArguments(
int verbosity,
XmlResultJargon xmlResultJargon,
string[]?skippedMethods,
string[]?skippedTestClasses,
ListenerTransport listenerTransport,
int listenerPort,
string listenerTmpFile)
{
var args = new MlaunchArguments();
for (var i = -1; i < verbosity; i++)
{
args.Add(new VerbosityArgument());
}
// Environment variables
var envVariables = GetEnvVariables(xmlResultJargon, skippedMethods, skippedTestClasses, listenerTransport, listenerPort, listenerTmpFile);
args.AddRange(envVariables.Select(pair => new SetEnvVariableArgument(pair.Key, pair.Value)));
// Arguments passed to the iOS app bundle
args.AddRange(_appArguments.Select(arg => new SetAppArgumentArgument(arg)));
return(args);
}
public void GenerateTestReport(StreamWriter writer, string resultsPath, XmlResultJargon xmlType)
{
using (var stream = new StreamReader(resultsPath))
using (var reader = XmlReader.Create(stream)) {
switch (xmlType)
{
case XmlResultJargon.NUnitV2:
case XmlResultJargon.TouchUnit:
GenerateNUnitV2TestReport(writer, reader);
break;
case XmlResultJargon.xUnit:
GeneratexUnitTestReport(writer, reader);
break;
case XmlResultJargon.NUnitV3:
GenerateNUnitV3TestReport(writer, reader);
break;
default:
writer.WriteLine($"<span style='padding-left: 15px;'>Could not parse {resultsPath}: Not supported format.</span><br />");
break;
}
}
}
public void IsValidXmlTest(XmlResultJargon jargon)
{
var path = CreateResultSample(jargon);
Assert.IsTrue(resultParser.IsValidXml(path, out var resultJargon), "is valid");
Assert.AreEqual(jargon, resultJargon, "jargon");
File.Delete(path);
}
public void GetXmlFilePathTest(string prefix, XmlResultJargon jargon)
{
var orignialPath = "/path/to/a/xml/result.xml";
var xmlPath = resultParser.GetXmlFilePath(orignialPath, jargon);
var fileName = Path.GetFileName(xmlPath);
StringAssert.StartsWith(prefix, fileName, "xml prefix");
}
public void CleanXmlPingTest(XmlResultJargon jargon)
{
var path = CreateResultSample(jargon, includePing: true);
var cleanPath = path + "_clean";
resultParser.CleanXml(path, cleanPath);
Assert.IsTrue(resultParser.IsValidXml(cleanPath, out var resultJargon), "is valid");
Assert.AreEqual(jargon, resultJargon, "jargon");
File.Delete(path);
File.Delete(cleanPath);
}
public RunDevice(IRunDeviceTask testTask,
IHardwareDeviceLoader devices,
IResourceManager resourceManager,
ILog mainLog,
ILog deviceLoadLog,
IErrorKnowledgeBase errorKnowledgeBase,
string defaultLogDirectory,
bool uninstallTestApp,
bool cleanSuccessfulTestRuns,
bool generateXmlFailures,
bool inCI,
bool useTcpTunnel,
XmlResultJargon xmlResultJargon)
{
this.testTask = testTask ?? throw new ArgumentNullException(nameof(testTask));
this.devices = devices ?? throw new ArgumentNullException(nameof(devices));
this.resourceManager = resourceManager ?? throw new ArgumentNullException(nameof(resourceManager));
this.mainLog = mainLog ?? throw new ArgumentNullException(nameof(mainLog));
this.deviceLoadLog = deviceLoadLog ?? throw new ArgumentNullException(nameof(deviceLoadLog));
this.errorKnowledgeBase = errorKnowledgeBase ?? throw new ArgumentNullException(nameof(errorKnowledgeBase));
this.uninstallTestApp = uninstallTestApp;
this.cleanSuccessfulTestRuns = cleanSuccessfulTestRuns;
this.generateXmlFailures = generateXmlFailures;
this.inCI = inCI;
this.useTcpTunnel = useTcpTunnel;
this.defaultLogDirectory = defaultLogDirectory ?? throw new ArgumentNullException(nameof(defaultLogDirectory)); // default should not be null
this.xmlResultJargon = xmlResultJargon;
switch (testTask.BuildTask.Platform)
{
case TestPlatform.iOS:
case TestPlatform.iOS_Unified:
case TestPlatform.iOS_Unified32:
case TestPlatform.iOS_Unified64:
testTask.AppRunnerTarget = TestTarget.Device_iOS;
break;
case TestPlatform.iOS_TodayExtension64:
testTask.AppRunnerTarget = TestTarget.Device_iOS;
break;
case TestPlatform.tvOS:
testTask.AppRunnerTarget = TestTarget.Device_tvOS;
break;
case TestPlatform.watchOS:
case TestPlatform.watchOS_32:
case TestPlatform.watchOS_64_32:
testTask.AppRunnerTarget = TestTarget.Device_watchOS;
break;
}
}
Task <ExitCode> IJustTestOrchestrator.OrchestrateTest(
string bundleIdentifier,
TestTargetOs target,
string?deviceName,
TimeSpan timeout,
TimeSpan launchTimeout,
CommunicationChannel communicationChannel,
XmlResultJargon xmlResultJargon,
IEnumerable <string> singleMethodFilters,
IEnumerable <string> classMethodFilters,
bool includeWirelessDevices,
bool enableLldb,
bool signalAppEnd,
IReadOnlyCollection <(string, string)> environmentalVariables,
public string GetXmlFilePath(string path, XmlResultJargon xmlType)
{
var fileName = Path.GetFileName(path);
switch (xmlType)
{
case XmlResultJargon.TouchUnit:
case XmlResultJargon.NUnitV2:
case XmlResultJargon.NUnitV3:
return path.Replace(fileName, $"nunit-{fileName}");
case XmlResultJargon.xUnit:
return path.Replace(fileName, $"xunit-{fileName}");
default:
return path;
}
}
public RunTest(IRunTestTask testTask,
IBuildToolTask buildTask,
IProcessManager processManager,
IEnvManager envManager,
ILog mainLog,
bool generateXmlFailures,
XmlResultJargon xmlResultJargon, bool dryRun)
{
this.testTask = testTask ?? throw new ArgumentNullException(nameof(testTask));
this.BuildTask = buildTask ?? throw new ArgumentNullException(nameof(buildTask));
this.ProcessManager = processManager ?? throw new ArgumentNullException(nameof(processManager));
this.envManager = envManager ?? throw new ArgumentNullException(nameof(envManager));
this.mainLog = mainLog ?? throw new ArgumentNullException(nameof(mainLog));
this.generateXmlFailures = generateXmlFailures;
this.dryRun = dryRun;
this.xmlResultJargon = xmlResultJargon;
}
public bool IsValidXml(string path, out XmlResultJargon type)
{
type = XmlResultJargon.Missing;
if (!File.Exists(path))
{
return(false);
}
using (var stream = File.OpenText(path)) {
string line;
while ((line = stream.ReadLine()) != null) // special case when get got the tcp connection
{
if (line.Contains("ping"))
{
continue;
}
if (line.Contains("test-run")) // first element of the NUnitV3 test collection
{
type = XmlResultJargon.NUnitV3;
return(true);
}
if (line.Contains("TouchUnitTestRun"))
{
type = XmlResultJargon.TouchUnit;
return(true);
}
if (line.Contains("test-results")) // first element of the NUnitV3 test collection
{
type = XmlResultJargon.NUnitV2;
return(true);
}
if (line.Contains("<assemblies>")) // first element of the xUnit test collection
{
type = XmlResultJargon.xUnit;
return(true);
}
if (line.Contains("<TestRun"))
{
type = XmlResultJargon.Trx;
return(true);
}
}
}
return(false);
}
public override void WriteResultsToFile(TextWriter writer, XmlResultJargon jargon)
{
if (_assembliesElement == null)
{
return;
}
// remove all the empty nodes
_assembliesElement.Descendants().Where(e => e.Name == "collection" && !e.Descendants().Any()).Remove();
var settings = new XmlWriterSettings {
Indent = true
};
using (var xmlWriter = XmlWriter.Create(writer, settings))
{
switch (jargon)
{
case XmlResultJargon.TouchUnit:
case XmlResultJargon.NUnitV2:
try
{
Transform_Results("NUnitXml.xslt", _assembliesElement, xmlWriter);
}
catch (Exception e)
{
writer.WriteLine(e);
}
break;
case XmlResultJargon.NUnitV3:
try
{
Transform_Results("NUnit3Xml.xslt", _assembliesElement, xmlWriter);
}
catch (Exception e)
{
writer.WriteLine(e);
}
break;
default: // xunit as default, includes when we got Missing
_assembliesElement.Save(xmlWriter);
break;
}
}
}
public TestReporter(IMlaunchProcessManager processManager,
IFileBackedLog mainLog,
IReadableLog runLog,
ILogs logs,
ICrashSnapshotReporter crashReporter,
ISimpleListener simpleListener,
IResultParser parser,
AppBundleInformation appInformation,
RunMode runMode,
XmlResultJargon xmlJargon,
string?device,
TimeSpan timeout,
string?additionalLogsDirectory = null,
ExceptionLogger?exceptionLogger = null,
bool generateHtml = false)
{
_processManager = processManager ?? throw new ArgumentNullException(nameof(processManager));
_deviceName = device; // can be null on simulators
_listener = simpleListener ?? throw new ArgumentNullException(nameof(simpleListener));
_mainLog = mainLog ?? throw new ArgumentNullException(nameof(mainLog));
_runLog = runLog ?? throw new ArgumentNullException(nameof(runLog));
_logs = logs ?? throw new ArgumentNullException(nameof(logs));
_crashReporter = crashReporter ?? throw new ArgumentNullException(nameof(crashReporter));
_crashLogs = new Logs(logs.Directory);
_resultParser = parser ?? throw new ArgumentNullException(nameof(parser));
_appInfo = appInformation ?? throw new ArgumentNullException(nameof(appInformation));
_runMode = runMode;
_xmlJargon = xmlJargon;
_timeout = timeout;
_additionalLogsDirectory = additionalLogsDirectory;
_exceptionLogger = exceptionLogger;
_timeoutWatch = Stopwatch.StartNew();
_generateHtml = generateHtml;
CallbackLog = new CallbackLog(line =>
{
// MT1111: Application launched successfully, but it's not possible to wait for the app to exit as
// requested because it's not possible to detect app termination when launching using gdbserver
_waitedForExit &= line?.Contains("MT1111: ") != true;
if (line?.Contains("error MT1007") == true)
{
_launchFailure = true;
}
});
}
private Dictionary <string, object> GetEnvVariables(
XmlResultJargon xmlResultJargon,
string[]?skippedMethods,
string[]?skippedTestClasses,
ListenerTransport listenerTransport,
int listenerPort,
string listenerTmpFile)
{
var variables = new Dictionary <string, object>
{
{ EnviromentVariables.AutoExit, true },
{ EnviromentVariables.HostPort, listenerPort },
// Let the runner know that we want to get an XML output and not plain text
{ EnviromentVariables.EnableXmlOutput, true },
{ EnviromentVariables.XmlVersion, $"{xmlResultJargon}" },
};
if (skippedMethods?.Any() ?? skippedTestClasses?.Any() ?? false)
{
// Do not run all the tests, we are using filters
variables.Add(EnviromentVariables.RunAllTestsByDefault, false);
// Add the skipped test classes and methods
if (skippedMethods != null && skippedMethods.Length > 0)
{
var skippedMethodsValue = string.Join(',', skippedMethods);
variables.Add(EnviromentVariables.SkippedMethods, skippedMethodsValue);
}
if (skippedTestClasses != null && skippedTestClasses !.Length > 0)
{
var skippedClassesValue = string.Join(',', skippedTestClasses);
variables.Add(EnviromentVariables.SkippedClasses, skippedClassesValue);
}
}
if (listenerTransport == ListenerTransport.File)
{
variables.Add(EnviromentVariables.LogFilePath, listenerTmpFile);
}
return(variables);
}
private MlaunchArguments GetSimulatorArguments(
AppBundleInformation appInformation,
ISimulatorDevice simulator,
int verbosity,
XmlResultJargon xmlResultJargon,
string[]?skippedMethods,
string[]?skippedTestClasses,
ListenerTransport deviceListenerTransport,
int deviceListenerPort,
string deviceListenerTmpFile)
{
var args = GetCommonArguments(
verbosity,
xmlResultJargon,
skippedMethods,
skippedTestClasses,
deviceListenerTransport,
deviceListenerPort,
deviceListenerTmpFile);
args.Add(new SetAppArgumentArgument("-hostname:127.0.0.1", true));
args.Add(new SetEnvVariableArgument(EnviromentVariables.HostName, "127.0.0.1"));
args.Add(new SimulatorUDIDArgument(simulator.UDID));
if (appInformation.Extension.HasValue)
{
switch (appInformation.Extension)
{
case Extension.TodayExtension:
args.Add(new LaunchSimulatorExtensionArgument(appInformation.LaunchAppPath, appInformation.BundleIdentifier));
break;
case Extension.WatchKit2:
default:
throw new NotImplementedException();
}
}
else
{
args.Add(new LaunchSimulatorArgument(appInformation.LaunchAppPath));
}
return(args);
}
public TestReporter(IProcessManager processManager,
ILog mainLog,
ILog runLog,
ILogs logs,
ICrashSnapshotReporter crashReporter,
ISimpleListener simpleListener,
IResultParser parser,
AppBundleInformation appInformation,
RunMode runMode,
XmlResultJargon xmlJargon,
string device,
TimeSpan timeout,
double launchTimeout,
string additionalLogsDirectory = null,
ExceptionLogger exceptionLogger = null)
{
this.processManager = processManager ?? throw new ArgumentNullException(nameof(processManager));
this.deviceName = device; // can be null on simulators
this.listener = simpleListener ?? throw new ArgumentNullException(nameof(simpleListener));
this.mainLog = mainLog ?? throw new ArgumentNullException(nameof(mainLog));
this.runLog = runLog ?? throw new ArgumentNullException(nameof(runLog));
this.logs = logs ?? throw new ArgumentNullException(nameof(logs));
this.crashReporter = crashReporter ?? throw new ArgumentNullException(nameof(crashReporter));
this.crashLogs = new Logs(logs.Directory);
this.resultParser = parser ?? throw new ArgumentNullException(nameof(parser));
this.appInfo = appInformation ?? throw new ArgumentNullException(nameof(appInformation));
this.runMode = runMode;
this.xmlJargon = xmlJargon;
this.timeout = timeout;
this.launchTimeout = launchTimeout;
this.additionalLogsDirectory = additionalLogsDirectory;
this.exceptionLogger = exceptionLogger;
this.timeoutWatch = Stopwatch.StartNew();
CallbackLog = new CallbackLog((line) => {
// MT1111: Application launched successfully, but it's not possible to wait for the app to exit as requested because it's not possible to detect app termination when launching using gdbserver
waitedForExit &= line?.Contains("MT1111: ") != true;
if (line?.Contains("error MT1007") == true)
{
launchFailure = true;
}
});
}
public (string resultLine, bool failed) ParseResults(string source, XmlResultJargon xmlType, string? humanReadableReportDestination = null)
{
StreamWriter? writer = null;
if (humanReadableReportDestination != null)
{
writer = new StreamWriter(humanReadableReportDestination, true);
}
var reader = new StreamReader(source);
var parsedData = xmlType switch
{
XmlResultJargon.TouchUnit => ParseTouchUnitXml(reader, writer),
XmlResultJargon.NUnitV2 => ParseNUnitXml(reader, writer),
XmlResultJargon.NUnitV3 => ParseNUnitV3Xml(reader, writer),
XmlResultJargon.xUnit => ParsexUnitXml(reader, writer),
_ => ("", true),
};
writer?.Dispose();
return parsedData;
}
private string CreateResultSample(XmlResultJargon jargon, bool includePing = false)
{
string sampleFileName = null;
switch (jargon)
{
case XmlResultJargon.NUnitV2:
sampleFileName = "NUnitV2Sample.xml";
break;
case XmlResultJargon.NUnitV3:
sampleFileName = "NUnitV3Sample.xml";
break;
case XmlResultJargon.TouchUnit:
sampleFileName = "TouchUnitSample.xml";
break;
case XmlResultJargon.xUnit:
sampleFileName = "xUnitSample.xml";
break;
}
Assert.NotNull(sampleFileName);
var name = GetType().Assembly.GetManifestResourceNames().Where(a => a.EndsWith(sampleFileName, StringComparison.Ordinal)).FirstOrDefault();
var tempPath = Path.GetTempFileName();
using (var outputStream = new StreamWriter(tempPath))
using (var sampleStream = new StreamReader(GetType().Assembly.GetManifestResourceStream(name)))
{
if (includePing)
{
outputStream.WriteLine("ping");
}
string line;
while ((line = sampleStream.ReadLine()) != null)
{
outputStream.WriteLine(line);
}
}
return(tempPath);
}
public (string resultLine, bool failed) GenerateHumanReadableResults(string source, string destination, XmlResultJargon xmlType)
{
(string resultLine, bool failed)parseData;
using (var reader = new StreamReader(source))
using (var writer = new StreamWriter(destination, true)) {
switch (xmlType)
{
case XmlResultJargon.TouchUnit:
parseData = ParseTouchUnitXml(reader, writer);
break;
case XmlResultJargon.NUnitV2:
parseData = ParseNUnitXml(reader, writer);
break;
case XmlResultJargon.NUnitV3:
parseData = ParseNUnitV3Xml(reader, writer);
break;
case XmlResultJargon.xUnit:
parseData = ParsexUnitXml(reader, writer);
break;
default:
parseData = ("", true);
break;
}
}
return(parseData);
}
public async Task<(string DeviceName, TestExecutingResult Result, string ResultMessage)> RunApp(
AppBundleInformation appInformation,
TestTarget target,
TimeSpan timeout,
TimeSpan testLaunchTimeout,
string? deviceName = null,
string? companionDeviceName = null,
bool ensureCleanSimulatorState = false,
int verbosity = 1,
XmlResultJargon xmlResultJargon = XmlResultJargon.xUnit,
CancellationToken cancellationToken = default)
{
var args = new MlaunchArguments
{
new SetAppArgumentArgument("-connection-mode"),
new SetAppArgumentArgument("none"), // This will prevent the app from trying to connect to any IDEs
new SetAppArgumentArgument("-autostart", true),
new SetEnvVariableArgument(EnviromentVariables.AutoStart, true),
new SetAppArgumentArgument("-autoexit", true),
new SetEnvVariableArgument(EnviromentVariables.AutoExit, true),
new SetAppArgumentArgument("-enablenetwork", true),
new SetEnvVariableArgument(EnviromentVariables.EnableNetwork, true),
// On macOS we can't edit the TCC database easily
// (it requires adding the mac has to be using MDM: https://carlashley.com/2018/09/28/tcc-round-up/)
// So by default ignore any tests that would pop up permission dialogs in CI.
new SetEnvVariableArgument(EnviromentVariables.DisableSystemPermissionTests, 1),
};
for (int i = -1; i < verbosity; i++)
{
args.Add(new VerbosityArgument());
}
var isSimulator = target.IsSimulator();
if (isSimulator)
{
args.Add(new SetAppArgumentArgument("-hostname:127.0.0.1", true));
args.Add(new SetEnvVariableArgument(EnviromentVariables.HostName, "127.0.0.1"));
}
else
{
var ipAddresses = System.Net.Dns.GetHostEntry(System.Net.Dns.GetHostName()).AddressList.Select(ip => ip.ToString());
var ips = string.Join(",", ipAddresses);
args.Add(new SetAppArgumentArgument($"-hostname:{ips}", true));
args.Add(new SetEnvVariableArgument(EnviromentVariables.HostName, ips));
}
var listenerLog = _logs.Create($"test-{target.AsString()}-{_helpers.Timestamp}.log", LogType.TestLog.ToString(), timestamp: true);
var (transport, listener, listenerTmpFile) = _listenerFactory.Create(target.ToRunMode(),
log: _mainLog,
testLog: listenerLog,
isSimulator: isSimulator,
autoExit: true,
xmlOutput: true); // cli always uses xml
// Initialize has to be called before we try to get Port (internal implementation of the listener says so)
// TODO: Improve this to not get into a broken state - it was really hard to debug when I moved this lower
listener.Initialize();
args.Add(new SetAppArgumentArgument($"-transport:{transport}", true));
args.Add(new SetEnvVariableArgument(EnviromentVariables.Transport, transport.ToString().ToUpper()));
if (transport == ListenerTransport.File)
{
args.Add(new SetEnvVariableArgument(EnviromentVariables.LogFilePath, listenerTmpFile));
}
args.Add(new SetAppArgumentArgument($"-hostport:{listener.Port}", true));
args.Add(new SetEnvVariableArgument(EnviromentVariables.HostPort, listener.Port));
if (_listenerFactory.UseTunnel && !isSimulator) // simulators do not support tunnels
{
args.Add(new SetEnvVariableArgument(EnviromentVariables.UseTcpTunnel, true));
}
if (_useXmlOutput)
{
// let the runner now via envars that we want to get a xml output, else the runner will default to plain text
args.Add (new SetEnvVariableArgument (EnviromentVariables.EnableXmlOutput, true));
args.Add (new SetEnvVariableArgument (EnviromentVariables.XmlMode, "wrapped"));
args.Add (new SetEnvVariableArgument (EnviromentVariables.XmlVersion, $"{xmlResultJargon}"));
}
listener.StartAsync();
var crashLogs = new Logs(_logs.Directory);
if (appInformation.Extension.HasValue)
{
switch (appInformation.Extension)
{
case Extension.TodayExtension:
args.Add(isSimulator
? (MlaunchArgument)new LaunchSimulatorExtensionArgument(appInformation.LaunchAppPath, appInformation.BundleIdentifier)
: new LaunchDeviceExtensionArgument(appInformation.LaunchAppPath, appInformation.BundleIdentifier));
break;
case Extension.WatchKit2:
default:
throw new NotImplementedException();
}
}
else
{
args.Add(isSimulator
? (MlaunchArgument)new LaunchSimulatorArgument(appInformation.LaunchAppPath)
: new LaunchDeviceArgument(appInformation.LaunchAppPath));
}
var runMode = target.ToRunMode();
ICrashSnapshotReporter crashReporter;
ITestReporter testReporter;
if (isSimulator)
{
crashReporter = _snapshotReporterFactory.Create(_mainLog, crashLogs, isDevice: !isSimulator, deviceName: null!);
testReporter = _testReporterFactory.Create(_mainLog,
_mainLog,
_logs,
crashReporter,
listener,
new XmlResultParser(),
appInformation,
runMode,
xmlResultJargon,
device: null,
timeout,
null,
(level, message) => _mainLog.WriteLine(message));
using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(testReporter.CancellationToken, cancellationToken);
listener.ConnectedTask
.TimeoutAfter(testLaunchTimeout)
.ContinueWith(testReporter.LaunchCallback)
.DoNotAwait();
await _simulatorLoader.LoadDevices(_logs.Create($"simulator-list-{_helpers.Timestamp}.log", "Simulator list"), false, false);
var simulators = await _simulatorLoader.FindSimulators(target, _mainLog);
if (!(simulators?.Any() ?? false))
{
_mainLog.WriteLine("Didn't find any suitable simulators");
throw new NoDeviceFoundException();
}
var simulator = string.IsNullOrEmpty(deviceName)
? simulators.FirstOrDefault()
: simulators.FirstOrDefault(s => string.Equals(s.Name, deviceName, StringComparison.InvariantCultureIgnoreCase));
if (simulator == null)
{
throw new NoDeviceFoundException();
}
deviceName = simulator.Name;
if (!target.IsWatchOSTarget())
{
var stderrTty = _helpers.GetTerminalName(2);
if (!string.IsNullOrEmpty(stderrTty))
{
args.Add(new SetStderrArgument(stderrTty));
}
else
{
var stdoutLog = _logs.CreateFile($"mlaunch-stdout-{_helpers.Timestamp}.log", "Standard output");
var stderrLog = _logs.CreateFile($"mlaunch-stderr-{_helpers.Timestamp}.log", "Standard error");
args.Add(new SetStdoutArgument(stdoutLog));
args.Add(new SetStderrArgument(stderrLog));
}
}
var systemLogs = new List<ICaptureLog>();
foreach (var sim in simulators)
{
// Upload the system log
_mainLog.WriteLine("System log for the '{1}' simulator is: {0}", sim.SystemLog, sim.Name);
bool isCompanion = sim != simulator;
var logDescription = isCompanion ? LogType.CompanionSystemLog.ToString() : LogType.SystemLog.ToString();
var log = _captureLogFactory.Create(
Path.Combine(_logs.Directory, sim.Name + ".log"),
sim.SystemLog,
true,
logDescription);
log.StartCapture();
_logs.Add(log);
systemLogs.Add(log);
}
_mainLog.WriteLine("*** Executing {0}/{1} in the simulator ***", appInformation.AppName, target);
if (ensureCleanSimulatorState)
{
foreach (var sim in simulators)
{
await sim.PrepareSimulator(_mainLog, appInformation.BundleIdentifier);
}
}
args.Add(new SimulatorUDIDArgument(simulator.UDID));
await crashReporter.StartCaptureAsync();
_mainLog.WriteLine("Starting test run");
var result = _processManager.ExecuteCommandAsync(args, _mainLog, timeout, cancellationToken: linkedCts.Token);
await testReporter.CollectSimulatorResult(result);
// cleanup after us
if (ensureCleanSimulatorState)
{
await simulator.KillEverything(_mainLog);
}
foreach (var log in systemLogs)
{
log.StopCapture();
}
}
else
{
args.Add(new DisableMemoryLimitsArgument());
if (deviceName == null)
{
IHardwareDevice? companionDevice = null;
IHardwareDevice device = await _hardwareDeviceLoader.FindDevice(runMode, _mainLog, includeLocked: false, force: false);
if (target.IsWatchOSTarget())
{
companionDevice = await _hardwareDeviceLoader.FindCompanionDevice(_mainLog, device);
}
deviceName = companionDevice?.Name ?? device.Name;
}
if (deviceName == null)
{
throw new NoDeviceFoundException();
}
crashReporter = _snapshotReporterFactory.Create(_mainLog, crashLogs, isDevice: !isSimulator, deviceName);
testReporter = _testReporterFactory.Create(_mainLog,
_mainLog,
_logs,
crashReporter,
listener,
new XmlResultParser(),
appInformation,
runMode,
xmlResultJargon,
deviceName,
timeout,
null,
(level, message) => _mainLog.WriteLine(message));
using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(testReporter.CancellationToken, cancellationToken);
listener.ConnectedTask
.TimeoutAfter(testLaunchTimeout)
.ContinueWith(testReporter.LaunchCallback)
.DoNotAwait();
_mainLog.WriteLine("*** Executing {0}/{1} on device '{2}' ***", appInformation.AppName, target, deviceName);
if (target.IsWatchOSTarget())
{
args.Add(new AttachNativeDebuggerArgument()); // this prevents the watch from backgrounding the app.
}
else
{
args.Add(new WaitForExitArgument());
}
args.Add(new DeviceNameArgument(deviceName));
var deviceSystemLog = _logs.Create($"device-{deviceName}-{_helpers.Timestamp}.log", "Device log");
var deviceLogCapturer = _deviceLogCapturerFactory.Create(_mainLog, deviceSystemLog, deviceName);
deviceLogCapturer.StartCapture();
try
{
await crashReporter.StartCaptureAsync();
// create a tunnel to communicate with the device
if (transport == ListenerTransport.Tcp && _listenerFactory.UseTunnel && listener is SimpleTcpListener tcpListener)
{
// create a new tunnel using the listener
var tunnel = _listenerFactory.TunnelBore.Create(deviceName, _mainLog);
tunnel.Open(deviceName, tcpListener, timeout, _mainLog);
// wait until we started the tunnel
await tunnel.Started;
}
_mainLog.WriteLine("Starting test run");
// We need to check for MT1111 (which means that mlaunch won't wait for the app to exit).
var aggregatedLog = Log.CreateAggregatedLog(testReporter.CallbackLog, _mainLog);
Task<ProcessExecutionResult> runTestTask = _processManager.ExecuteCommandAsync(
args,
aggregatedLog,
timeout,
cancellationToken: linkedCts.Token);
await testReporter.CollectDeviceResult(runTestTask);
}
finally
{
deviceLogCapturer.StopCapture();
deviceSystemLog.Dispose();
// close a tunnel if it was created
if (!isSimulator && _listenerFactory.UseTunnel)
await _listenerFactory.TunnelBore.Close(deviceName);
}
// Upload the system log
if (File.Exists(deviceSystemLog.FullPath))
{
_mainLog.WriteLine("A capture of the device log is: {0}", deviceSystemLog.FullPath);
}
}
listener.Cancel();
listener.Dispose();
// check the final status, copy all the required data
var (testResult, resultMessage) = await testReporter.ParseResult();
return (deviceName, testResult, resultMessage);
}
public static string ToXmlResultValue(this TestStatus status, XmlResultJargon jargon) => jargon switch
{
/// <summary>
/// Runs the MacCatalyst app by executing its binary (or if not found, via `open -W path.to.app`).
/// </summary>
private async Task <(TestExecutingResult Result, string ResultMessage)> RunMacCatalystTests(
ListenerTransport deviceListenerTransport,
ISimpleListener deviceListener,
string deviceListenerTmpFile,
AppBundleInformation appInformation,
TimeSpan timeout,
TimeSpan testLaunchTimeout,
XmlResultJargon xmlResultJargon,
string[]?skippedMethods,
string[]?skippedTestClasses,
CancellationToken cancellationToken)
{
var deviceListenerPort = deviceListener.InitializeAndGetPort();
deviceListener.StartAsync();
var crashLogs = new Logs(_logs.Directory);
ICrashSnapshotReporter crashReporter = _snapshotReporterFactory.Create(_mainLog, crashLogs, isDevice: false, null);
ITestReporter testReporter = _testReporterFactory.Create(
_mainLog,
_mainLog,
_logs,
crashReporter,
deviceListener,
_resultParser,
appInformation,
RunMode.MacOS,
xmlResultJargon,
null,
timeout,
null,
(level, message) => _mainLog.WriteLine(message));
deviceListener.ConnectedTask
.TimeoutAfter(testLaunchTimeout)
.ContinueWith(testReporter.LaunchCallback)
.DoNotAwait();
_mainLog.WriteLine($"*** Executing '{appInformation.AppName}' on MacCatalyst ***");
try
{
using var combinedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(testReporter.CancellationToken, cancellationToken);
var envVariables = GetEnvVariables(
xmlResultJargon,
skippedMethods,
skippedTestClasses,
deviceListenerTransport,
deviceListenerPort,
deviceListenerTmpFile);
envVariables[EnviromentVariables.HostName] = "127.0.0.1";
var arguments = new List <string>
{
"-W",
appInformation.LaunchAppPath,
};
arguments.AddRange(_appArguments);
await crashReporter.StartCaptureAsync();
var result = await RunMacCatalystApp(appInformation, timeout, _appArguments, envVariables, combinedCancellationToken.Token);
await testReporter.CollectSimulatorResult(result);
}
finally
{
deviceListener.Cancel();
deviceListener.Dispose();
}
return(await testReporter.ParseResult());
}
public void GenerateFailureTest(XmlResultJargon jargon)
{
var src = "test-case";
var appName = "MyUnitTest";
var variation = "Debug";
var title = "Testing";
var message = "This is a test";
var stderrMessage = "Something went very wrong";
var stderrPath = Path.GetTempFileName();
// write the message in the stderrParh that should be read
using (var writer = new StreamWriter(stderrPath)) {
writer.WriteLine(stderrMessage);
}
// create a path with data in it
var logs = new Mock <ILogs> ();
var tmpLogMock = new Mock <ILog> ();
var xmlLogMock = new Mock <ILog> ();
var tmpPath = Path.GetTempFileName();
var finalPath = Path.GetTempFileName();
// create a number of fake logs to be added to the failure
var logsDir = Path.GetTempFileName();
File.Delete(logsDir);
Directory.CreateDirectory(logsDir);
var failureLogs = new [] { "first.txt", "second.txt", "last.txt" };
foreach (var file in failureLogs)
{
var path = Path.Combine(logsDir, file);
File.WriteAllText(path, "");
}
// expect the creation of the two diff xml file logs
_ = logs.Setup(l => l.Create(It.IsAny <string> (), "Failure Log tmp", null)).Returns(tmpLogMock.Object);
_ = logs.Setup(l => l.Create(It.IsAny <string> (), LogType.XmlLog.ToString(), null)).Returns(xmlLogMock.Object);
if (jargon == XmlResultJargon.NUnitV3)
{
_ = logs.Setup(l => l.Directory).Returns(logsDir);
_ = tmpLogMock.Setup(tmpLog => tmpLog.FullPath).Returns(tmpPath);
}
// return the two temp files so that we can later validate that everything is present
_ = xmlLogMock.Setup(xmlLog => xmlLog.FullPath).Returns(finalPath);
resultParser.GenerateFailure(logs.Object, src, appName, variation, title, message, stderrPath, jargon);
// actual assertions do happen in the validation functions
ValidationMap [jargon] (src, appName, variation, title, message, stderrMessage, finalPath, failureLogs.Length);
// verify that we are correctly adding the logs
logs.Verify(l => l.Create(It.IsAny <string> (), It.IsAny <string> (), null), jargon == XmlResultJargon.NUnitV3 ? Times.AtMost(2) : Times.AtMostOnce());
if (jargon == XmlResultJargon.NUnitV3)
{
logs.Verify(l => l.Directory, Times.Once);
tmpLogMock.Verify(l => l.FullPath, Times.AtLeastOnce);
}
xmlLogMock.Verify(l => l.FullPath, Times.AtLeastOnce);
// clean files
File.Delete(stderrPath);
File.Delete(tmpPath);
File.Delete(finalPath);
Directory.Delete(logsDir, true);
}
public void IsExcludedAsAssembly(TestStatus status, XmlResultJargon jargon, string expectedResult) => Assert.Equal(status.ToXmlResultValue(jargon), expectedResult);
public async Task <(string DeviceName, TestExecutingResult Result, string ResultMessage)> RunApp(
AppBundleInformation appInformation,
TestTargetOs target,
TimeSpan timeout,
TimeSpan testLaunchTimeout,
string?deviceName = null,
string?companionDeviceName = null,
bool ensureCleanSimulatorState = false,
int verbosity = 1,
XmlResultJargon xmlResultJargon = XmlResultJargon.xUnit,
string[]?skippedMethods = null,
string[]?skippedTestClasses = null,
CancellationToken cancellationToken = default)
{
var runMode = target.Platform.ToRunMode();
bool isSimulator = target.Platform.IsSimulator();
var deviceListenerLog = _logs.Create($"test-{target.AsString()}-{_helpers.Timestamp}.log", LogType.TestLog.ToString(), timestamp: true);
var(deviceListenerTransport, deviceListener, deviceListenerTmpFile) = _listenerFactory.Create(
runMode,
log: _mainLog,
testLog: deviceListenerLog,
isSimulator: isSimulator,
autoExit: true,
xmlOutput: true); // cli always uses xml
ISimulatorDevice?simulator = null;
ISimulatorDevice?companionSimulator = null;
// Find devices
if (isSimulator)
{
int attempt = 1;
const int maxAttempts = 3;
while (true)
{
try
{
(simulator, companionSimulator) = await _simulatorLoader.FindSimulators(target, _mainLog);
break;
}
catch (Exception e)
{
_mainLog.WriteLine($"Failed to find/create simulator (attempt {attempt}/{maxAttempts}):" + Environment.NewLine + e);
if (attempt == maxAttempts)
{
throw new NoDeviceFoundException("Failed to find/create suitable simulator");
}
}
finally
{
attempt++;
}
}
deviceName = companionSimulator?.Name ?? simulator.Name;
}
else
{
deviceName ??= await FindDevice(target) ?? throw new NoDeviceFoundException();
}
int deviceListenerPort = deviceListener.InitializeAndGetPort();
deviceListener.StartAsync();
var crashLogs = new Logs(_logs.Directory);
ICrashSnapshotReporter crashReporter = _snapshotReporterFactory.Create(_mainLog, crashLogs, isDevice: !isSimulator, deviceName);
ITestReporter testReporter = _testReporterFactory.Create(_mainLog,
_mainLog,
_logs,
crashReporter,
deviceListener,
_resultParser,
appInformation,
runMode,
xmlResultJargon,
deviceName,
timeout,
null,
(level, message) => _mainLog.WriteLine(message));
deviceListener.ConnectedTask
.TimeoutAfter(testLaunchTimeout)
.ContinueWith(testReporter.LaunchCallback)
.DoNotAwait();
_mainLog.WriteLine($"*** Executing '{appInformation.AppName}' on {target} '{deviceName}' ***");
try
{
using var combinedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(testReporter.CancellationToken, cancellationToken);
if (isSimulator)
{
if (simulator == null)
{
_mainLog.WriteLine("Didn't find any suitable simulator");
throw new NoDeviceFoundException();
}
var mlaunchArguments = GetSimulatorArguments(
appInformation,
simulator,
verbosity,
xmlResultJargon,
skippedMethods,
skippedTestClasses,
deviceListenerTransport,
deviceListenerPort,
deviceListenerTmpFile);
await RunSimulatorTests(
mlaunchArguments,
appInformation,
crashReporter,
testReporter,
simulator,
companionSimulator,
ensureCleanSimulatorState,
timeout,
combinedCancellationToken.Token);
}
else
{
var mlaunchArguments = GetDeviceArguments(
appInformation,
deviceName,
target.Platform.IsWatchOSTarget(),
verbosity,
xmlResultJargon,
skippedMethods,
skippedTestClasses,
deviceListenerTransport,
deviceListenerPort,
deviceListenerTmpFile);
await RunDeviceTests(
mlaunchArguments,
crashReporter,
testReporter,
deviceListener,
deviceName,
timeout,
combinedCancellationToken.Token);
}
}
finally
{
deviceListener.Cancel();
deviceListener.Dispose();
}
// Check the final status, copy all the required data
var(testResult, resultMessage) = await testReporter.ParseResult();
return(deviceName, testResult, resultMessage);
}
private MlaunchArguments GetDeviceArguments(
AppBundleInformation appInformation,
string deviceName,
bool isWatchTarget,
int verbosity,
XmlResultJargon xmlResultJargon,
string[]?skippedMethods,
string[]?skippedTestClasses,
ListenerTransport deviceListenerTransport,
int deviceListenerPort,
string deviceListenerTmpFile)
{
var args = GetCommonArguments(
verbosity,
xmlResultJargon,
skippedMethods,
skippedTestClasses,
deviceListenerTransport,
deviceListenerPort,
deviceListenerTmpFile);
var ips = string.Join(",", _helpers.GetLocalIpAddresses().Select(ip => ip.ToString()));
args.Add(new SetAppArgumentArgument($"-hostname:{ips}", true));
args.Add(new SetEnvVariableArgument(EnviromentVariables.HostName, ips));
args.Add(new DisableMemoryLimitsArgument());
args.Add(new DeviceNameArgument(deviceName));
if (_listenerFactory.UseTunnel)
{
args.Add(new SetEnvVariableArgument(EnviromentVariables.UseTcpTunnel, true));
}
if (appInformation.Extension.HasValue)
{
switch (appInformation.Extension)
{
case Extension.TodayExtension:
args.Add(new LaunchDeviceExtensionArgument(appInformation.LaunchAppPath, appInformation.BundleIdentifier));
break;
case Extension.WatchKit2:
default:
throw new NotImplementedException();
}
}
else
{
args.Add(new LaunchDeviceArgument(appInformation.LaunchAppPath));
}
if (isWatchTarget)
{
args.Add(new AttachNativeDebuggerArgument()); // this prevents the watch from backgrounding the app.
}
else
{
args.Add(new WaitForExitArgument());
}
return(args);
}
private MlaunchArguments GetCommonArguments(
int verbosity,
XmlResultJargon xmlResultJargon,
string[]?skippedMethods,
string[]?skippedTestClasses,
ListenerTransport listenerTransport,
int listenerPort,
string listenerTmpFile)
{
var args = new MlaunchArguments
{
new SetAppArgumentArgument("-connection-mode"),
new SetAppArgumentArgument("none"), // This will prevent the app from trying to connect to any IDEs
new SetAppArgumentArgument("-autostart", true),
new SetEnvVariableArgument(EnviromentVariables.AutoStart, true),
new SetAppArgumentArgument("-autoexit", true),
new SetEnvVariableArgument(EnviromentVariables.AutoExit, true),
new SetAppArgumentArgument("-enablenetwork", true),
new SetEnvVariableArgument(EnviromentVariables.EnableNetwork, true),
// On macOS we can't edit the TCC database easily
// (it requires adding the mac has to be using MDM: https://carlashley.com/2018/09/28/tcc-round-up/)
// So by default ignore any tests that would pop up permission dialogs in CI.
new SetEnvVariableArgument(EnviromentVariables.DisableSystemPermissionTests, 1),
};
if (skippedMethods?.Any() ?? skippedTestClasses?.Any() ?? false)
{
// do not run all the tests, we are using filters
args.Add(new SetEnvVariableArgument(EnviromentVariables.RunAllTestsByDefault, false));
// add the skipped test classes and methods
if (skippedMethods != null && skippedMethods.Length > 0)
{
var skippedMethodsValue = string.Join(',', skippedMethods);
args.Add(new SetEnvVariableArgument(EnviromentVariables.SkippedMethods, skippedMethodsValue));
}
if (skippedTestClasses != null && skippedTestClasses !.Length > 0)
{
var skippedClassesValue = string.Join(',', skippedTestClasses);
args.Add(new SetEnvVariableArgument(EnviromentVariables.SkippedClasses, skippedClassesValue));
}
}
for (int i = -1; i < verbosity; i++)
{
args.Add(new VerbosityArgument());
}
// let the runner now via envars that we want to get a xml output, else the runner will default to plain text
args.Add(new SetEnvVariableArgument(EnviromentVariables.EnableXmlOutput, true));
args.Add(new SetEnvVariableArgument(EnviromentVariables.XmlMode, "wrapped"));
args.Add(new SetEnvVariableArgument(EnviromentVariables.XmlVersion, $"{xmlResultJargon}"));
args.Add(new SetAppArgumentArgument($"-transport:{listenerTransport}", true));
args.Add(new SetEnvVariableArgument(EnviromentVariables.Transport, listenerTransport.ToString().ToUpper()));
if (listenerTransport == ListenerTransport.File)
{
args.Add(new SetEnvVariableArgument(EnviromentVariables.LogFilePath, listenerTmpFile));
}
args.Add(new SetAppArgumentArgument($"-hostport:{listenerPort}", true));
args.Add(new SetEnvVariableArgument(EnviromentVariables.HostPort, listenerPort));
// Arguments passed to the iOS app bundle
args.AddRange(_appArguments.Select(arg => new SetAppArgumentArgument(arg, true)));
return(args);
}
public static OutputWriter GetWriter(this XmlResultJargon jargon) => jargon switch
{
