public ITestReporter Create(IFileBackedLog mainLog,
IReadableLog runLog,
ILogs logs,
ICrashSnapshotReporter crashReporter,
ISimpleListener simpleListener,
IResultParser parser,
AppBundleInformation appInformation,
RunMode runMode,
XmlResultJargon xmlJargon,
string?device,
TimeSpan timeout,
string?additionalLogsDirectory = null,
ExceptionLogger?exceptionLogger = null,
bool generateHtml = false) => new TestReporter(_processManager,
mainLog,
runLog,
logs,
crashReporter,
simpleListener,
parser,
appInformation,
runMode,
xmlJargon,
device,
timeout,
additionalLogsDirectory,
exceptionLogger,
generateHtml);
public ITestReporter Create(ILog mainLog,
ILog runLog,
ILogs logs,
ICrashSnapshotReporter crashReporter,
ISimpleListener simpleListener,
IResultParser parser,
AppBundleInformation appInformation,
RunMode runMode,
XmlResultJargon xmlJargon,
string device,
TimeSpan timeout,
string additionalLogsDirectory = null,
ExceptionLogger exceptionLogger = null)
{
return(new TestReporter(processManager,
mainLog,
runLog,
logs,
crashReporter,
simpleListener,
parser,
appInformation,
runMode,
xmlJargon,
device,
timeout,
additionalLogsDirectory,
exceptionLogger));
}
private async Task <ProcessExecutionResult> RunDeviceApp(
MlaunchArguments mlaunchArguments,
ICrashSnapshotReporter crashReporter,
string deviceName,
TimeSpan timeout,
CancellationToken cancellationToken)
{
var deviceSystemLog = _logs.Create($"device-{deviceName}-{_helpers.Timestamp}.log", LogType.SystemLog.ToString());
var deviceLogCapturer = _deviceLogCapturerFactory.Create(_mainLog, deviceSystemLog, deviceName);
deviceLogCapturer.StartCapture();
try
{
await crashReporter.StartCaptureAsync();
_mainLog.WriteLine("Starting the app");
return(await _processManager.ExecuteCommandAsync(
mlaunchArguments,
_mainLog,
timeout,
cancellationToken : cancellationToken));
}
finally
{
deviceLogCapturer.StopCapture();
deviceSystemLog.Dispose();
}
}
private async Task RunDeviceTests(
MlaunchArguments mlaunchArguments,
ICrashSnapshotReporter crashReporter,
ITestReporter testReporter,
ISimpleListener deviceListener,
string deviceName,
TimeSpan timeout,
CancellationToken cancellationToken)
{
var deviceSystemLog = _logs.Create($"device-{deviceName}-{_helpers.Timestamp}.log", "Device log");
var deviceLogCapturer = _deviceLogCapturerFactory.Create(_mainLog, deviceSystemLog, deviceName);
deviceLogCapturer.StartCapture();
try
{
await crashReporter.StartCaptureAsync();
// create a tunnel to communicate with the device
if (_listenerFactory.UseTunnel && deviceListener is SimpleTcpListener tcpListener)
{
// create a new tunnel using the listener
var tunnel = _listenerFactory.TunnelBore.Create(deviceName, _mainLog);
tunnel.Open(deviceName, tcpListener, timeout, _mainLog);
// wait until we started the tunnel
await tunnel.Started;
}
_mainLog.WriteLine("Starting test run");
// We need to check for MT1111 (which means that mlaunch won't wait for the app to exit).
var aggregatedLog = Log.CreateReadableAggregatedLog(_mainLog, testReporter.CallbackLog);
var result = _processManager.ExecuteCommandAsync(
mlaunchArguments,
aggregatedLog,
timeout,
cancellationToken: cancellationToken);
await testReporter.CollectDeviceResult(result);
}
finally
{
deviceLogCapturer.StopCapture();
deviceSystemLog.Dispose();
// close a tunnel if it was created
if (_listenerFactory.UseTunnel)
{
await _listenerFactory.TunnelBore.Close(deviceName);
}
}
// Upload the system log
if (File.Exists(deviceSystemLog.FullPath))
{
_mainLog.WriteLine("A capture of the device log is: {0}", deviceSystemLog.FullPath);
}
}
public TestReporter(IMlaunchProcessManager processManager,
IFileBackedLog mainLog,
IReadableLog runLog,
ILogs logs,
ICrashSnapshotReporter crashReporter,
ISimpleListener simpleListener,
IResultParser parser,
AppBundleInformation appInformation,
RunMode runMode,
XmlResultJargon xmlJargon,
string?device,
TimeSpan timeout,
string?additionalLogsDirectory = null,
ExceptionLogger?exceptionLogger = null,
bool generateHtml = false)
{
_processManager = processManager ?? throw new ArgumentNullException(nameof(processManager));
_deviceName = device; // can be null on simulators
_listener = simpleListener ?? throw new ArgumentNullException(nameof(simpleListener));
_mainLog = mainLog ?? throw new ArgumentNullException(nameof(mainLog));
_runLog = runLog ?? throw new ArgumentNullException(nameof(runLog));
_logs = logs ?? throw new ArgumentNullException(nameof(logs));
_crashReporter = crashReporter ?? throw new ArgumentNullException(nameof(crashReporter));
_crashLogs = new Logs(logs.Directory);
_resultParser = parser ?? throw new ArgumentNullException(nameof(parser));
_appInfo = appInformation ?? throw new ArgumentNullException(nameof(appInformation));
_runMode = runMode;
_xmlJargon = xmlJargon;
_timeout = timeout;
_additionalLogsDirectory = additionalLogsDirectory;
_exceptionLogger = exceptionLogger;
_timeoutWatch = Stopwatch.StartNew();
_generateHtml = generateHtml;
CallbackLog = new CallbackLog(line =>
{
// MT1111: Application launched successfully, but it's not possible to wait for the app to exit as
// requested because it's not possible to detect app termination when launching using gdbserver
_waitedForExit &= line?.Contains("MT1111: ") != true;
if (line?.Contains("error MT1007") == true)
{
_launchFailure = true;
}
});
}
public TestReporter(IProcessManager processManager,
ILog mainLog,
ILog runLog,
ILogs logs,
ICrashSnapshotReporter crashReporter,
ISimpleListener simpleListener,
IResultParser parser,
AppBundleInformation appInformation,
RunMode runMode,
XmlResultJargon xmlJargon,
string device,
TimeSpan timeout,
double launchTimeout,
string additionalLogsDirectory = null,
ExceptionLogger exceptionLogger = null)
{
this.processManager = processManager ?? throw new ArgumentNullException(nameof(processManager));
this.deviceName = device; // can be null on simulators
this.listener = simpleListener ?? throw new ArgumentNullException(nameof(simpleListener));
this.mainLog = mainLog ?? throw new ArgumentNullException(nameof(mainLog));
this.runLog = runLog ?? throw new ArgumentNullException(nameof(runLog));
this.logs = logs ?? throw new ArgumentNullException(nameof(logs));
this.crashReporter = crashReporter ?? throw new ArgumentNullException(nameof(crashReporter));
this.crashLogs = new Logs(logs.Directory);
this.resultParser = parser ?? throw new ArgumentNullException(nameof(parser));
this.appInfo = appInformation ?? throw new ArgumentNullException(nameof(appInformation));
this.runMode = runMode;
this.xmlJargon = xmlJargon;
this.timeout = timeout;
this.launchTimeout = launchTimeout;
this.additionalLogsDirectory = additionalLogsDirectory;
this.exceptionLogger = exceptionLogger;
this.timeoutWatch = Stopwatch.StartNew();
CallbackLog = new CallbackLog((line) => {
// MT1111: Application launched successfully, but it's not possible to wait for the app to exit as requested because it's not possible to detect app termination when launching using gdbserver
waitedForExit &= line?.Contains("MT1111: ") != true;
if (line?.Contains("error MT1007") == true)
{
launchFailure = true;
}
});
}
private async Task RunDeviceApp(
MlaunchArguments mlaunchArguments,
ICrashSnapshotReporter crashReporter,
string deviceName,
TimeSpan timeout,
CancellationToken cancellationToken)
{
var deviceSystemLog = _logs.Create($"device-{deviceName}-{_helpers.Timestamp}.log", LogType.SystemLog.ToString());
var deviceLogCapturer = _deviceLogCapturerFactory.Create(_mainLog, deviceSystemLog, deviceName);
deviceLogCapturer.StartCapture();
try
{
await crashReporter.StartCaptureAsync();
_mainLog.WriteLine("Starting test run");
await _processManager.ExecuteCommandAsync(
mlaunchArguments,
_mainLog,
timeout,
cancellationToken : cancellationToken);
}
finally
{
deviceLogCapturer.StopCapture();
deviceSystemLog.Dispose();
}
// Upload the system log
if (File.Exists(deviceSystemLog.FullPath))
{
_mainLog.WriteLine("A capture of the device log is: {0}", deviceSystemLog.FullPath);
}
}
public override async Task RunTestAsync()
{
var projectDir = System.IO.Path.GetDirectoryName(ProjectFile);
var name = System.IO.Path.GetFileName(projectDir);
if (string.Equals("mac", name, StringComparison.OrdinalIgnoreCase))
{
name = System.IO.Path.GetFileName(System.IO.Path.GetDirectoryName(projectDir));
}
var suffix = string.Empty;
switch (Platform)
{
case TestPlatform.Mac_Modern:
suffix = "-modern";
break;
case TestPlatform.Mac_Full:
suffix = "-full";
break;
case TestPlatform.Mac_System:
suffix = "-system";
break;
}
if (ProjectFile.EndsWith(".sln", StringComparison.Ordinal))
{
Path = System.IO.Path.Combine(System.IO.Path.GetDirectoryName(ProjectFile), "bin", BuildTask.ProjectPlatform, BuildTask.ProjectConfiguration + suffix, name + ".app", "Contents", "MacOS", name);
}
else
{
var project = new XmlDocument();
project.LoadWithoutNetworkAccess(ProjectFile);
string outputPath;
if (TestProject?.IsDotNetProject == true)
{
outputPath = await Harness.AppBundleLocator.LocateAppBundle(project, ProjectFile, TestTarget.None, BuildTask.ProjectConfiguration);
}
else
{
outputPath = project.GetOutputPath(BuildTask.ProjectPlatform, BuildTask.ProjectConfiguration).Replace('\\', '/');
}
var assemblyName = project.GetAssemblyName();
Path = System.IO.Path.Combine(System.IO.Path.GetDirectoryName(ProjectFile), outputPath, assemblyName + ".app", "Contents", "MacOS", assemblyName);
}
using (var resource = await NotifyAndAcquireDesktopResourceAsync()) {
using (var proc = new Process()) {
proc.StartInfo.FileName = Path;
var arguments = new List <string> ();
IFileBackedLog xmlLog = null;
var useXmlOutput = Harness.InCI || true;
if (IsUnitTest)
{
var extension = useXmlOutput ? "xml" : "log";
var type = useXmlOutput ? LogType.XmlLog : LogType.NUnitResult;
xmlLog = Logs.Create($"test-{Platform}-{Timestamp}.{extension}", type.ToString());
arguments.Add($"-transport:FILE");
proc.StartInfo.EnvironmentVariables ["NUNIT_TRANSPORT"] = "FILE";
arguments.Add($"--logfile:{xmlLog.FullPath}");
proc.StartInfo.EnvironmentVariables ["NUNIT_LOG_FILE"] = xmlLog.FullPath;
if (useXmlOutput)
{
arguments.Add("--enablexml");
proc.StartInfo.EnvironmentVariables ["NUNIT_ENABLE_XML_OUTPUT"] = "true";
arguments.Add("--xmlmode=wrapped");
proc.StartInfo.EnvironmentVariables ["NUNIT_ENABLE_XML_MODE"] = "wrapped";
arguments.Add("--xmlversion=nunitv3");
proc.StartInfo.EnvironmentVariables ["NUNIT_XML_VERSION"] = "nunitv3";
}
arguments.Add("--autostart");
proc.StartInfo.EnvironmentVariables ["NUNIT_AUTOSTART"] = "true";
arguments.Add("--autoexit");
proc.StartInfo.EnvironmentVariables ["NUNIT_AUTOEXIT"] = "true";
}
if (!Harness.GetIncludeSystemPermissionTests(Platform, false))
{
proc.StartInfo.EnvironmentVariables ["DISABLE_SYSTEM_PERMISSION_TESTS"] = "1";
}
proc.StartInfo.EnvironmentVariables ["MONO_DEBUG"] = "no-gdb-backtrace";
proc.StartInfo.EnvironmentVariables.Remove("DYLD_FALLBACK_LIBRARY_PATH"); // VSMac might set this, and the test may end up crashing
proc.StartInfo.Arguments = StringUtils.FormatArguments(arguments);
Jenkins.MainLog.WriteLine("Executing {0} ({1})", TestName, Mode);
var log = Logs.Create($"execute-{Platform}-{Timestamp}.txt", LogType.ExecutionLog.ToString());
ICrashSnapshotReporter snapshot = null;
if (!Jenkins.Harness.DryRun)
{
ExecutionResult = TestExecutingResult.Running;
snapshot = CrashReportSnapshotFactory.Create(log, Logs, isDevice: false, deviceName: null);
await snapshot.StartCaptureAsync();
ProcessExecutionResult result = null;
try {
var timeout = TimeSpan.FromMinutes(20);
result = await ProcessManager.RunAsync(proc, log, timeout);
if (result.TimedOut)
{
FailureMessage = $"Execution timed out after {timeout.TotalSeconds} seconds.";
log.WriteLine(FailureMessage);
ExecutionResult = TestExecutingResult.TimedOut;
}
else if (result.Succeeded)
{
ExecutionResult = TestExecutingResult.Succeeded;
}
else
{
ExecutionResult = TestExecutingResult.Failed;
FailureMessage = result.ExitCode != 1 ? $"Test run crashed (exit code: {result.ExitCode})." : "Test run failed.";
log.WriteLine(FailureMessage);
}
} finally {
await snapshot.EndCaptureAsync(TimeSpan.FromSeconds(Succeeded ? 0 : result?.ExitCode > 1 ? 120 : 5));
}
}
Jenkins.MainLog.WriteLine("Executed {0} ({1})", TestName, Mode);
if (IsUnitTest)
{
var reporterFactory = new TestReporterFactory(ProcessManager);
var listener = new Microsoft.DotNet.XHarness.iOS.Shared.Listeners.SimpleFileListener(xmlLog.FullPath, log, xmlLog, useXmlOutput);
var reporter = reporterFactory.Create(Harness.HarnessLog, log, Logs, snapshot, listener, Harness.ResultParser, new AppBundleInformation("N/A", "N/A", "N/A", "N/A", true, null), RunMode.MacOS, Harness.XmlJargon, "no device here", TimeSpan.Zero);
var rv = await reporter.ParseResult();
if (ExecutionResult == TestExecutingResult.Succeeded)
{
// The process might have crashed, timed out at exit, or otherwise returned a non-zero exit code when all the unit tests passed, in which we shouldn't override the execution result here,
ExecutionResult = rv.ExecutingResult;
}
// Set or replace the failure message, depending on whether there already is a failure message or not.
if (string.IsNullOrEmpty(FailureMessage))
{
FailureMessage = rv.ResultMessage;
}
else if (!string.IsNullOrEmpty(rv.ResultMessage))
{
FailureMessage += "\n" + rv.ResultMessage;
}
}
}
}
}
public override async Task RunTestAsync()
{
var projectDir = System.IO.Path.GetDirectoryName(ProjectFile);
var name = System.IO.Path.GetFileName(projectDir);
if (string.Equals("mac", name, StringComparison.OrdinalIgnoreCase))
{
name = System.IO.Path.GetFileName(System.IO.Path.GetDirectoryName(projectDir));
}
var suffix = string.Empty;
switch (Platform)
{
case TestPlatform.Mac_Modern:
suffix = "-modern";
break;
case TestPlatform.Mac_Full:
suffix = "-full";
break;
case TestPlatform.Mac_System:
suffix = "-system";
break;
}
if (ProjectFile.EndsWith(".sln", StringComparison.Ordinal))
{
Path = System.IO.Path.Combine(System.IO.Path.GetDirectoryName(ProjectFile), "bin", BuildTask.ProjectPlatform, BuildTask.ProjectConfiguration + suffix, name + ".app", "Contents", "MacOS", name);
}
else
{
var project = new XmlDocument();
project.LoadWithoutNetworkAccess(ProjectFile);
var outputPath = project.GetOutputPath(BuildTask.ProjectPlatform, BuildTask.ProjectConfiguration).Replace('\\', '/');
var assemblyName = project.GetAssemblyName();
Path = System.IO.Path.Combine(System.IO.Path.GetDirectoryName(ProjectFile), outputPath, assemblyName + ".app", "Contents", "MacOS", assemblyName);
}
using (var resource = await NotifyAndAcquireDesktopResourceAsync()) {
using (var proc = new Process()) {
proc.StartInfo.FileName = Path;
var arguments = new List <string> ();
IFileBackedLog xmlLog = null;
var useXmlOutput = Harness.InCI || true;
if (IsUnitTest)
{
var extension = useXmlOutput ? "xml" : "log";
var type = useXmlOutput ? LogType.XmlLog : LogType.NUnitResult;
xmlLog = Logs.Create($"test-{Platform}-{Timestamp}.{extension}", type.ToString());
arguments.Add($"-transport:FILE");
arguments.Add($"--logfile:{xmlLog.FullPath}");
if (useXmlOutput)
{
arguments.Add("--enablexml");
arguments.Add("--xmlmode=wrapped");
arguments.Add("--xmlversion=nunitv3");
}
}
if (!Harness.GetIncludeSystemPermissionTests(Platform, false))
{
proc.StartInfo.EnvironmentVariables ["DISABLE_SYSTEM_PERMISSION_TESTS"] = "1";
}
proc.StartInfo.EnvironmentVariables ["MONO_DEBUG"] = "no-gdb-backtrace";
proc.StartInfo.Arguments = StringUtils.FormatArguments(arguments);
Jenkins.MainLog.WriteLine("Executing {0} ({1})", TestName, Mode);
var log = Logs.Create($"execute-{Platform}-{Timestamp}.txt", LogType.ExecutionLog.ToString());
ICrashSnapshotReporter snapshot = null;
if (!Jenkins.Harness.DryRun)
{
ExecutionResult = TestExecutingResult.Running;
snapshot = CrashReportSnapshotFactory.Create(log, Logs, isDevice: false, deviceName: null);
await snapshot.StartCaptureAsync();
ProcessExecutionResult result = null;
try {
var timeout = TimeSpan.FromMinutes(20);
result = await ProcessManager.RunAsync(proc, log, timeout);
if (result.TimedOut)
{
FailureMessage = $"Execution timed out after {timeout.TotalSeconds} seconds.";
log.WriteLine(FailureMessage);
ExecutionResult = TestExecutingResult.TimedOut;
}
else if (result.Succeeded)
{
ExecutionResult = TestExecutingResult.Succeeded;
}
else
{
ExecutionResult = TestExecutingResult.Failed;
FailureMessage = result.ExitCode != 1 ? $"Test run crashed (exit code: {result.ExitCode})." : "Test run failed.";
log.WriteLine(FailureMessage);
}
} finally {
await snapshot.EndCaptureAsync(TimeSpan.FromSeconds(Succeeded ? 0 : result?.ExitCode > 1 ? 120 : 5));
}
}
Jenkins.MainLog.WriteLine("Executed {0} ({1})", TestName, Mode);
if (IsUnitTest)
{
var reporterFactory = new TestReporterFactory(ProcessManager);
var listener = new Microsoft.DotNet.XHarness.iOS.Shared.Listeners.SimpleFileListener(xmlLog.FullPath, log, xmlLog, useXmlOutput);
var reporter = reporterFactory.Create(Harness.HarnessLog, log, Logs, snapshot, listener, Harness.ResultParser, new AppBundleInformation("N/A", "N/A", "N/A", "N/A", true, null), RunMode.MacOS, Harness.XmlJargon, "no device here", TimeSpan.Zero);
var rv = await reporter.ParseResult();
ExecutionResult = rv.ExecutingResult;
FailureMessage = rv.ExecutingResult == TestExecutingResult.Succeeded ? null : rv.ResultMessage;
}
}
}
}
public async Task <(string DeviceName, TestExecutingResult Result, string ResultMessage)> RunApp(
AppBundleInformation appInformation,
TestTargetOs target,
TimeSpan timeout,
TimeSpan testLaunchTimeout,
string?deviceName = null,
string?companionDeviceName = null,
bool ensureCleanSimulatorState = false,
int verbosity = 1,
XmlResultJargon xmlResultJargon = XmlResultJargon.xUnit,
string[]?skippedMethods = null,
string[]?skippedTestClasses = null,
CancellationToken cancellationToken = default)
{
var runMode = target.Platform.ToRunMode();
bool isSimulator = target.Platform.IsSimulator();
var deviceListenerLog = _logs.Create($"test-{target.AsString()}-{_helpers.Timestamp}.log", LogType.TestLog.ToString(), timestamp: true);
var(deviceListenerTransport, deviceListener, deviceListenerTmpFile) = _listenerFactory.Create(
runMode,
log: _mainLog,
testLog: deviceListenerLog,
isSimulator: isSimulator,
autoExit: true,
xmlOutput: true); // cli always uses xml
ISimulatorDevice?simulator = null;
ISimulatorDevice?companionSimulator = null;
// Find devices
if (isSimulator)
{
int attempt = 1;
const int maxAttempts = 3;
while (true)
{
try
{
(simulator, companionSimulator) = await _simulatorLoader.FindSimulators(target, _mainLog);
break;
}
catch (Exception e)
{
_mainLog.WriteLine($"Failed to find/create simulator (attempt {attempt}/{maxAttempts}):" + Environment.NewLine + e);
if (attempt == maxAttempts)
{
throw new NoDeviceFoundException("Failed to find/create suitable simulator");
}
}
finally
{
attempt++;
}
}
deviceName = companionSimulator?.Name ?? simulator.Name;
}
else
{
deviceName ??= await FindDevice(target) ?? throw new NoDeviceFoundException();
}
int deviceListenerPort = deviceListener.InitializeAndGetPort();
deviceListener.StartAsync();
var crashLogs = new Logs(_logs.Directory);
ICrashSnapshotReporter crashReporter = _snapshotReporterFactory.Create(_mainLog, crashLogs, isDevice: !isSimulator, deviceName);
ITestReporter testReporter = _testReporterFactory.Create(_mainLog,
_mainLog,
_logs,
crashReporter,
deviceListener,
_resultParser,
appInformation,
runMode,
xmlResultJargon,
deviceName,
timeout,
null,
(level, message) => _mainLog.WriteLine(message));
deviceListener.ConnectedTask
.TimeoutAfter(testLaunchTimeout)
.ContinueWith(testReporter.LaunchCallback)
.DoNotAwait();
_mainLog.WriteLine($"*** Executing '{appInformation.AppName}' on {target} '{deviceName}' ***");
try
{
using var combinedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(testReporter.CancellationToken, cancellationToken);
if (isSimulator)
{
if (simulator == null)
{
_mainLog.WriteLine("Didn't find any suitable simulator");
throw new NoDeviceFoundException();
}
var mlaunchArguments = GetSimulatorArguments(
appInformation,
simulator,
verbosity,
xmlResultJargon,
skippedMethods,
skippedTestClasses,
deviceListenerTransport,
deviceListenerPort,
deviceListenerTmpFile);
await RunSimulatorTests(
mlaunchArguments,
appInformation,
crashReporter,
testReporter,
simulator,
companionSimulator,
ensureCleanSimulatorState,
timeout,
combinedCancellationToken.Token);
}
else
{
var mlaunchArguments = GetDeviceArguments(
appInformation,
deviceName,
target.Platform.IsWatchOSTarget(),
verbosity,
xmlResultJargon,
skippedMethods,
skippedTestClasses,
deviceListenerTransport,
deviceListenerPort,
deviceListenerTmpFile);
await RunDeviceTests(
mlaunchArguments,
crashReporter,
testReporter,
deviceListener,
deviceName,
timeout,
combinedCancellationToken.Token);
}
}
finally
{
deviceListener.Cancel();
deviceListener.Dispose();
}
// Check the final status, copy all the required data
var(testResult, resultMessage) = await testReporter.ParseResult();
return(deviceName, testResult, resultMessage);
}
private async Task RunSimulatorTests(
MlaunchArguments mlaunchArguments,
AppBundleInformation appInformation,
ICrashSnapshotReporter crashReporter,
ITestReporter testReporter,
ISimulatorDevice simulator,
ISimulatorDevice?companionSimulator,
bool ensureCleanSimulatorState,
TimeSpan timeout,
CancellationToken cancellationToken)
{
var systemLogs = new List <ICaptureLog>();
try
{
_mainLog.WriteLine("System log for the '{1}' simulator is: {0}", simulator.SystemLog, simulator.Name);
var simulatorLog = _captureLogFactory.Create(
path: Path.Combine(_logs.Directory, simulator.Name + ".log"),
systemLogPath: simulator.SystemLog,
entireFile: true,
LogType.SystemLog.ToString());
simulatorLog.StartCapture();
_logs.Add(simulatorLog);
systemLogs.Add(simulatorLog);
if (companionSimulator != null)
{
_mainLog.WriteLine("System log for the '{1}' companion simulator is: {0}", companionSimulator.SystemLog, companionSimulator.Name);
var companionLog = _captureLogFactory.Create(
path: Path.Combine(_logs.Directory, companionSimulator.Name + ".log"),
systemLogPath: companionSimulator.SystemLog,
entireFile: true,
LogType.CompanionSystemLog.ToString());
companionLog.StartCapture();
_logs.Add(companionLog);
systemLogs.Add(companionLog);
}
if (ensureCleanSimulatorState)
{
await simulator.PrepareSimulator(_mainLog, appInformation.BundleIdentifier);
if (companionSimulator != null)
{
await companionSimulator.PrepareSimulator(_mainLog, appInformation.BundleIdentifier);
}
}
await crashReporter.StartCaptureAsync();
_mainLog.WriteLine("Starting test run");
var result = _processManager.ExecuteCommandAsync(mlaunchArguments, _mainLog, timeout, cancellationToken: cancellationToken);
await testReporter.CollectSimulatorResult(result);
// cleanup after us
if (ensureCleanSimulatorState)
{
await simulator.KillEverything(_mainLog);
if (companionSimulator != null)
{
await companionSimulator.KillEverything(_mainLog);
}
}
}
finally
{
foreach (ICaptureLog?log in systemLogs)
{
log.StopCapture();
log.Dispose();
}
}
}
public async Task <(string DeviceName, int?exitCode)> RunApp(
AppBundleInformation appInformation,
TestTargetOs target,
TimeSpan timeout,
string?deviceName = null,
string?companionDeviceName = null,
bool ensureCleanSimulatorState = false,
int verbosity = 1,
CancellationToken cancellationToken = default)
{
bool isSimulator = target.Platform.IsSimulator();
ISimulatorDevice?simulator = null;
ISimulatorDevice?companionSimulator = null;
// Find devices
if (isSimulator)
{
(simulator, companionSimulator) = await _simulatorLoader.FindSimulators(target, _mainLog, 3);
deviceName = companionSimulator?.Name ?? simulator.Name;
}
else
{
deviceName ??= await FindDevice(target) ?? throw new NoDeviceFoundException();
}
var crashLogs = new Logs(_logs.Directory);
ICrashSnapshotReporter crashReporter = _snapshotReporterFactory.Create(_mainLog, crashLogs, isDevice: !isSimulator, deviceName);
_mainLog.WriteLine($"*** Executing '{appInformation.AppName}' on {target.AsString()} '{deviceName}' ***");
if (isSimulator)
{
if (simulator == null)
{
_mainLog.WriteLine("Didn't find any suitable simulator");
throw new NoDeviceFoundException();
}
var mlaunchArguments = GetSimulatorArguments(appInformation, simulator, verbosity);
await RunSimulatorApp(
mlaunchArguments,
appInformation,
crashReporter,
simulator,
companionSimulator,
ensureCleanSimulatorState,
timeout,
cancellationToken);
}
else
{
var mlaunchArguments = GetDeviceArguments(appInformation, deviceName, target.Platform.IsWatchOSTarget(), verbosity);
await RunDeviceApp(
mlaunchArguments,
crashReporter,
deviceName,
timeout,
cancellationToken);
}
var systemLog = _logs.FirstOrDefault(log => log.Description == LogType.SystemLog.ToString());
if (systemLog == null)
{
_mainLog.WriteLine("App run ended but failed to detect exit code (no system log found)");
return(deviceName, null);
}
var exitCode = _exitCodeDetector.DetectExitCode(appInformation, systemLog);
_mainLog.WriteLine($"App run ended with {exitCode}");
return(deviceName, exitCode);
}
/// <summary>
/// Runs the MacCatalyst app by executing its binary (or if not found, via `open -W path.to.app`).
/// </summary>
private async Task <(TestExecutingResult Result, string ResultMessage)> RunMacCatalystTests(
ListenerTransport deviceListenerTransport,
ISimpleListener deviceListener,
string deviceListenerTmpFile,
AppBundleInformation appInformation,
TimeSpan timeout,
TimeSpan testLaunchTimeout,
XmlResultJargon xmlResultJargon,
string[]?skippedMethods,
string[]?skippedTestClasses,
CancellationToken cancellationToken)
{
var deviceListenerPort = deviceListener.InitializeAndGetPort();
deviceListener.StartAsync();
var crashLogs = new Logs(_logs.Directory);
ICrashSnapshotReporter crashReporter = _snapshotReporterFactory.Create(_mainLog, crashLogs, isDevice: false, null);
ITestReporter testReporter = _testReporterFactory.Create(
_mainLog,
_mainLog,
_logs,
crashReporter,
deviceListener,
_resultParser,
appInformation,
RunMode.MacOS,
xmlResultJargon,
null,
timeout,
null,
(level, message) => _mainLog.WriteLine(message));
deviceListener.ConnectedTask
.TimeoutAfter(testLaunchTimeout)
.ContinueWith(testReporter.LaunchCallback)
.DoNotAwait();
_mainLog.WriteLine($"*** Executing '{appInformation.AppName}' on MacCatalyst ***");
try
{
using var combinedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(testReporter.CancellationToken, cancellationToken);
var envVariables = GetEnvVariables(
xmlResultJargon,
skippedMethods,
skippedTestClasses,
deviceListenerTransport,
deviceListenerPort,
deviceListenerTmpFile);
envVariables[EnviromentVariables.HostName] = "127.0.0.1";
var arguments = new List <string>
{
"-W",
appInformation.LaunchAppPath,
};
arguments.AddRange(_appArguments);
await crashReporter.StartCaptureAsync();
var result = await RunMacCatalystApp(appInformation, timeout, _appArguments, envVariables, combinedCancellationToken.Token);
await testReporter.CollectSimulatorResult(result);
}
finally
{
deviceListener.Cancel();
deviceListener.Dispose();
}
return(await testReporter.ParseResult());
}
public async Task <(string DeviceName, ProcessExecutionResult result)> RunApp(
AppBundleInformation appInformation,
TestTargetOs target,
TimeSpan timeout,
string?deviceName = null,
string?companionDeviceName = null,
bool ensureCleanSimulatorState = false,
int verbosity = 1,
CancellationToken cancellationToken = default)
{
var isSimulator = target.Platform.IsSimulator();
ProcessExecutionResult result;
ISimulatorDevice? simulator = null;
ISimulatorDevice? companionSimulator = null;
if (target.Platform == TestTarget.MacCatalyst)
{
_mainLog.WriteLine($"*** Executing '{appInformation.AppName}' on MacCatalyst ***");
result = await RunMacCatalystApp(appInformation, timeout, _appArguments, new Dictionary <string, object>(), cancellationToken);
return("MacCatalyst", result);
}
// Find devices
if (isSimulator)
{
(simulator, companionSimulator) = await _simulatorLoader.FindSimulators(target, _mainLog, 3);
deviceName = companionSimulator?.Name ?? simulator.Name;
}
else
{
deviceName ??= await FindDevice(target) ?? throw new NoDeviceFoundException();
}
var crashLogs = new Logs(_logs.Directory);
ICrashSnapshotReporter crashReporter = _snapshotReporterFactory.Create(
_mainLog,
crashLogs,
isDevice: !isSimulator,
deviceName);
_mainLog.WriteLine($"*** Executing '{appInformation.AppName}' on {target.AsString()} '{deviceName}' ***");
if (isSimulator)
{
if (simulator == null)
{
_mainLog.WriteLine("Didn't find any suitable simulator");
throw new NoDeviceFoundException();
}
var mlaunchArguments = GetSimulatorArguments(appInformation, simulator, verbosity);
result = await RunSimulatorApp(
mlaunchArguments,
appInformation,
crashReporter,
simulator,
companionSimulator,
ensureCleanSimulatorState,
timeout,
cancellationToken);
}
else
{
var mlaunchArguments = GetDeviceArguments(appInformation, deviceName, target.Platform.IsWatchOSTarget(), verbosity);
result = await RunDeviceApp(
mlaunchArguments,
crashReporter,
deviceName,
timeout,
cancellationToken);
}
return(deviceName, result);
}
protected async Task <ProcessExecutionResult> RunSimulatorApp(
AppBundleInformation appInformation,
MlaunchArguments mlaunchArguments,
ICrashSnapshotReporter crashReporter,
ISimulatorDevice simulator,
ISimulatorDevice?companionSimulator,
TimeSpan timeout,
bool waitForExit,
CancellationToken cancellationToken)
{
_mainLog.WriteLine("System log for the '{1}' simulator is: {0}", simulator.SystemLog, simulator.Name);
var simulatorLog = _captureLogFactory.Create(
path: Path.Combine(_logs.Directory, simulator.Name + ".log"),
systemLogPath: simulator.SystemLog,
entireFile: false,
LogType.SystemLog);
simulatorLog.StartCapture();
_logs.Add(simulatorLog);
var simulatorScanToken = await CaptureSimulatorLog(simulator, appInformation, cancellationToken);
using var systemLogs = new DisposableList <ICaptureLog>
{
simulatorLog
};
if (companionSimulator != null)
{
_mainLog.WriteLine("System log for the '{1}' companion simulator is: {0}", companionSimulator.SystemLog, companionSimulator.Name);
var companionLog = _captureLogFactory.Create(
path: Path.Combine(_logs.Directory, companionSimulator.Name + ".log"),
systemLogPath: companionSimulator.SystemLog,
entireFile: false,
LogType.CompanionSystemLog);
companionLog.StartCapture();
_logs.Add(companionLog);
systemLogs.Add(companionLog);
var companionScanToken = await CaptureSimulatorLog(companionSimulator, appInformation, cancellationToken);
if (companionScanToken != null)
{
simulatorScanToken = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, companionScanToken.Token);
}
}
await crashReporter.StartCaptureAsync();
_mainLog.WriteLine("Launching the app");
if (waitForExit)
{
var result = await _processManager.ExecuteCommandAsync(mlaunchArguments, _mainLog, timeout, cancellationToken : cancellationToken);
simulatorScanToken?.Cancel();
return(result);
}
TaskCompletionSource appLaunched = new();
var scanLog = new ScanLog($"Launched {appInformation.BundleIdentifier} with pid", () =>
{
_mainLog.WriteLine("App launch detected");
appLaunched.SetResult();
});
_mainLog.WriteLine("Waiting for the app to launch..");
var runTask = _processManager.ExecuteCommandAsync(mlaunchArguments, Log.CreateAggregatedLog(_mainLog, scanLog), timeout, cancellationToken: cancellationToken);
await Task.WhenAny(runTask, appLaunched.Task);
if (!appLaunched.Task.IsCompleted)
{
// In case the other task completes first, it is because one of these scenarios happened:
// - The app crashed and never launched
// - We missed the launch signal somehow and the app timed out
// - The app launched and quit immediately and race condition noticed that before the scan log did its job
// In all cases, we should return the result of the run task, it will be most likely 137 + Timeout (killed by us)
// If not, it will be a success because the app ran for a super short amount of time
_mainLog.WriteLine("App launch was not detected in time");
return(runTask.Result);
}
_mainLog.WriteLine("Not waiting for the app to exit");
return(new ProcessExecutionResult
{
ExitCode = 0
});
}
