private async Task <HashSet <string> > CreateCrashReportsSnapshotAsync()
{
var crashes = new HashSet <string>();
if (!_isDevice)
{
var dir = Path.Combine(Environment.GetEnvironmentVariable("HOME"), "Library", "Logs", "DiagnosticReports");
if (Directory.Exists(dir))
{
crashes.UnionWith(Directory.EnumerateFiles(dir));
}
}
else
{
var tempFile = _tempFileProvider();
try
{
var args = new MlaunchArguments(new ListCrashReportsArgument(tempFile));
if (!string.IsNullOrEmpty(_deviceName))
{
args.Add(new DeviceNameArgument(_deviceName));
}
var result = await _processManager.ExecuteCommandAsync(args, _log, TimeSpan.FromMinutes(1));
if (result.Succeeded)
{
crashes.UnionWith(File.ReadAllLines(tempFile));
}
}
finally
{
File.Delete(tempFile);
}
}
return(crashes);
}
[InlineData(true)] // timeoout
public void LoadAsyncProcessErrorTest(bool timeout)
{
string processPath = null;
MlaunchArguments passedArguments = null;
// moq It.Is is not working as nicelly as we would like it, we capture data and use asserts
_processManager.Setup(p => p.RunAsync(It.IsAny <Process>(), It.IsAny <MlaunchArguments>(), It.IsAny <ILog>(), It.IsAny <TimeSpan?>(), It.IsAny <Dictionary <string, string> >(), It.IsAny <int>(), It.IsAny <CancellationToken?>(), It.IsAny <bool?>()))
.Returns <Process, MlaunchArguments, ILog, TimeSpan?, Dictionary <string, string>, int, CancellationToken?, bool?>((p, args, log, t, env, verbosity, token, d) =>
{
// we are going set the used args to validate them later, will always return an error from this method
processPath = p.StartInfo.FileName;
passedArguments = args;
if (!timeout)
{
return(Task.FromResult(new ProcessExecutionResult {
ExitCode = 1, TimedOut = false
}));
}
else
{
return(Task.FromResult(new ProcessExecutionResult {
ExitCode = 0, TimedOut = true
}));
}
});
Assert.ThrowsAsync <Exception>(async() =>
{
await _devices.LoadDevices(_executionLog.Object);
});
MlaunchArgument listDevArg = passedArguments.Where(a => a is ListDevicesArgument).FirstOrDefault();
Assert.NotNull(listDevArg);
MlaunchArgument outputFormatArg = passedArguments.Where(a => a is XmlOutputFormatArgument).FirstOrDefault();
Assert.NotNull(outputFormatArg);
}
private MlaunchArguments GetCommonArguments(int verbosity)
{
var args = new MlaunchArguments
{
new SetAppArgumentArgument("-connection-mode"),
new SetAppArgumentArgument("none"), // This will prevent the app from trying to connect to any IDEs
// On macOS we can't edit the TCC database easily
// (it requires adding the mac has to be using MDM: https://carlashley.com/2018/09/28/tcc-round-up/)
// So by default ignore any tests that would pop up permission dialogs in CI.
new SetEnvVariableArgument(EnviromentVariables.DisableSystemPermissionTests, 1),
};
for (int i = -1; i < verbosity; i++)
{
args.Add(new VerbosityArgument());
}
// Arguments passed to the iOS app bundle
args.AddRange(_appArguments.Select(arg => new SetAppArgumentArgument(arg, true)));
return(args);
}
public void MlaunchArgumentAndProcessManagerTest()
{
var oldArgs = new List <string>()
{
"--download-crash-report-to=/path/with spaces.txt",
"--sdkroot",
"/path to xcode/spaces",
"--devname",
"Premek's iPhone",
};
var newArgs = new MlaunchArguments()
{
new DownloadCrashReportToArgument("/path/with spaces.txt"),
new SdkRootArgument("/path to xcode/spaces"),
new DeviceNameArgument("Premek's iPhone"),
};
var oldWayOfPassingArgs = StringUtils.FormatArguments(oldArgs);
var newWayOfPassingArgs = newArgs.AsCommandLine();
Assert.Equal(oldWayOfPassingArgs, newWayOfPassingArgs);
}
public async Task <bool> Boot(ILog log, CancellationToken cancellationToken)
{
if (State == DeviceState.Booted)
{
log.WriteLine($"Simulator '{Name}' is already booted");
return(true);
}
log.WriteLine($"Booting simulator '{Name}'");
var args = new MlaunchArguments
{
new SimulatorUDIDArgument(this),
new LaunchSimulatorArgument(),
};
var watch = Stopwatch.StartNew();
var result = await _processManager.ExecuteCommandAsync(
args,
log,
TimeSpan.FromSeconds(30),
verbosity : 2,
cancellationToken : cancellationToken);
if (!result.Succeeded)
{
log.WriteLine($"Failed to boot the simulator '{Name}'");
return(false);
}
log.WriteLine($"Simulator '{Name}' booted in {(int)watch.Elapsed.TotalSeconds} seconds");
State = DeviceState.Booted;
return(true);
}
private async Task RunDeviceApp(
MlaunchArguments mlaunchArguments,
ICrashSnapshotReporter crashReporter,
string deviceName,
TimeSpan timeout,
CancellationToken cancellationToken)
{
var deviceSystemLog = _logs.Create($"device-{deviceName}-{_helpers.Timestamp}.log", LogType.SystemLog.ToString());
var deviceLogCapturer = _deviceLogCapturerFactory.Create(_mainLog, deviceSystemLog, deviceName);
deviceLogCapturer.StartCapture();
try
{
await crashReporter.StartCaptureAsync();
_mainLog.WriteLine("Starting test run");
await _processManager.ExecuteCommandAsync(
mlaunchArguments,
_mainLog,
timeout,
cancellationToken : cancellationToken);
}
finally
{
deviceLogCapturer.StopCapture();
deviceSystemLog.Dispose();
}
// Upload the system log
if (File.Exists(deviceSystemLog.FullPath))
{
_mainLog.WriteLine("A capture of the device log is: {0}", deviceSystemLog.FullPath);
}
}
public async Task FindAsyncDoNotCreateTest(TestTarget target, int expected)
{
string processPath = null;
MlaunchArguments passedArguments = null;
_processManager
.Setup(h => h.ExecuteXcodeCommandAsync("simctl", It.Is <string[]>(args => args[0] == "create"), _executionLog.Object, TimeSpan.FromMinutes(1)))
.ReturnsAsync(new ProcessExecutionResult()
{
ExitCode = 0
});
// moq It.Is is not working as nicelly as we would like it, we capture data and use asserts
_processManager
.Setup(p => p.RunAsync(It.IsAny <Process>(), It.IsAny <MlaunchArguments>(), It.IsAny <ILog>(), It.IsAny <TimeSpan?>(), It.IsAny <Dictionary <string, string> >(), It.IsAny <CancellationToken?>(), It.IsAny <bool?>()))
.Returns <Process, MlaunchArguments, ILog, TimeSpan?, Dictionary <string, string>, CancellationToken?, bool?>((p, args, log, t, env, token, d) =>
{
processPath = p.StartInfo.FileName;
passedArguments = args;
// we get the temp file that was passed as the args, and write our sample xml, which will be parsed to get the devices :)
var tempPath = args.Where(a => a is ListSimulatorsArgument).First().AsCommandLineArgument();
tempPath = tempPath.Substring(tempPath.IndexOf('=') + 1).Replace("\"", string.Empty);
CopySampleData(tempPath);
return(Task.FromResult(new ProcessExecutionResult {
ExitCode = 0, TimedOut = false
}));
});
await _simulators.LoadDevices(_executionLog.Object);
var sims = await _simulators.FindSimulators(target, _executionLog.Object, false, false);
Assert.Equal(expected, sims.Count());
}
public async Task LoadDevices(ILog log, bool includeLocked = false, bool forceRefresh = false, bool listExtraData = false)
{
await semaphore.WaitAsync();
if (loaded)
{
if (!forceRefresh)
{
semaphore.Release();
return;
}
supported_runtimes.Reset();
supported_device_types.Reset();
available_devices.Reset();
available_device_pairs.Reset();
}
loaded = true;
await Task.Run(async() =>
{
var tmpfile = Path.GetTempFileName();
try
{
using (var process = new Process())
{
var arguments = new MlaunchArguments(
new ListSimulatorsArgument(tmpfile),
new XmlOutputFormatArgument());
var task = processManager.RunAsync(process, arguments, log, timeout: TimeSpan.FromSeconds(30));
log.WriteLine("Launching {0} {1}", process.StartInfo.FileName, process.StartInfo.Arguments);
var result = await task;
if (!result.Succeeded)
{
throw new Exception("Failed to list simulators.");
}
log.WriteLine("Result:");
log.WriteLine(File.ReadAllText(tmpfile));
var simulator_data = new XmlDocument();
simulator_data.LoadWithoutNetworkAccess(tmpfile);
foreach (XmlNode sim in simulator_data.SelectNodes("/MTouch/Simulator/SupportedRuntimes/SimRuntime"))
{
supported_runtimes.Add(new SimRuntime()
{
Name = sim.SelectSingleNode("Name").InnerText,
Identifier = sim.SelectSingleNode("Identifier").InnerText,
Version = long.Parse(sim.SelectSingleNode("Version").InnerText),
});
}
foreach (XmlNode sim in simulator_data.SelectNodes("/MTouch/Simulator/SupportedDeviceTypes/SimDeviceType"))
{
supported_device_types.Add(new SimDeviceType()
{
Name = sim.SelectSingleNode("Name").InnerText,
Identifier = sim.SelectSingleNode("Identifier").InnerText,
ProductFamilyId = sim.SelectSingleNode("ProductFamilyId").InnerText,
MinRuntimeVersion = long.Parse(sim.SelectSingleNode("MinRuntimeVersion").InnerText),
MaxRuntimeVersion = long.Parse(sim.SelectSingleNode("MaxRuntimeVersion").InnerText),
Supports64Bits = bool.Parse(sim.SelectSingleNode("Supports64Bits").InnerText),
});
}
foreach (XmlNode sim in simulator_data.SelectNodes("/MTouch/Simulator/AvailableDevices/SimDevice"))
{
available_devices.Add(new SimulatorDevice(processManager, new TCCDatabase(processManager))
{
Name = sim.Attributes["Name"].Value,
UDID = sim.Attributes["UDID"].Value,
SimRuntime = sim.SelectSingleNode("SimRuntime").InnerText,
SimDeviceType = sim.SelectSingleNode("SimDeviceType").InnerText,
DataPath = sim.SelectSingleNode("DataPath").InnerText,
LogPath = sim.SelectSingleNode("LogPath").InnerText,
});
}
var sim_device_pairs = simulator_data.
SelectNodes("/MTouch/Simulator/AvailableDevicePairs/SimDevicePair").
Cast <XmlNode>().
// There can be duplicates, so remove those.
Distinct(new SimulatorXmlNodeComparer());
foreach (XmlNode sim in sim_device_pairs)
{
available_device_pairs.Add(new SimDevicePair()
{
UDID = sim.Attributes["UDID"].Value,
Companion = sim.SelectSingleNode("Companion").InnerText,
Gizmo = sim.SelectSingleNode("Gizmo").InnerText,
});
}
}
}
finally
{
supported_runtimes.SetCompleted();
supported_device_types.SetCompleted();
available_devices.SetCompleted();
available_device_pairs.SetCompleted();
File.Delete(tmpfile);
semaphore.Release();
}
});
}
public async Task LoadDevices(ILog log, bool includeLocked = false, bool forceRefresh = false, bool listExtraData = false)
{
if (loaded)
{
if (!forceRefresh)
{
return;
}
connectedDevices.Reset();
}
loaded = true;
var tmpfile = Path.GetTempFileName();
try {
using (var process = new Process()) {
var arguments = new MlaunchArguments(
new ListDevicesArgument(tmpfile),
new XmlOutputFormatArgument());
if (listExtraData)
{
arguments.Add(new ListExtraDataArgument());
}
var task = processManager.RunAsync(process, arguments, log, timeout: TimeSpan.FromSeconds(120));
log.WriteLine("Launching {0} {1}", process.StartInfo.FileName, process.StartInfo.Arguments);
var result = await task;
if (!result.Succeeded)
{
throw new Exception("Failed to list devices.");
}
log.WriteLine("Result:");
log.WriteLine(File.ReadAllText(tmpfile));
log.Flush();
var doc = new XmlDocument();
doc.LoadWithoutNetworkAccess(tmpfile);
foreach (XmlNode dev in doc.SelectNodes("/MTouch/Device"))
{
var d = GetDevice(dev);
if (d == null)
{
continue;
}
if (!includeLocked && d.IsLocked)
{
log.WriteLine($"Skipping device {d.Name} ({d.DeviceIdentifier}) because it's locked.");
continue;
}
if (d.IsUsableForDebugging.HasValue && !d.IsUsableForDebugging.Value)
{
log.WriteLine($"Skipping device {d.Name} ({d.DeviceIdentifier}) because it's not usable for debugging.");
continue;
}
connectedDevices.Add(d);
}
}
} finally {
connectedDevices.SetCompleted();
File.Delete(tmpfile);
log.Flush();
}
}
public async Task<(string DeviceName, TestExecutingResult Result, string ResultMessage)> RunApp(
AppBundleInformation appInformation,
TestTarget target,
TimeSpan timeout,
TimeSpan testLaunchTimeout,
string? deviceName = null,
string? companionDeviceName = null,
bool ensureCleanSimulatorState = false,
int verbosity = 1,
XmlResultJargon xmlResultJargon = XmlResultJargon.xUnit,
CancellationToken cancellationToken = default)
{
var args = new MlaunchArguments
{
new SetAppArgumentArgument("-connection-mode"),
new SetAppArgumentArgument("none"), // This will prevent the app from trying to connect to any IDEs
new SetAppArgumentArgument("-autostart", true),
new SetEnvVariableArgument(EnviromentVariables.AutoStart, true),
new SetAppArgumentArgument("-autoexit", true),
new SetEnvVariableArgument(EnviromentVariables.AutoExit, true),
new SetAppArgumentArgument("-enablenetwork", true),
new SetEnvVariableArgument(EnviromentVariables.EnableNetwork, true),
// On macOS we can't edit the TCC database easily
// (it requires adding the mac has to be using MDM: https://carlashley.com/2018/09/28/tcc-round-up/)
// So by default ignore any tests that would pop up permission dialogs in CI.
new SetEnvVariableArgument(EnviromentVariables.DisableSystemPermissionTests, 1),
};
for (int i = -1; i < verbosity; i++)
{
args.Add(new VerbosityArgument());
}
var isSimulator = target.IsSimulator();
if (isSimulator)
{
args.Add(new SetAppArgumentArgument("-hostname:127.0.0.1", true));
args.Add(new SetEnvVariableArgument(EnviromentVariables.HostName, "127.0.0.1"));
}
else
{
var ipAddresses = System.Net.Dns.GetHostEntry(System.Net.Dns.GetHostName()).AddressList.Select(ip => ip.ToString());
var ips = string.Join(",", ipAddresses);
args.Add(new SetAppArgumentArgument($"-hostname:{ips}", true));
args.Add(new SetEnvVariableArgument(EnviromentVariables.HostName, ips));
}
var listenerLog = _logs.Create($"test-{target.AsString()}-{_helpers.Timestamp}.log", LogType.TestLog.ToString(), timestamp: true);
var (transport, listener, listenerTmpFile) = _listenerFactory.Create(target.ToRunMode(),
log: _mainLog,
testLog: listenerLog,
isSimulator: isSimulator,
autoExit: true,
xmlOutput: true); // cli always uses xml
// Initialize has to be called before we try to get Port (internal implementation of the listener says so)
// TODO: Improve this to not get into a broken state - it was really hard to debug when I moved this lower
listener.Initialize();
args.Add(new SetAppArgumentArgument($"-transport:{transport}", true));
args.Add(new SetEnvVariableArgument(EnviromentVariables.Transport, transport.ToString().ToUpper()));
if (transport == ListenerTransport.File)
{
args.Add(new SetEnvVariableArgument(EnviromentVariables.LogFilePath, listenerTmpFile));
}
args.Add(new SetAppArgumentArgument($"-hostport:{listener.Port}", true));
args.Add(new SetEnvVariableArgument(EnviromentVariables.HostPort, listener.Port));
if (_listenerFactory.UseTunnel && !isSimulator) // simulators do not support tunnels
{
args.Add(new SetEnvVariableArgument(EnviromentVariables.UseTcpTunnel, true));
}
if (_useXmlOutput)
{
// let the runner now via envars that we want to get a xml output, else the runner will default to plain text
args.Add (new SetEnvVariableArgument (EnviromentVariables.EnableXmlOutput, true));
args.Add (new SetEnvVariableArgument (EnviromentVariables.XmlMode, "wrapped"));
args.Add (new SetEnvVariableArgument (EnviromentVariables.XmlVersion, $"{xmlResultJargon}"));
}
listener.StartAsync();
var crashLogs = new Logs(_logs.Directory);
if (appInformation.Extension.HasValue)
{
switch (appInformation.Extension)
{
case Extension.TodayExtension:
args.Add(isSimulator
? (MlaunchArgument)new LaunchSimulatorExtensionArgument(appInformation.LaunchAppPath, appInformation.BundleIdentifier)
: new LaunchDeviceExtensionArgument(appInformation.LaunchAppPath, appInformation.BundleIdentifier));
break;
case Extension.WatchKit2:
default:
throw new NotImplementedException();
}
}
else
{
args.Add(isSimulator
? (MlaunchArgument)new LaunchSimulatorArgument(appInformation.LaunchAppPath)
: new LaunchDeviceArgument(appInformation.LaunchAppPath));
}
var runMode = target.ToRunMode();
ICrashSnapshotReporter crashReporter;
ITestReporter testReporter;
if (isSimulator)
{
crashReporter = _snapshotReporterFactory.Create(_mainLog, crashLogs, isDevice: !isSimulator, deviceName: null!);
testReporter = _testReporterFactory.Create(_mainLog,
_mainLog,
_logs,
crashReporter,
listener,
new XmlResultParser(),
appInformation,
runMode,
xmlResultJargon,
device: null,
timeout,
null,
(level, message) => _mainLog.WriteLine(message));
using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(testReporter.CancellationToken, cancellationToken);
listener.ConnectedTask
.TimeoutAfter(testLaunchTimeout)
.ContinueWith(testReporter.LaunchCallback)
.DoNotAwait();
await _simulatorLoader.LoadDevices(_logs.Create($"simulator-list-{_helpers.Timestamp}.log", "Simulator list"), false, false);
var simulators = await _simulatorLoader.FindSimulators(target, _mainLog);
if (!(simulators?.Any() ?? false))
{
_mainLog.WriteLine("Didn't find any suitable simulators");
throw new NoDeviceFoundException();
}
var simulator = string.IsNullOrEmpty(deviceName)
? simulators.FirstOrDefault()
: simulators.FirstOrDefault(s => string.Equals(s.Name, deviceName, StringComparison.InvariantCultureIgnoreCase));
if (simulator == null)
{
throw new NoDeviceFoundException();
}
deviceName = simulator.Name;
if (!target.IsWatchOSTarget())
{
var stderrTty = _helpers.GetTerminalName(2);
if (!string.IsNullOrEmpty(stderrTty))
{
args.Add(new SetStderrArgument(stderrTty));
}
else
{
var stdoutLog = _logs.CreateFile($"mlaunch-stdout-{_helpers.Timestamp}.log", "Standard output");
var stderrLog = _logs.CreateFile($"mlaunch-stderr-{_helpers.Timestamp}.log", "Standard error");
args.Add(new SetStdoutArgument(stdoutLog));
args.Add(new SetStderrArgument(stderrLog));
}
}
var systemLogs = new List<ICaptureLog>();
foreach (var sim in simulators)
{
// Upload the system log
_mainLog.WriteLine("System log for the '{1}' simulator is: {0}", sim.SystemLog, sim.Name);
bool isCompanion = sim != simulator;
var logDescription = isCompanion ? LogType.CompanionSystemLog.ToString() : LogType.SystemLog.ToString();
var log = _captureLogFactory.Create(
Path.Combine(_logs.Directory, sim.Name + ".log"),
sim.SystemLog,
true,
logDescription);
log.StartCapture();
_logs.Add(log);
systemLogs.Add(log);
}
_mainLog.WriteLine("*** Executing {0}/{1} in the simulator ***", appInformation.AppName, target);
if (ensureCleanSimulatorState)
{
foreach (var sim in simulators)
{
await sim.PrepareSimulator(_mainLog, appInformation.BundleIdentifier);
}
}
args.Add(new SimulatorUDIDArgument(simulator.UDID));
await crashReporter.StartCaptureAsync();
_mainLog.WriteLine("Starting test run");
var result = _processManager.ExecuteCommandAsync(args, _mainLog, timeout, cancellationToken: linkedCts.Token);
await testReporter.CollectSimulatorResult(result);
// cleanup after us
if (ensureCleanSimulatorState)
{
await simulator.KillEverything(_mainLog);
}
foreach (var log in systemLogs)
{
log.StopCapture();
}
}
else
{
args.Add(new DisableMemoryLimitsArgument());
if (deviceName == null)
{
IHardwareDevice? companionDevice = null;
IHardwareDevice device = await _hardwareDeviceLoader.FindDevice(runMode, _mainLog, includeLocked: false, force: false);
if (target.IsWatchOSTarget())
{
companionDevice = await _hardwareDeviceLoader.FindCompanionDevice(_mainLog, device);
}
deviceName = companionDevice?.Name ?? device.Name;
}
if (deviceName == null)
{
throw new NoDeviceFoundException();
}
crashReporter = _snapshotReporterFactory.Create(_mainLog, crashLogs, isDevice: !isSimulator, deviceName);
testReporter = _testReporterFactory.Create(_mainLog,
_mainLog,
_logs,
crashReporter,
listener,
new XmlResultParser(),
appInformation,
runMode,
xmlResultJargon,
deviceName,
timeout,
null,
(level, message) => _mainLog.WriteLine(message));
using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(testReporter.CancellationToken, cancellationToken);
listener.ConnectedTask
.TimeoutAfter(testLaunchTimeout)
.ContinueWith(testReporter.LaunchCallback)
.DoNotAwait();
_mainLog.WriteLine("*** Executing {0}/{1} on device '{2}' ***", appInformation.AppName, target, deviceName);
if (target.IsWatchOSTarget())
{
args.Add(new AttachNativeDebuggerArgument()); // this prevents the watch from backgrounding the app.
}
else
{
args.Add(new WaitForExitArgument());
}
args.Add(new DeviceNameArgument(deviceName));
var deviceSystemLog = _logs.Create($"device-{deviceName}-{_helpers.Timestamp}.log", "Device log");
var deviceLogCapturer = _deviceLogCapturerFactory.Create(_mainLog, deviceSystemLog, deviceName);
deviceLogCapturer.StartCapture();
try
{
await crashReporter.StartCaptureAsync();
// create a tunnel to communicate with the device
if (transport == ListenerTransport.Tcp && _listenerFactory.UseTunnel && listener is SimpleTcpListener tcpListener)
{
// create a new tunnel using the listener
var tunnel = _listenerFactory.TunnelBore.Create(deviceName, _mainLog);
tunnel.Open(deviceName, tcpListener, timeout, _mainLog);
// wait until we started the tunnel
await tunnel.Started;
}
_mainLog.WriteLine("Starting test run");
// We need to check for MT1111 (which means that mlaunch won't wait for the app to exit).
var aggregatedLog = Log.CreateAggregatedLog(testReporter.CallbackLog, _mainLog);
Task<ProcessExecutionResult> runTestTask = _processManager.ExecuteCommandAsync(
args,
aggregatedLog,
timeout,
cancellationToken: linkedCts.Token);
await testReporter.CollectDeviceResult(runTestTask);
}
finally
{
deviceLogCapturer.StopCapture();
deviceSystemLog.Dispose();
// close a tunnel if it was created
if (!isSimulator && _listenerFactory.UseTunnel)
await _listenerFactory.TunnelBore.Close(deviceName);
}
// Upload the system log
if (File.Exists(deviceSystemLog.FullPath))
{
_mainLog.WriteLine("A capture of the device log is: {0}", deviceSystemLog.FullPath);
}
}
listener.Cancel();
listener.Dispose();
// check the final status, copy all the required data
var (testResult, resultMessage) = await testReporter.ParseResult();
return (deviceName, testResult, resultMessage);
}
private MlaunchArguments GetCommonArguments(
int verbosity,
XmlResultJargon xmlResultJargon,
string[]?skippedMethods,
string[]?skippedTestClasses,
ListenerTransport listenerTransport,
int listenerPort,
string listenerTmpFile)
{
var args = new MlaunchArguments
{
new SetAppArgumentArgument("-connection-mode"),
new SetAppArgumentArgument("none"), // This will prevent the app from trying to connect to any IDEs
new SetAppArgumentArgument("-autostart", true),
new SetEnvVariableArgument(EnviromentVariables.AutoStart, true),
new SetAppArgumentArgument("-autoexit", true),
new SetEnvVariableArgument(EnviromentVariables.AutoExit, true),
new SetAppArgumentArgument("-enablenetwork", true),
new SetEnvVariableArgument(EnviromentVariables.EnableNetwork, true),
// On macOS we can't edit the TCC database easily
// (it requires adding the mac has to be using MDM: https://carlashley.com/2018/09/28/tcc-round-up/)
// So by default ignore any tests that would pop up permission dialogs in CI.
new SetEnvVariableArgument(EnviromentVariables.DisableSystemPermissionTests, 1),
};
if (skippedMethods?.Any() ?? skippedTestClasses?.Any() ?? false)
{
// do not run all the tests, we are using filters
args.Add(new SetEnvVariableArgument(EnviromentVariables.RunAllTestsByDefault, false));
// add the skipped test classes and methods
if (skippedMethods != null && skippedMethods.Length > 0)
{
var skippedMethodsValue = string.Join(',', skippedMethods);
args.Add(new SetEnvVariableArgument(EnviromentVariables.SkippedMethods, skippedMethodsValue));
}
if (skippedTestClasses != null && skippedTestClasses !.Length > 0)
{
var skippedClassesValue = string.Join(',', skippedTestClasses);
args.Add(new SetEnvVariableArgument(EnviromentVariables.SkippedClasses, skippedClassesValue));
}
}
for (int i = -1; i < verbosity; i++)
{
args.Add(new VerbosityArgument());
}
// let the runner now via envars that we want to get a xml output, else the runner will default to plain text
args.Add(new SetEnvVariableArgument(EnviromentVariables.EnableXmlOutput, true));
args.Add(new SetEnvVariableArgument(EnviromentVariables.XmlMode, "wrapped"));
args.Add(new SetEnvVariableArgument(EnviromentVariables.XmlVersion, $"{xmlResultJargon}"));
args.Add(new SetAppArgumentArgument($"-transport:{listenerTransport}", true));
args.Add(new SetEnvVariableArgument(EnviromentVariables.Transport, listenerTransport.ToString().ToUpper()));
if (listenerTransport == ListenerTransport.File)
{
args.Add(new SetEnvVariableArgument(EnviromentVariables.LogFilePath, listenerTmpFile));
}
args.Add(new SetAppArgumentArgument($"-hostport:{listenerPort}", true));
args.Add(new SetEnvVariableArgument(EnviromentVariables.HostPort, listenerPort));
// Arguments passed to the iOS app bundle
args.AddRange(_appArguments.Select(arg => new SetAppArgumentArgument(arg, true)));
return(args);
}
private async Task RunSimulatorTests(
MlaunchArguments mlaunchArguments,
AppBundleInformation appInformation,
ICrashSnapshotReporter crashReporter,
ITestReporter testReporter,
ISimulatorDevice simulator,
ISimulatorDevice?companionSimulator,
bool ensureCleanSimulatorState,
TimeSpan timeout,
CancellationToken cancellationToken)
{
var systemLogs = new List <ICaptureLog>();
try
{
_mainLog.WriteLine("System log for the '{1}' simulator is: {0}", simulator.SystemLog, simulator.Name);
var simulatorLog = _captureLogFactory.Create(
path: Path.Combine(_logs.Directory, simulator.Name + ".log"),
systemLogPath: simulator.SystemLog,
entireFile: true,
LogType.SystemLog.ToString());
simulatorLog.StartCapture();
_logs.Add(simulatorLog);
systemLogs.Add(simulatorLog);
if (companionSimulator != null)
{
_mainLog.WriteLine("System log for the '{1}' companion simulator is: {0}", companionSimulator.SystemLog, companionSimulator.Name);
var companionLog = _captureLogFactory.Create(
path: Path.Combine(_logs.Directory, companionSimulator.Name + ".log"),
systemLogPath: companionSimulator.SystemLog,
entireFile: true,
LogType.CompanionSystemLog.ToString());
companionLog.StartCapture();
_logs.Add(companionLog);
systemLogs.Add(companionLog);
}
if (ensureCleanSimulatorState)
{
await simulator.PrepareSimulator(_mainLog, appInformation.BundleIdentifier);
if (companionSimulator != null)
{
await companionSimulator.PrepareSimulator(_mainLog, appInformation.BundleIdentifier);
}
}
await crashReporter.StartCaptureAsync();
_mainLog.WriteLine("Starting test run");
var result = _processManager.ExecuteCommandAsync(mlaunchArguments, _mainLog, timeout, cancellationToken: cancellationToken);
await testReporter.CollectSimulatorResult(result);
// cleanup after us
if (ensureCleanSimulatorState)
{
await simulator.KillEverything(_mainLog);
if (companionSimulator != null)
{
await companionSimulator.KillEverything(_mainLog);
}
}
}
finally
{
foreach (ICaptureLog?log in systemLogs)
{
log.StopCapture();
log.Dispose();
}
}
}
public async Task LoadDevices(ILog log, bool includeLocked = false, bool forceRefresh = false, bool listExtraData = false)
{
await _semaphore.WaitAsync();
if (_loaded)
{
if (!forceRefresh)
{
_semaphore.Release();
return;
}
_supportedRuntimes.Reset();
_supportedDeviceTypes.Reset();
_availableDevices.Reset();
_availableDevicePairs.Reset();
}
var tmpfile = Path.GetTempFileName();
try
{
var arguments = new MlaunchArguments(
new ListSimulatorsArgument(tmpfile),
new XmlOutputFormatArgument());
var result = await _processManager.ExecuteCommandAsync(arguments, log, timeout : TimeSpan.FromSeconds(30));
if (!result.Succeeded)
{
// mlaunch can sometimes return 0 but hang and timeout. It still outputs returns valid content to the tmp file
log.WriteLine($"mlaunch failed when listing simulators but trying to parse the results anyway");
}
var fileInfo = new FileInfo(tmpfile);
if (!fileInfo.Exists || fileInfo.Length == 0)
{
throw new Exception($"Failed to list simulators - no XML with devices found. " +
$"mlaunch {(result.TimedOut ? "timed out" : "exited")} with {result.ExitCode})");
}
var xmlContent = File.ReadAllText(tmpfile);
log.WriteLine("Simulator listing returned:" + Environment.NewLine + xmlContent);
var simulatorData = new XmlDocument();
simulatorData.LoadWithoutNetworkAccess(tmpfile);
foreach (XmlNode?sim in simulatorData.SelectNodes("/MTouch/Simulator/SupportedRuntimes/SimRuntime"))
{
if (sim == null)
{
continue;
}
_supportedRuntimes.Add(new SimRuntime(
name: sim.SelectSingleNode("Name").InnerText,
identifier: sim.SelectSingleNode("Identifier").InnerText,
version: long.Parse(sim.SelectSingleNode("Version").InnerText)));
}
foreach (XmlNode?sim in simulatorData.SelectNodes("/MTouch/Simulator/SupportedDeviceTypes/SimDeviceType"))
{
if (sim == null)
{
continue;
}
_supportedDeviceTypes.Add(new SimDeviceType(
name: sim.SelectSingleNode("Name").InnerText,
identifier: sim.SelectSingleNode("Identifier").InnerText,
productFamilyId: sim.SelectSingleNode("ProductFamilyId").InnerText,
minRuntimeVersion: long.Parse(sim.SelectSingleNode("MinRuntimeVersion").InnerText),
maxRuntimeVersion: long.Parse(sim.SelectSingleNode("MaxRuntimeVersion").InnerText),
supports64Bits: bool.Parse(sim.SelectSingleNode("Supports64Bits").InnerText)));
}
foreach (XmlNode?sim in simulatorData.SelectNodes("/MTouch/Simulator/AvailableDevices/SimDevice"))
{
if (sim == null)
{
continue;
}
_availableDevices.Add(new SimulatorDevice(_processManager, new TCCDatabase(_processManager))
{
Name = sim.Attributes["Name"].Value,
UDID = sim.Attributes["UDID"].Value,
SimRuntime = sim.SelectSingleNode("SimRuntime").InnerText,
SimDeviceType = sim.SelectSingleNode("SimDeviceType").InnerText,
DataPath = sim.SelectSingleNode("DataPath").InnerText,
LogPath = sim.SelectSingleNode("LogPath").InnerText,
});
}
var sim_device_pairs = simulatorData.
SelectNodes("/MTouch/Simulator/AvailableDevicePairs/SimDevicePair").
Cast <XmlNode>().
// There can be duplicates, so remove those.
Distinct(new SimulatorXmlNodeComparer());
foreach (XmlNode sim in sim_device_pairs)
{
_availableDevicePairs.Add(new SimDevicePair(
uDID: sim.Attributes["UDID"].Value,
companion: sim.SelectSingleNode("Companion").InnerText,
gizmo: sim.SelectSingleNode("Gizmo").InnerText));
}
}
finally
{
_loaded = true;
_supportedRuntimes.SetCompleted();
_supportedDeviceTypes.SetCompleted();
_availableDevices.SetCompleted();
_availableDevicePairs.SetCompleted();
File.Delete(tmpfile);
_semaphore.Release();
}
}
public async Task <(string deviceName, ProcessExecutionResult result)> InstallApp(AppBundleInformation appBundleInformation, TestTargetOs target, string?deviceName = null, CancellationToken cancellationToken = default)
{
if (target.Platform.IsSimulator())
{
// We reset the simulator when running, so a separate install step does not make much sense.
throw new InvalidOperationException("Installing to a simulator is not supported.");
}
if (!Directory.Exists(appBundleInformation.LaunchAppPath))
{
throw new DirectoryNotFoundException("Failed to find the app bundle directory");
}
if (deviceName == null)
{
// the _deviceLoader.FindDevice will return the fist device of the type, but we want to make sure that
// the device we use is if the correct arch, therefore, we will use the LoadDevices and return the
// correct one
await _deviceLoader.LoadDevices(_mainLog, false, false);
IHardwareDevice?device = null;
if (appBundleInformation.Supports32Bit)
{
// we only support 32b on iOS, therefore we can ignore the target
device = _deviceLoader.Connected32BitIOS.FirstOrDefault();
}
else
{
device = target.Platform switch
{
TestTarget.Device_iOS => _deviceLoader.Connected64BitIOS.FirstOrDefault(),
TestTarget.Device_tvOS => _deviceLoader.ConnectedTV.FirstOrDefault(),
_ => device
};
}
deviceName = target.Platform.IsWatchOSTarget() ? (await _deviceLoader.FindCompanionDevice(_mainLog, device)).Name : device?.Name;
}
if (deviceName == null)
{
throw new NoDeviceFoundException();
}
var args = new MlaunchArguments();
for (int i = -1; i < _verbosity; i++)
{
args.Add(new VerbosityArgument());
}
args.Add(new InstallAppOnDeviceArgument(appBundleInformation.LaunchAppPath));
args.Add(new DeviceNameArgument(deviceName));
if (target.Platform.IsWatchOSTarget())
{
args.Add(new DeviceArgument("ios,watchos"));
}
var totalSize = Directory.GetFiles(appBundleInformation.LaunchAppPath, "*", SearchOption.AllDirectories).Select((v) => new FileInfo(v).Length).Sum();
_mainLog.WriteLine($"Installing '{appBundleInformation.LaunchAppPath}' to '{deviceName}' ({totalSize / 1024.0 / 1024.0:N2} MB)");
ProcessExecutionResult result = await _processManager.ExecuteCommandAsync(args, _mainLog, TimeSpan.FromHours(1), cancellationToken : cancellationToken);
return(deviceName, result);
}
}
protected async Task <ProcessExecutionResult> RunSimulatorApp(
AppBundleInformation appInformation,
MlaunchArguments mlaunchArguments,
ICrashSnapshotReporter crashReporter,
ISimulatorDevice simulator,
ISimulatorDevice?companionSimulator,
TimeSpan timeout,
bool waitForExit,
CancellationToken cancellationToken)
{
_mainLog.WriteLine("System log for the '{1}' simulator is: {0}", simulator.SystemLog, simulator.Name);
var simulatorLog = _captureLogFactory.Create(
path: Path.Combine(_logs.Directory, simulator.Name + ".log"),
systemLogPath: simulator.SystemLog,
entireFile: false,
LogType.SystemLog);
simulatorLog.StartCapture();
_logs.Add(simulatorLog);
var simulatorScanToken = await CaptureSimulatorLog(simulator, appInformation, cancellationToken);
using var systemLogs = new DisposableList <ICaptureLog>
{
simulatorLog
};
if (companionSimulator != null)
{
_mainLog.WriteLine("System log for the '{1}' companion simulator is: {0}", companionSimulator.SystemLog, companionSimulator.Name);
var companionLog = _captureLogFactory.Create(
path: Path.Combine(_logs.Directory, companionSimulator.Name + ".log"),
systemLogPath: companionSimulator.SystemLog,
entireFile: false,
LogType.CompanionSystemLog);
companionLog.StartCapture();
_logs.Add(companionLog);
systemLogs.Add(companionLog);
var companionScanToken = await CaptureSimulatorLog(companionSimulator, appInformation, cancellationToken);
if (companionScanToken != null)
{
simulatorScanToken = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, companionScanToken.Token);
}
}
await crashReporter.StartCaptureAsync();
_mainLog.WriteLine("Launching the app");
if (waitForExit)
{
var result = await _processManager.ExecuteCommandAsync(mlaunchArguments, _mainLog, timeout, cancellationToken : cancellationToken);
simulatorScanToken?.Cancel();
return(result);
}
TaskCompletionSource appLaunched = new();
var scanLog = new ScanLog($"Launched {appInformation.BundleIdentifier} with pid", () =>
{
_mainLog.WriteLine("App launch detected");
appLaunched.SetResult();
});
_mainLog.WriteLine("Waiting for the app to launch..");
var runTask = _processManager.ExecuteCommandAsync(mlaunchArguments, Log.CreateAggregatedLog(_mainLog, scanLog), timeout, cancellationToken: cancellationToken);
await Task.WhenAny(runTask, appLaunched.Task);
if (!appLaunched.Task.IsCompleted)
{
// In case the other task completes first, it is because one of these scenarios happened:
// - The app crashed and never launched
// - We missed the launch signal somehow and the app timed out
// - The app launched and quit immediately and race condition noticed that before the scan log did its job
// In all cases, we should return the result of the run task, it will be most likely 137 + Timeout (killed by us)
// If not, it will be a success because the app ran for a super short amount of time
_mainLog.WriteLine("App launch was not detected in time");
return(runTask.Result);
}
_mainLog.WriteLine("Not waiting for the app to exit");
return(new ProcessExecutionResult
{
ExitCode = 0
});
}
public Task <ProcessExecutionResult> RunAsync(Process process, MlaunchArguments args, ILog log, TimeSpan?timeout = null, Dictionary <string, string> environment_variables = null, CancellationToken?cancellation_token = null, bool?diagnostics = null)
{
process.StartInfo.Arguments = args.AsCommandLine();
return(RunAsync(process, log, timeout, environment_variables, cancellation_token, diagnostics));
}
