private readonly IEnumerable <string> _appArguments; // Arguments that will be passed to the iOS application
public AppRunner(
IMlaunchProcessManager processManager,
IHardwareDeviceLoader hardwareDeviceLoader,
ISimulatorLoader simulatorLoader,
ICrashSnapshotReporterFactory snapshotReporterFactory,
ICaptureLogFactory captureLogFactory,
IDeviceLogCapturerFactory deviceLogCapturerFactory,
IExitCodeDetector exitCodeDetector,
IFileBackedLog mainLog,
ILogs logs,
IHelpers helpers,
IEnumerable <string> appArguments,
Action <string>?logCallback = null)
: base(hardwareDeviceLoader, mainLog, logCallback)
{
_processManager = processManager ?? throw new ArgumentNullException(nameof(processManager));
_simulatorLoader = simulatorLoader ?? throw new ArgumentNullException(nameof(simulatorLoader));
_snapshotReporterFactory = snapshotReporterFactory ?? throw new ArgumentNullException(nameof(snapshotReporterFactory));
_captureLogFactory = captureLogFactory ?? throw new ArgumentNullException(nameof(captureLogFactory));
_deviceLogCapturerFactory = deviceLogCapturerFactory ?? throw new ArgumentNullException(nameof(deviceLogCapturerFactory));
_exitCodeDetector = exitCodeDetector ?? throw new ArgumentNullException(nameof(exitCodeDetector));
_logs = logs ?? throw new ArgumentNullException(nameof(logs));
_helpers = helpers ?? throw new ArgumentNullException(nameof(helpers));
_appArguments = appArguments;
}
public ITestReporter Create(IFileBackedLog mainLog,
IReadableLog runLog,
ILogs logs,
ICrashSnapshotReporter crashReporter,
ISimpleListener simpleListener,
IResultParser parser,
AppBundleInformation appInformation,
RunMode runMode,
XmlResultJargon xmlJargon,
string?device,
TimeSpan timeout,
string?additionalLogsDirectory = null,
ExceptionLogger?exceptionLogger = null,
bool generateHtml = false) => new TestReporter(_processManager,
mainLog,
runLog,
logs,
crashReporter,
simpleListener,
parser,
appInformation,
runMode,
xmlJargon,
device,
timeout,
additionalLogsDirectory,
exceptionLogger,
generateHtml);
private async Task <IFileBackedLog> GetSymbolicateCrashReportAsync(IFileBackedLog report)
{
if (_symbolicateCrashPath == null)
{
_log.WriteLine("Can't symbolicate {0} because the symbolicatecrash script {1} does not exist", report.FullPath, _symbolicateCrashPath);
return(report);
}
var name = Path.GetFileName(report.FullPath);
var symbolicated = _logs.Create(Path.ChangeExtension(name, ".symbolicated.log"), $"Symbolicated crash report: {name}", timestamp: false);
var environment = new Dictionary <string, string> {
{ "DEVELOPER_DIR", Path.Combine(_processManager.XcodeRoot, "Contents", "Developer") }
};
var result = await _processManager.ExecuteCommandAsync(_symbolicateCrashPath, new[] { report.FullPath }, symbolicated, TimeSpan.FromMinutes(1), environment);
if (result.Succeeded)
{
_log.WriteLine("Symbolicated {0} successfully.", report.FullPath);
return(symbolicated);
}
else
{
_log.WriteLine("Failed to symbolicate {0}.", report.FullPath);
return(report);
}
}
public bool IsKnownInstallIssue(IFileBackedLog installLog, [NotNullWhen(true)] out string?knownFailureMessage)
{
knownFailureMessage = null;
if (installLog == null)
{
return(false);
}
if (File.Exists(installLog.FullPath) && new FileInfo(installLog.FullPath).Length > 0)
{
using StreamReader reader = installLog.GetReader();
while (!reader.EndOfStream)
{
string line = reader.ReadLine();
if (line == null)
{
continue;
}
var index = line.IndexOf(IncorrectArchPrefix, StringComparison.Ordinal);
if (index >= 0)
{
// add the information from the line, which is good enough
knownFailureMessage = line.Substring(index); // remove the timestamp if any
return(true);
}
}
}
return(false);
}
public AppRunner(IMLaunchProcessManager processManager,
IHardwareDeviceLoader hardwareDeviceLoader,
ISimulatorLoader simulatorLoader,
ISimpleListenerFactory simpleListenerFactory,
ICrashSnapshotReporterFactory snapshotReporterFactory,
ICaptureLogFactory captureLogFactory,
IDeviceLogCapturerFactory deviceLogCapturerFactory,
ITestReporterFactory reporterFactory,
IFileBackedLog mainLog,
ILogs logs,
IHelpers helpers,
bool useXmlOutput,
Action <string>?logCallback = null)
{
_processManager = processManager ?? throw new ArgumentNullException(nameof(processManager));
_hardwareDeviceLoader = hardwareDeviceLoader ?? throw new ArgumentNullException(nameof(hardwareDeviceLoader));
_simulatorLoader = simulatorLoader ?? throw new ArgumentNullException(nameof(simulatorLoader));
_listenerFactory = simpleListenerFactory ?? throw new ArgumentNullException(nameof(simpleListenerFactory));
_snapshotReporterFactory = snapshotReporterFactory ?? throw new ArgumentNullException(nameof(snapshotReporterFactory));
_captureLogFactory = captureLogFactory ?? throw new ArgumentNullException(nameof(captureLogFactory));
_deviceLogCapturerFactory = deviceLogCapturerFactory ?? throw new ArgumentNullException(nameof(deviceLogCapturerFactory));
_testReporterFactory = reporterFactory ?? throw new ArgumentNullException(nameof(_testReporterFactory));
_logs = logs ?? throw new ArgumentNullException(nameof(logs));
_helpers = helpers ?? throw new ArgumentNullException(nameof(helpers));
_useXmlOutput = useXmlOutput;
if (logCallback == null)
{
_mainLog = mainLog ?? throw new ArgumentNullException(nameof(mainLog));
}
else
{
// create using the main as the default log
_mainLog = Log.CreateReadableAggregatedLog(mainLog, new CallbackLog(logCallback));
}
}
private readonly IEnumerable <string> _appArguments; // Arguments that will be passed to the iOS application
public AppTester(
IMlaunchProcessManager processManager,
IHardwareDeviceLoader hardwareDeviceLoader,
ISimulatorLoader simulatorLoader,
ISimpleListenerFactory simpleListenerFactory,
ICrashSnapshotReporterFactory snapshotReporterFactory,
ICaptureLogFactory captureLogFactory,
IDeviceLogCapturerFactory deviceLogCapturerFactory,
ITestReporterFactory reporterFactory,
IResultParser resultParser,
IFileBackedLog mainLog,
ILogs logs,
IHelpers helpers,
IEnumerable <string> appArguments,
Action <string>?logCallback = null)
: base(processManager, hardwareDeviceLoader, captureLogFactory, logs, mainLog, logCallback)
{
_processManager = processManager ?? throw new ArgumentNullException(nameof(processManager));
_simulatorLoader = simulatorLoader ?? throw new ArgumentNullException(nameof(simulatorLoader));
_listenerFactory = simpleListenerFactory ?? throw new ArgumentNullException(nameof(simpleListenerFactory));
_snapshotReporterFactory = snapshotReporterFactory ?? throw new ArgumentNullException(nameof(snapshotReporterFactory));
_captureLogFactory = captureLogFactory ?? throw new ArgumentNullException(nameof(captureLogFactory));
_deviceLogCapturerFactory = deviceLogCapturerFactory ?? throw new ArgumentNullException(nameof(deviceLogCapturerFactory));
_testReporterFactory = reporterFactory ?? throw new ArgumentNullException(nameof(_testReporterFactory));
_resultParser = resultParser ?? throw new ArgumentNullException(nameof(resultParser));
_mainLog = mainLog ?? throw new ArgumentNullException(nameof(mainLog));
_logs = logs ?? throw new ArgumentNullException(nameof(logs));
_helpers = helpers ?? throw new ArgumentNullException(nameof(helpers));
_appArguments = appArguments;
}
public ReadableAggregatedLog(IFileBackedLog defaultLog, params ILog[] logs) : base(logs)
{
_defaultLog = defaultLog ?? throw new ArgumentNullException(nameof(defaultLog));
// make sure that we also write in the default log
_logs.Add(defaultLog);
Timestamp = false;
}
public RunOrchestrator(
IAppBundleInformationParser appBundleInformationParser,
IAppInstaller appInstaller,
IAppUninstaller appUninstaller,
IAppRunnerFactory appRunnerFactory,
IDeviceFinder deviceFinder,
IiOSExitCodeDetector iOSExitCodeDetector,
IMacCatalystExitCodeDetector macCatalystExitCodeDetector,
ILogger consoleLogger,
ILogs logs,
IFileBackedLog mainLog,
IErrorKnowledgeBase errorKnowledgeBase,
IDiagnosticsData diagnosticsData,
IHelpers helpers)
: base(appBundleInformationParser, appInstaller, appUninstaller, deviceFinder, consoleLogger, logs, mainLog, errorKnowledgeBase, diagnosticsData, helpers)
{
_iOSExitCodeDetector = iOSExitCodeDetector ?? throw new ArgumentNullException(nameof(iOSExitCodeDetector));
_macCatalystExitCodeDetector = macCatalystExitCodeDetector ?? throw new ArgumentNullException(nameof(macCatalystExitCodeDetector));
_logger = consoleLogger ?? throw new ArgumentNullException(nameof(consoleLogger));
_logs = logs ?? throw new ArgumentNullException(nameof(logs));
_errorKnowledgeBase = errorKnowledgeBase ?? throw new ArgumentNullException(nameof(errorKnowledgeBase));
// Only add the extra callback if we do know that the feature was indeed enabled
Action <string>?logCallback = IsLldbEnabled() ? (l) => NotifyUserLldbCommand(_logger, l) : null;
_appRunner = appRunnerFactory.Create(mainLog, logs, logCallback);
}
public AppInstallMonitorLog(IFileBackedLog copy_to)
: base($"Watch transfer log for {copy_to.Description}")
{
_copyTo = copy_to;
_cancellationSource = new CancellationTokenSource();
_cancellationSource.Token.Register(() =>
{
copy_to.WriteLine("App installation cancelled: it timed out after no output for 1 minute.");
});
}
public async Task <(TestExecutingResult ExecutionResult, (string HumanMessage, string IssueLink)? KnownFailure)> ExecuteAsync(
string projectPlatform,
string projectConfiguration,
string projectFile,
IAcquiredResource resource,
bool dryRun,
IFileBackedLog buildLog,
ILog mainLog)
{
BuildLog = buildLog;
(TestExecutingResult ExecutionResult, (string HumanMessage, string IssueLink)? KnownFailure)result = (TestExecutingResult.NotStarted, ((string HumanMessage, string IssueLink)?)null);
var restoreResult = await RestoreNugetsAsync(buildLog, resource);
if ((restoreResult & TestExecutingResult.Failed) == TestExecutingResult.Failed)
{
BuildLog.WriteLine($"Failed to restore nugets: {restoreResult}");
result.ExecutionResult = restoreResult;
return(result);
}
using (var xbuild = new Process()) {
xbuild.StartInfo.FileName = msbuildPath();
xbuild.StartInfo.Arguments = StringUtils.FormatArguments(GetToolArguments(projectPlatform, projectConfiguration, projectFile, buildLog));
xbuild.StartInfo.WorkingDirectory = Path.GetDirectoryName(projectFile);
EnvironmentManager.SetEnvironmentVariables(xbuild);
xbuild.StartInfo.EnvironmentVariables ["MSBuildExtensionsPath"] = null;
EventLogger.LogEvent(buildLog, "Building {0} ({1})", TestName, Mode);
if (!dryRun)
{
var timeout = TimeSpan.FromMinutes(60);
var processResult = await ProcessManager.RunAsync(xbuild, buildLog, timeout);
if (processResult.TimedOut)
{
result.ExecutionResult = TestExecutingResult.TimedOut;
buildLog.WriteLine("Build timed out after {0} seconds.", timeout.TotalSeconds);
}
else if (processResult.Succeeded)
{
result.ExecutionResult = TestExecutingResult.Succeeded;
}
else
{
result.ExecutionResult = TestExecutingResult.Failed;
if (errorKnowledgeBase.IsKnownBuildIssue(buildLog, out result.KnownFailure))
{
buildLog.WriteLine($"Build has a known failure: '{result.KnownFailure}'");
}
}
}
mainLog.WriteLine("Built {0} ({1})", TestName, Mode);
}
return(result);
}
protected override async Task <ExitCode> RunAppInternal(
AppBundleInformation appBundleInfo,
string?deviceName,
ILogger logger,
TestTargetOs target,
Logs logs,
IFileBackedLog mainLog,
CancellationToken cancellationToken)
{
// only add the extra callback if we do know that the feature was indeed enabled
Action <string>?logCallback = IsLldbEnabled() ? (l) => NotifyUserLldbCommand(logger, l) : (Action <string>?)null;
var appRunner = new AppRunner(
ProcessManager,
DeviceLoader,
SimulatorLoader,
new CrashSnapshotReporterFactory(ProcessManager),
new CaptureLogFactory(),
new DeviceLogCapturerFactory(ProcessManager),
new ExitCodeDetector(),
mainLog,
logs,
new Helpers(),
PassThroughArguments,
logCallback);
int?exitCode = null;
(deviceName, exitCode) = await appRunner.RunApp(
appBundleInfo,
target,
_arguments.Timeout,
deviceName,
verbosity : GetMlaunchVerbosity(_arguments.Verbosity),
cancellationToken : cancellationToken);
if (exitCode.HasValue)
{
if (_arguments.ExpectedExitCode != exitCode)
{
logger.LogError($"Application has finished with exit code {exitCode} but {_arguments.ExpectedExitCode} was expected");
return(ExitCode.GENERAL_FAILURE);
}
logger.LogInformation("Application has finished with exit code: " + exitCode + (_arguments.ExpectedExitCode != 0 ? " (as expected)" : null));
return(ExitCode.SUCCESS);
}
else
{
logger.LogError("Application has finished but no system log found. Failed to determine the exit code!");
return(ExitCode.RETURN_CODE_NOT_SET);
}
}
public InstallOrchestrator(
IAppInstaller appInstaller,
IAppUninstaller appUninstaller,
IAppBundleInformationParser appBundleInformationParser,
IDeviceFinder deviceFinder,
ILogger consoleLogger,
ILogs logs,
IFileBackedLog mainLog,
IErrorKnowledgeBase errorKnowledgeBase,
IDiagnosticsData diagnosticsData,
IHelpers helpers)
: base(appBundleInformationParser, appInstaller, appUninstaller, deviceFinder, consoleLogger, logs, mainLog, errorKnowledgeBase, diagnosticsData, helpers)
{
}
protected AppRunnerBase(
IHardwareDeviceLoader hardwareDeviceLoader,
IFileBackedLog mainLog,
Action <string>?logCallback = null)
{
_hardwareDeviceLoader = hardwareDeviceLoader ?? throw new ArgumentNullException(nameof(hardwareDeviceLoader));
if (logCallback == null)
{
_mainLog = mainLog ?? throw new ArgumentNullException(nameof(mainLog));
}
else
{
// create using the main as the default log
_mainLog = Log.CreateReadableAggregatedLog(mainLog, new CallbackLog(logCallback));
}
}
public JustRunOrchestrator(
IAppBundleInformationParser appBundleInformationParser,
IAppInstaller appInstaller,
IAppUninstaller appUninstaller,
IAppRunnerFactory appRunnerFactory,
IDeviceFinder deviceFinder,
IiOSExitCodeDetector iOSExitCodeDetector,
IMacCatalystExitCodeDetector macCatalystExitCodeDetector,
ILogger consoleLogger,
ILogs logs,
IFileBackedLog mainLog,
IErrorKnowledgeBase errorKnowledgeBase,
IDiagnosticsData diagnosticsData,
IHelpers helpers)
: base(appBundleInformationParser, appInstaller, appUninstaller, appRunnerFactory, deviceFinder, iOSExitCodeDetector, macCatalystExitCodeDetector, consoleLogger, logs, mainLog, errorKnowledgeBase, diagnosticsData, helpers)
{
}
public TestReporter(IMlaunchProcessManager processManager,
IFileBackedLog mainLog,
IReadableLog runLog,
ILogs logs,
ICrashSnapshotReporter crashReporter,
ISimpleListener simpleListener,
IResultParser parser,
AppBundleInformation appInformation,
RunMode runMode,
XmlResultJargon xmlJargon,
string?device,
TimeSpan timeout,
string?additionalLogsDirectory = null,
ExceptionLogger?exceptionLogger = null,
bool generateHtml = false)
{
_processManager = processManager ?? throw new ArgumentNullException(nameof(processManager));
_deviceName = device; // can be null on simulators
_listener = simpleListener ?? throw new ArgumentNullException(nameof(simpleListener));
_mainLog = mainLog ?? throw new ArgumentNullException(nameof(mainLog));
_runLog = runLog ?? throw new ArgumentNullException(nameof(runLog));
_logs = logs ?? throw new ArgumentNullException(nameof(logs));
_crashReporter = crashReporter ?? throw new ArgumentNullException(nameof(crashReporter));
_crashLogs = new Logs(logs.Directory);
_resultParser = parser ?? throw new ArgumentNullException(nameof(parser));
_appInfo = appInformation ?? throw new ArgumentNullException(nameof(appInformation));
_runMode = runMode;
_xmlJargon = xmlJargon;
_timeout = timeout;
_additionalLogsDirectory = additionalLogsDirectory;
_exceptionLogger = exceptionLogger;
_timeoutWatch = Stopwatch.StartNew();
_generateHtml = generateHtml;
CallbackLog = new CallbackLog(line =>
{
// MT1111: Application launched successfully, but it's not possible to wait for the app to exit as
// requested because it's not possible to detect app termination when launching using gdbserver
_waitedForExit &= line?.Contains("MT1111: ") != true;
if (line?.Contains("error MT1007") == true)
{
_launchFailure = true;
}
});
}
public TestOrchestrator(
IAppBundleInformationParser appBundleInformationParser,
IAppInstaller appInstaller,
IAppUninstaller appUninstaller,
IAppTesterFactory appTesterFactory,
IDeviceFinder deviceFinder,
ILogger consoleLogger,
ILogs logs,
IFileBackedLog mainLog,
IErrorKnowledgeBase errorKnowledgeBase,
IDiagnosticsData diagnosticsData,
IHelpers helpers)
: base(appBundleInformationParser, appInstaller, appUninstaller, deviceFinder, consoleLogger, logs, mainLog, errorKnowledgeBase, diagnosticsData, helpers)
{
_appTesterFactory = appTesterFactory ?? throw new ArgumentNullException(nameof(appTesterFactory));
_logger = consoleLogger ?? throw new ArgumentNullException(nameof(consoleLogger));
_logs = logs ?? throw new ArgumentNullException(nameof(logs));
_mainLog = mainLog ?? throw new ArgumentNullException(nameof(mainLog));
_errorKnowledgeBase = errorKnowledgeBase ?? throw new ArgumentNullException(nameof(errorKnowledgeBase));
}
protected BaseOrchestrator(
IAppBundleInformationParser appBundleInformationParser,
IAppInstaller appInstaller,
IAppUninstaller appUninstaller,
IDeviceFinder deviceFinder,
ILogger consoleLogger,
ILogs logs,
IFileBackedLog mainLog,
IErrorKnowledgeBase errorKnowledgeBase,
IDiagnosticsData diagnosticsData,
IHelpers helpers)
{
_appBundleInformationParser = appBundleInformationParser ?? throw new ArgumentNullException(nameof(appBundleInformationParser));
_appInstaller = appInstaller ?? throw new ArgumentNullException(nameof(appInstaller));
_appUninstaller = appUninstaller ?? throw new ArgumentNullException(nameof(appUninstaller));
_deviceFinder = deviceFinder ?? throw new ArgumentNullException(nameof(deviceFinder));
_logger = consoleLogger ?? throw new ArgumentNullException(nameof(consoleLogger));
_logs = logs ?? throw new ArgumentNullException(nameof(logs));
_mainLog = mainLog ?? throw new ArgumentNullException(nameof(mainLog));
_errorKnowledgeBase = errorKnowledgeBase ?? throw new ArgumentNullException(nameof(errorKnowledgeBase));
_diagnosticsData = diagnosticsData ?? throw new ArgumentNullException(nameof(diagnosticsData));
_helpers = helpers ?? throw new ArgumentNullException(nameof(helpers));
}
protected AppRunnerBase(
IProcessManager processManager,
IHardwareDeviceLoader hardwareDeviceLoader,
ICaptureLogFactory captureLogFactory,
ILogs logs,
IFileBackedLog mainLog,
Action <string>?logCallback = null)
{
_hardwareDeviceLoader = hardwareDeviceLoader ?? throw new ArgumentNullException(nameof(hardwareDeviceLoader));
_captureLogFactory = captureLogFactory ?? throw new ArgumentNullException(nameof(captureLogFactory));
_logs = logs ?? throw new ArgumentNullException(nameof(logs));
_processManager = processManager ?? throw new ArgumentNullException(nameof(processManager));
if (logCallback == null)
{
_mainLog = mainLog ?? throw new ArgumentNullException(nameof(mainLog));
}
else
{
// create using the main as the default log
_mainLog = Log.CreateReadableAggregatedLog(mainLog, new CallbackLog(logCallback));
}
}
public IAppTester Create(
CommunicationChannel communicationChannel,
bool isSimulator,
IFileBackedLog log,
ILogs logs,
Action <string>?logCallback)
{
var tunnelBore = (communicationChannel == CommunicationChannel.UsbTunnel && !isSimulator)
? new TunnelBore(_processManager)
: null;
return(new AppTester(
_processManager,
new SimpleListenerFactory(tunnelBore),
_snapshotReporterFactory,
_captureLogFactory,
_deviceLogCapturerFactory,
_reporterFactory,
_resultParser,
log,
logs,
_helpers,
logCallback));
}
public bool IsKnownInstallIssue(IFileBackedLog installLog, [NotNullWhen(true)] out KnownIssue?knownFailureMessage)
{
// nothing yet that we are aware of
knownFailureMessage = null;
return(false);
}
public bool IsKnownTestIssue(IFileBackedLog runLog, [NotNullWhen(true)] out KnownIssue?knownFailureMessage) => TryFindErrors(runLog, testErrorMaps, out knownFailureMessage);
public bool IsKnownBuildIssue(IFileBackedLog buildLog, [NotNullWhen(true)] out KnownIssue?knownFailureMessage) => TryFindErrors(buildLog, buildErrorMaps, out knownFailureMessage);
public override async Task RunTestAsync()
{
var projectDir = System.IO.Path.GetDirectoryName(ProjectFile);
var name = System.IO.Path.GetFileName(projectDir);
if (string.Equals("mac", name, StringComparison.OrdinalIgnoreCase))
{
name = System.IO.Path.GetFileName(System.IO.Path.GetDirectoryName(projectDir));
}
var suffix = string.Empty;
switch (Platform)
{
case TestPlatform.Mac_Modern:
suffix = "-modern";
break;
case TestPlatform.Mac_Full:
suffix = "-full";
break;
case TestPlatform.Mac_System:
suffix = "-system";
break;
}
if (ProjectFile.EndsWith(".sln", StringComparison.Ordinal))
{
Path = System.IO.Path.Combine(System.IO.Path.GetDirectoryName(ProjectFile), "bin", BuildTask.ProjectPlatform, BuildTask.ProjectConfiguration + suffix, name + ".app", "Contents", "MacOS", name);
}
else
{
var project = new XmlDocument();
project.LoadWithoutNetworkAccess(ProjectFile);
string outputPath;
if (TestProject?.IsDotNetProject == true)
{
outputPath = await Harness.AppBundleLocator.LocateAppBundle(project, ProjectFile, TestTarget.None, BuildTask.ProjectConfiguration);
}
else
{
outputPath = project.GetOutputPath(BuildTask.ProjectPlatform, BuildTask.ProjectConfiguration).Replace('\\', '/');
}
var assemblyName = project.GetAssemblyName();
Path = System.IO.Path.Combine(System.IO.Path.GetDirectoryName(ProjectFile), outputPath, assemblyName + ".app", "Contents", "MacOS", assemblyName);
}
using (var resource = await NotifyAndAcquireDesktopResourceAsync()) {
using (var proc = new Process()) {
proc.StartInfo.FileName = Path;
var arguments = new List <string> ();
IFileBackedLog xmlLog = null;
var useXmlOutput = Harness.InCI || true;
if (IsUnitTest)
{
var extension = useXmlOutput ? "xml" : "log";
var type = useXmlOutput ? LogType.XmlLog : LogType.NUnitResult;
xmlLog = Logs.Create($"test-{Platform}-{Timestamp}.{extension}", type.ToString());
arguments.Add($"-transport:FILE");
proc.StartInfo.EnvironmentVariables ["NUNIT_TRANSPORT"] = "FILE";
arguments.Add($"--logfile:{xmlLog.FullPath}");
proc.StartInfo.EnvironmentVariables ["NUNIT_LOG_FILE"] = xmlLog.FullPath;
if (useXmlOutput)
{
arguments.Add("--enablexml");
proc.StartInfo.EnvironmentVariables ["NUNIT_ENABLE_XML_OUTPUT"] = "true";
arguments.Add("--xmlmode=wrapped");
proc.StartInfo.EnvironmentVariables ["NUNIT_ENABLE_XML_MODE"] = "wrapped";
arguments.Add("--xmlversion=nunitv3");
proc.StartInfo.EnvironmentVariables ["NUNIT_XML_VERSION"] = "nunitv3";
}
arguments.Add("--autostart");
proc.StartInfo.EnvironmentVariables ["NUNIT_AUTOSTART"] = "true";
arguments.Add("--autoexit");
proc.StartInfo.EnvironmentVariables ["NUNIT_AUTOEXIT"] = "true";
}
if (!Harness.GetIncludeSystemPermissionTests(Platform, false))
{
proc.StartInfo.EnvironmentVariables ["DISABLE_SYSTEM_PERMISSION_TESTS"] = "1";
}
proc.StartInfo.EnvironmentVariables ["MONO_DEBUG"] = "no-gdb-backtrace";
proc.StartInfo.EnvironmentVariables.Remove("DYLD_FALLBACK_LIBRARY_PATH"); // VSMac might set this, and the test may end up crashing
proc.StartInfo.Arguments = StringUtils.FormatArguments(arguments);
Jenkins.MainLog.WriteLine("Executing {0} ({1})", TestName, Mode);
var log = Logs.Create($"execute-{Platform}-{Timestamp}.txt", LogType.ExecutionLog.ToString());
ICrashSnapshotReporter snapshot = null;
if (!Jenkins.Harness.DryRun)
{
ExecutionResult = TestExecutingResult.Running;
snapshot = CrashReportSnapshotFactory.Create(log, Logs, isDevice: false, deviceName: null);
await snapshot.StartCaptureAsync();
ProcessExecutionResult result = null;
try {
var timeout = TimeSpan.FromMinutes(20);
result = await ProcessManager.RunAsync(proc, log, timeout);
if (result.TimedOut)
{
FailureMessage = $"Execution timed out after {timeout.TotalSeconds} seconds.";
log.WriteLine(FailureMessage);
ExecutionResult = TestExecutingResult.TimedOut;
}
else if (result.Succeeded)
{
ExecutionResult = TestExecutingResult.Succeeded;
}
else
{
ExecutionResult = TestExecutingResult.Failed;
FailureMessage = result.ExitCode != 1 ? $"Test run crashed (exit code: {result.ExitCode})." : "Test run failed.";
log.WriteLine(FailureMessage);
}
} finally {
await snapshot.EndCaptureAsync(TimeSpan.FromSeconds(Succeeded ? 0 : result?.ExitCode > 1 ? 120 : 5));
}
}
Jenkins.MainLog.WriteLine("Executed {0} ({1})", TestName, Mode);
if (IsUnitTest)
{
var reporterFactory = new TestReporterFactory(ProcessManager);
var listener = new Microsoft.DotNet.XHarness.iOS.Shared.Listeners.SimpleFileListener(xmlLog.FullPath, log, xmlLog, useXmlOutput);
var reporter = reporterFactory.Create(Harness.HarnessLog, log, Logs, snapshot, listener, Harness.ResultParser, new AppBundleInformation("N/A", "N/A", "N/A", "N/A", true, null), RunMode.MacOS, Harness.XmlJargon, "no device here", TimeSpan.Zero);
var rv = await reporter.ParseResult();
if (ExecutionResult == TestExecutingResult.Succeeded)
{
// The process might have crashed, timed out at exit, or otherwise returned a non-zero exit code when all the unit tests passed, in which we shouldn't override the execution result here,
ExecutionResult = rv.ExecutingResult;
}
// Set or replace the failure message, depending on whether there already is a failure message or not.
if (string.IsNullOrEmpty(FailureMessage))
{
FailureMessage = rv.ResultMessage;
}
else if (!string.IsNullOrEmpty(rv.ResultMessage))
{
FailureMessage += "\n" + rv.ResultMessage;
}
}
}
}
}
protected override async Task <ExitCode> RunAppInternal(
AppBundleInformation appBundleInfo,
string?deviceName,
ILogger logger,
TestTargetOs target,
Logs logs,
IFileBackedLog mainLog,
CancellationToken cancellationToken)
{
// only add the extra callback if we do know that the feature was indeed enabled
Action <string>?logCallback = IsLldbEnabled() ? (l) => NotifyUserLldbCommand(logger, l) : (Action <string>?)null;
var appRunner = new AppRunner(
ProcessManager,
DeviceLoader,
SimulatorLoader,
new CrashSnapshotReporterFactory(ProcessManager),
new CaptureLogFactory(),
new DeviceLogCapturerFactory(ProcessManager),
mainLog,
logs,
new Helpers(),
PassThroughArguments,
logCallback);
ProcessExecutionResult result;
(deviceName, result) = await appRunner.RunApp(
appBundleInfo,
target,
_arguments.Timeout,
deviceName,
verbosity : GetMlaunchVerbosity(_arguments.Verbosity),
cancellationToken : cancellationToken);
if (!result.Succeeded)
{
if (result.TimedOut)
{
logger.LogError($"App run has timed out");
return(ExitCode.TIMED_OUT);
}
logger.LogError($"App run has failed. mlaunch exited with {result.ExitCode}");
return(ExitCode.APP_LAUNCH_FAILURE);
}
var systemLog = logs.FirstOrDefault(log => log.Description == LogType.SystemLog.ToString());
if (systemLog == null)
{
logger.LogError("Application has finished but no system log found. Failed to determine the exit code!");
return(ExitCode.RETURN_CODE_NOT_SET);
}
var exitCode = new ExitCodeDetector().DetectExitCode(appBundleInfo, systemLog);
logger.LogInformation($"App run ended with {exitCode}");
if (_arguments.ExpectedExitCode != exitCode)
{
logger.LogError($"Application has finished with exit code {exitCode} but {_arguments.ExpectedExitCode} was expected");
return(ExitCode.GENERAL_FAILURE);
}
logger.LogInformation("Application has finished with exit code: " + exitCode +
(_arguments.ExpectedExitCode != 0 ? " (as expected)" : null));
return(ExitCode.SUCCESS);
}
public SimpleTcpListener(int port, ILog log, IFileBackedLog testLog, bool autoExit, bool tunnel = false) : this(log, testLog, autoExit, tunnel)
{
Port = port;
}
public SimpleTcpListener(ILog log, IFileBackedLog testLog, bool autoExit, bool tunnel = false) : base(log, testLog)
{
_autoExit = autoExit;
_useTcpTunnel = tunnel;
}
public static IFileBackedLog CreateReadableAggregatedLog(IFileBackedLog defaultLog, params ILog[] logs) => new ReadableAggregatedLog(defaultLog, logs);
public virtual List <string> GetToolArguments(string projectPlatform, string projectConfiguration, string projectFile, IFileBackedLog buildLog)
{
var binlogPath = buildLog.FullPath.Replace(".txt", ".binlog");
var args = new List <string> ();
args.Add("--");
args.Add("/verbosity:diagnostic");
args.Add($"/bl:{binlogPath}");
if (SpecifyPlatform)
{
args.Add($"/p:Platform={projectPlatform}");
}
if (SpecifyConfiguration)
{
args.Add($"/p:Configuration={projectConfiguration}");
}
if (Platform == TestPlatform.MacCatalyst)
{
args.Add("/r");
}
args.Add(projectFile);
if (Constants.Count > 0)
{
args.Add($"/p:DefineConstants=\"{string.Join(";", Constants)}\"");
}
return(args);
}
public SimpleFileListener(string path, ILog log, IFileBackedLog testLog, bool xmlOutput) : base(log, testLog)
{
Path = path ?? throw new ArgumentNullException(nameof(path));
_xmlOutput = xmlOutput;
}
public override async Task RunTestAsync()
{
var projectDir = System.IO.Path.GetDirectoryName(ProjectFile);
var name = System.IO.Path.GetFileName(projectDir);
if (string.Equals("mac", name, StringComparison.OrdinalIgnoreCase))
{
name = System.IO.Path.GetFileName(System.IO.Path.GetDirectoryName(projectDir));
}
var suffix = string.Empty;
switch (Platform)
{
case TestPlatform.Mac_Modern:
suffix = "-modern";
break;
case TestPlatform.Mac_Full:
suffix = "-full";
break;
case TestPlatform.Mac_System:
suffix = "-system";
break;
}
if (ProjectFile.EndsWith(".sln", StringComparison.Ordinal))
{
Path = System.IO.Path.Combine(System.IO.Path.GetDirectoryName(ProjectFile), "bin", BuildTask.ProjectPlatform, BuildTask.ProjectConfiguration + suffix, name + ".app", "Contents", "MacOS", name);
}
else
{
var project = new XmlDocument();
project.LoadWithoutNetworkAccess(ProjectFile);
var outputPath = project.GetOutputPath(BuildTask.ProjectPlatform, BuildTask.ProjectConfiguration).Replace('\\', '/');
var assemblyName = project.GetAssemblyName();
Path = System.IO.Path.Combine(System.IO.Path.GetDirectoryName(ProjectFile), outputPath, assemblyName + ".app", "Contents", "MacOS", assemblyName);
}
using (var resource = await NotifyAndAcquireDesktopResourceAsync()) {
using (var proc = new Process()) {
proc.StartInfo.FileName = Path;
var arguments = new List <string> ();
IFileBackedLog xmlLog = null;
var useXmlOutput = Harness.InCI || true;
if (IsUnitTest)
{
var extension = useXmlOutput ? "xml" : "log";
var type = useXmlOutput ? LogType.XmlLog : LogType.NUnitResult;
xmlLog = Logs.Create($"test-{Platform}-{Timestamp}.{extension}", type.ToString());
arguments.Add($"-transport:FILE");
arguments.Add($"--logfile:{xmlLog.FullPath}");
if (useXmlOutput)
{
arguments.Add("--enablexml");
arguments.Add("--xmlmode=wrapped");
arguments.Add("--xmlversion=nunitv3");
}
}
if (!Harness.GetIncludeSystemPermissionTests(Platform, false))
{
proc.StartInfo.EnvironmentVariables ["DISABLE_SYSTEM_PERMISSION_TESTS"] = "1";
}
proc.StartInfo.EnvironmentVariables ["MONO_DEBUG"] = "no-gdb-backtrace";
proc.StartInfo.Arguments = StringUtils.FormatArguments(arguments);
Jenkins.MainLog.WriteLine("Executing {0} ({1})", TestName, Mode);
var log = Logs.Create($"execute-{Platform}-{Timestamp}.txt", LogType.ExecutionLog.ToString());
ICrashSnapshotReporter snapshot = null;
if (!Jenkins.Harness.DryRun)
{
ExecutionResult = TestExecutingResult.Running;
snapshot = CrashReportSnapshotFactory.Create(log, Logs, isDevice: false, deviceName: null);
await snapshot.StartCaptureAsync();
ProcessExecutionResult result = null;
try {
var timeout = TimeSpan.FromMinutes(20);
result = await ProcessManager.RunAsync(proc, log, timeout);
if (result.TimedOut)
{
FailureMessage = $"Execution timed out after {timeout.TotalSeconds} seconds.";
log.WriteLine(FailureMessage);
ExecutionResult = TestExecutingResult.TimedOut;
}
else if (result.Succeeded)
{
ExecutionResult = TestExecutingResult.Succeeded;
}
else
{
ExecutionResult = TestExecutingResult.Failed;
FailureMessage = result.ExitCode != 1 ? $"Test run crashed (exit code: {result.ExitCode})." : "Test run failed.";
log.WriteLine(FailureMessage);
}
} finally {
await snapshot.EndCaptureAsync(TimeSpan.FromSeconds(Succeeded ? 0 : result?.ExitCode > 1 ? 120 : 5));
}
}
Jenkins.MainLog.WriteLine("Executed {0} ({1})", TestName, Mode);
if (IsUnitTest)
{
var reporterFactory = new TestReporterFactory(ProcessManager);
var listener = new Microsoft.DotNet.XHarness.iOS.Shared.Listeners.SimpleFileListener(xmlLog.FullPath, log, xmlLog, useXmlOutput);
var reporter = reporterFactory.Create(Harness.HarnessLog, log, Logs, snapshot, listener, Harness.ResultParser, new AppBundleInformation("N/A", "N/A", "N/A", "N/A", true, null), RunMode.MacOS, Harness.XmlJargon, "no device here", TimeSpan.Zero);
var rv = await reporter.ParseResult();
ExecutionResult = rv.ExecutingResult;
FailureMessage = rv.ExecutingResult == TestExecutingResult.Succeeded ? null : rv.ResultMessage;
}
}
}
}
