public static async Task <ProcessExecutionResult> RunAsync(this Process process, Log log, TextWriter StdoutStream, TextWriter StderrStream, TimeSpan?timeout = null, Dictionary <string, string> environment_variables = null, CancellationToken?cancellation_token = null)
{
var stdout_completion = new TaskCompletionSource <bool> ();
var stderr_completion = new TaskCompletionSource <bool> ();
var exit_completion = new TaskCompletionSource <bool> ();
var rv = new ProcessExecutionResult();
process.StartInfo.RedirectStandardError = true;
process.StartInfo.RedirectStandardOutput = true;
process.StartInfo.UseShellExecute = false;
if (environment_variables != null)
{
foreach (var kvp in environment_variables)
{
process.StartInfo.EnvironmentVariables [kvp.Key] = kvp.Value;
}
}
process.OutputDataReceived += (object sender, DataReceivedEventArgs e) =>
{
if (e.Data != null)
{
lock (StdoutStream) {
StdoutStream.WriteLine(e.Data);
StdoutStream.Flush();
}
}
else
{
stdout_completion.TrySetResult(true);
}
};
process.ErrorDataReceived += (object sender, DataReceivedEventArgs e) =>
{
if (e.Data != null)
{
lock (StderrStream) {
StderrStream.WriteLine(e.Data);
StderrStream.Flush();
}
}
else
{
stderr_completion.TrySetResult(true);
}
};
log.WriteLine("{0} {1}", process.StartInfo.FileName, process.StartInfo.Arguments);
process.Start();
process.BeginErrorReadLine();
process.BeginOutputReadLine();
cancellation_token?.Register(() => {
if (!exit_completion.Task.IsCompleted)
{
StderrStream.WriteLine($"Execution was cancelled.");
ProcessHelper.kill(process.Id, 9);
}
});
new Thread(() =>
{
if (timeout.HasValue)
{
if (!process.WaitForExit((int)timeout.Value.TotalMilliseconds))
{
process.KillTreeAsync(log, true).Wait();
rv.TimedOut = true;
lock (StderrStream)
log.WriteLine($"Execution timed out after {timeout.Value.TotalSeconds} seconds and the process was killed.");
}
}
else
{
process.WaitForExit();
}
exit_completion.TrySetResult(true);
Task.WaitAll(new Task [] { stderr_completion.Task, stdout_completion.Task }, TimeSpan.FromSeconds(1));
stderr_completion.TrySetResult(false);
stdout_completion.TrySetResult(false);
})
{
IsBackground = true,
}.Start();
await Task.WhenAll(stderr_completion.Task, stdout_completion.Task, exit_completion.Task);
rv.ExitCode = process.ExitCode;
return(rv);
}
public async Task EndCaptureAsync(TimeSpan timeout)
{
// Check for crash reports
var crash_report_search_done = false;
var crash_report_search_timeout = timeout.TotalSeconds;
var watch = new Stopwatch();
watch.Start();
do
{
var end_crashes = await Harness.CreateCrashReportsSnapshotAsync(Log, !Device, DeviceName);
end_crashes.ExceptWith(InitialSet);
Reports = end_crashes;
if (end_crashes.Count > 0)
{
Log.WriteLine("Found {0} new crash report(s)", end_crashes.Count);
List <LogFile> crash_reports;
if (!Device)
{
crash_reports = new List <LogFile> (end_crashes.Count);
foreach (var path in end_crashes)
{
var logPath = Path.Combine(LogDirectory, Path.GetFileName(path));
File.Copy(path, logPath, true);
crash_reports.Add(Logs.CreateFile("Crash report: " + Path.GetFileName(path), logPath));
}
}
else
{
// Download crash reports from the device. We put them in the project directory so that they're automatically deleted on wrench
// (if we put them in /tmp, they'd never be deleted).
var downloaded_crash_reports = new List <LogFile> ();
foreach (var file in end_crashes)
{
var crash_report_target = Logs.CreateFile("Crash report: " + Path.GetFileName(file), Path.Combine(LogDirectory, Path.GetFileName(file)));
var sb = new StringBuilder();
sb.Append(" --download-crash-report=").Append(StringUtils.Quote(file));
sb.Append(" --download-crash-report-to=").Append(StringUtils.Quote(crash_report_target.Path));
sb.Append(" --sdkroot ").Append(StringUtils.Quote(Harness.XcodeRoot));
if (!string.IsNullOrEmpty(DeviceName))
{
sb.Append(" --devname ").Append(StringUtils.Quote(DeviceName));
}
var result = await ProcessHelper.ExecuteCommandAsync(Harness.MlaunchPath, sb.ToString(), Log, TimeSpan.FromMinutes(1));
if (result.Succeeded)
{
Log.WriteLine("Downloaded crash report {0} to {1}", file, crash_report_target.Path);
crash_report_target = await Harness.SymbolicateCrashReportAsync(Log, crash_report_target);
Logs.Add(crash_report_target);
downloaded_crash_reports.Add(crash_report_target);
}
else
{
Log.WriteLine("Could not download crash report {0}", file);
}
}
crash_reports = downloaded_crash_reports;
}
foreach (var cp in crash_reports)
{
Harness.LogWrench("@MonkeyWrench: AddFile: {0}", cp.Path);
Log.WriteLine(" {0}", cp.Path);
}
crash_report_search_done = true;
}
else
{
if (watch.Elapsed.TotalSeconds > crash_report_search_timeout)
{
crash_report_search_done = true;
}
else
{
Log.WriteLine("No crash reports, waiting a second to see if the crash report service just didn't complete in time ({0})", (int)(crash_report_search_timeout - watch.Elapsed.TotalSeconds));
Thread.Sleep(TimeSpan.FromSeconds(1));
}
}
} while (!crash_report_search_done);
}
public Task <ProcessExecutionResult> ExecuteXcodeCommandAsync(string executable, string args, Log log, TimeSpan timeout)
{
return(ProcessHelper.ExecuteCommandAsync(Path.Combine(XcodeRoot, "Contents", "Developer", "usr", "bin", executable), args, log, timeout: timeout));
}
public async Task AgreeToPromptsAsync(Log log, params string[] bundle_identifiers)
{
if (bundle_identifiers == null || bundle_identifiers.Length == 0)
{
log.WriteLine("No bundle identifiers given when requested permission editing.");
return;
}
var TCC_db = Path.Combine(DataPath, "data", "Library", "TCC", "TCC.db");
var sim_services = new string [] {
"kTCCServiceAddressBook",
"kTCCServicePhotos",
"kTCCServiceMediaLibrary",
"kTCCServiceUbiquity",
"kTCCServiceWillow"
};
var failure = false;
var tcc_edit_timeout = 5;
var watch = new Stopwatch();
watch.Start();
do
{
if (failure)
{
log.WriteLine("Failed to edit TCC.db, trying again in 1 second... ", (int)(tcc_edit_timeout - watch.Elapsed.TotalSeconds));
await Task.Delay(TimeSpan.FromSeconds(1));
}
failure = false;
foreach (var bundle_identifier in bundle_identifiers)
{
var sql = new System.Text.StringBuilder();
sql.Append(StringUtils.Quote(TCC_db));
sql.Append(" \"");
foreach (var service in sim_services)
{
sql.AppendFormat("INSERT INTO access VALUES('{0}','{1}',0,1,0,NULL,NULL);", service, bundle_identifier);
sql.AppendFormat("INSERT INTO access VALUES('{0}','{1}',0,1,0,NULL,NULL);", service, bundle_identifier + ".watchkitapp");
}
sql.Append("\"");
var rv = await ProcessHelper.ExecuteCommandAsync("sqlite3", sql.ToString(), log, TimeSpan.FromSeconds(5));
if (!rv.Succeeded)
{
failure = true;
break;
}
}
} while (failure && watch.Elapsed.TotalSeconds <= tcc_edit_timeout);
if (failure)
{
log.WriteLine("Failed to edit TCC.db, the test run might hang due to permission request dialogs");
}
else
{
log.WriteLine("Successfully edited TCC.db");
}
}
public async Task <int> RunAsync()
{
CrashReportSnapshot crash_reports;
LogStream device_system_log = null;
LogStream listener_log = null;
Log run_log = main_log;
Initialize();
if (!isSimulator)
{
FindDevice();
}
crash_reports = new CrashReportSnapshot()
{
Device = !isSimulator,
DeviceName = device_name,
Harness = Harness,
Log = main_log,
Logs = Logs,
LogDirectory = LogDirectory,
};
var args = new StringBuilder();
if (!string.IsNullOrEmpty(Harness.XcodeRoot))
{
args.Append(" --sdkroot ").Append(Harness.XcodeRoot);
}
for (int i = -1; i < Harness.Verbosity; i++)
{
args.Append(" -v ");
}
args.Append(" -argument=-connection-mode -argument=none"); // This will prevent the app from trying to connect to any IDEs
args.Append(" -argument=-app-arg:-autostart");
args.Append(" -setenv=NUNIT_AUTOSTART=true");
args.Append(" -argument=-app-arg:-autoexit");
args.Append(" -setenv=NUNIT_AUTOEXIT=true");
args.Append(" -argument=-app-arg:-enablenetwork");
args.Append(" -setenv=NUNIT_ENABLE_NETWORK=true");
// detect if we are using a jenkins bot.
var useXmlOutput = Harness.InJenkins;
if (useXmlOutput)
{
args.Append(" -setenv=NUNIT_ENABLE_XML_OUTPUT=true");
args.Append(" -setenv=NUNIT_ENABLE_XML_MODE=wrapped");
}
if (!IncludeSystemPermissionTests)
{
args.Append(" -setenv=DISABLE_SYSTEM_PERMISSION_TESTS=1");
}
if (isSimulator)
{
args.Append(" -argument=-app-arg:-hostname:127.0.0.1");
args.Append(" -setenv=NUNIT_HOSTNAME=127.0.0.1");
}
else
{
var ips = new StringBuilder();
var ipAddresses = System.Net.Dns.GetHostEntry(System.Net.Dns.GetHostName()).AddressList;
for (int i = 0; i < ipAddresses.Length; i++)
{
if (i > 0)
{
ips.Append(',');
}
ips.Append(ipAddresses [i].ToString());
}
args.AppendFormat(" -argument=-app-arg:-hostname:{0}", ips.ToString());
args.AppendFormat(" -setenv=NUNIT_HOSTNAME={0}", ips.ToString());
}
string transport;
if (mode == "watchos")
{
transport = isSimulator ? "FILE" : "HTTP";
}
else
{
transport = "TCP";
}
args.AppendFormat(" -argument=-app-arg:-transport:{0}", transport);
args.AppendFormat(" -setenv=NUNIT_TRANSPORT={0}", transport);
listener_log = Logs.CreateStream(LogDirectory, string.Format("test-{0}-{1:yyyyMMdd_HHmmss}.log", mode, DateTime.Now), "Test log");
SimpleListener listener;
switch (transport)
{
case "FILE":
var fn = listener_log.FullPath + ".tmp";
listener = new SimpleFileListener(fn);
args.Append(" -setenv=NUNIT_LOG_FILE=").Append(StringUtils.Quote(fn));
break;
case "HTTP":
listener = new SimpleHttpListener();
break;
case "TCP":
listener = new SimpleTcpListener();
break;
default:
throw new NotImplementedException();
}
listener.TestLog = listener_log;
listener.Log = main_log;
listener.AutoExit = true;
listener.Address = System.Net.IPAddress.Any;
listener.XmlOutput = useXmlOutput;
listener.Initialize();
args.AppendFormat(" -argument=-app-arg:-hostport:{0}", listener.Port);
args.AppendFormat(" -setenv=NUNIT_HOSTPORT={0}", listener.Port);
listener.StartAsync();
var cancellation_source = new CancellationTokenSource();
var timed_out = false;
ThreadPool.QueueUserWorkItem((v) =>
{
if (!listener.WaitForConnection(TimeSpan.FromMinutes(Harness.LaunchTimeout)))
{
cancellation_source.Cancel();
main_log.WriteLine("Test launch timed out after {0} minute(s).", Harness.LaunchTimeout);
timed_out = true;
}
else
{
main_log.WriteLine("Test run started");
}
});
foreach (var kvp in Harness.EnvironmentVariables)
{
args.AppendFormat(" -setenv={0}={1}", kvp.Key, kvp.Value);
}
bool?success = null;
bool launch_failure = false;
if (isExtension)
{
switch (extension)
{
case Extension.TodayExtension:
args.Append(isSimulator ? " --launchsimbundleid" : " --launchdevbundleid");
args.Append(" todayviewforextensions:");
args.Append(BundleIdentifier);
args.Append(" --observe-extension ");
args.Append(StringUtils.Quote(launchAppPath));
break;
case Extension.WatchKit2:
default:
throw new NotImplementedException();
}
}
else
{
args.Append(isSimulator ? " --launchsim " : " --launchdev ");
args.Append(StringUtils.Quote(launchAppPath));
}
if (isSimulator)
{
if (!await FindSimulatorAsync())
{
return(1);
}
if (mode != "watchos")
{
var stderr_tty = Marshal.PtrToStringAuto(ttyname(2));
if (!string.IsNullOrEmpty(stderr_tty))
{
args.Append(" --stdout=").Append(StringUtils.Quote(stderr_tty));
args.Append(" --stderr=").Append(StringUtils.Quote(stderr_tty));
}
else
{
var stdout_log = Logs.CreateFile("Standard output", Path.Combine(LogDirectory, "stdout.log"));
var stderr_log = Logs.CreateFile("Standard error", Path.Combine(LogDirectory, "stderr.log"));
args.Append(" --stdout=").Append(StringUtils.Quote(stdout_log.FullPath));
args.Append(" --stderr=").Append(StringUtils.Quote(stderr_log.FullPath));
}
}
var systemLogs = new List <CaptureLog> ();
foreach (var sim in simulators)
{
// Upload the system log
main_log.WriteLine("System log for the '{1}' simulator is: {0}", sim.SystemLog, sim.Name);
bool isCompanion = sim != simulator;
var log = new CaptureLog(sim.SystemLog, entire_file: Harness.Action != HarnessAction.Jenkins)
{
Path = Path.Combine(LogDirectory, sim.Name + ".log"),
Description = isCompanion ? "System log (companion)" : "System log",
};
log.StartCapture();
Logs.Add(log);
systemLogs.Add(log);
Harness.LogWrench("@MonkeyWrench: AddFile: {0}", log.Path);
}
main_log.WriteLine("*** Executing {0}/{1} in the simulator ***", appName, mode);
if (EnsureCleanSimulatorState)
{
foreach (var sim in simulators)
{
await sim.PrepareSimulatorAsync(main_log, bundle_identifier);
}
}
args.Append(" --device=:v2:udid=").Append(simulator.UDID).Append(" ");
await crash_reports.StartCaptureAsync();
main_log.WriteLine("Starting test run");
var result = await ProcessHelper.ExecuteCommandAsync(Harness.MlaunchPath, args.ToString(), run_log, TimeSpan.FromMinutes(Harness.Timeout), cancellation_token : cancellation_source.Token);
if (result.TimedOut)
{
timed_out = true;
success = false;
main_log.WriteLine("Test run timed out after {0} minute(s).", Harness.Timeout);
}
else if (result.Succeeded)
{
main_log.WriteLine("Test run completed");
success = true;
}
else
{
main_log.WriteLine("Test run failed");
success = false;
}
if (!success.Value)
{
// find pid
var pid = -1;
using (var reader = run_log.GetReader()) {
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
if (line.StartsWith("Application launched. PID = ", StringComparison.Ordinal))
{
var pidstr = line.Substring("Application launched. PID = ".Length);
if (!int.TryParse(pidstr, out pid))
{
main_log.WriteLine("Could not parse pid: {0}", pidstr);
}
}
else if (line.Contains("Xamarin.Hosting: Launched ") && line.Contains(" with pid "))
{
var pidstr = line.Substring(line.LastIndexOf(' '));
if (!int.TryParse(pidstr, out pid))
{
main_log.WriteLine("Could not parse pid: {0}", pidstr);
}
}
else if (line.Contains("error MT1008"))
{
launch_failure = true;
}
}
}
if (pid > 0)
{
var launchTimedout = cancellation_source.IsCancellationRequested;
var timeoutType = launchTimedout ? "Launch" : "Completion";
var timeoutValue = launchTimedout ? Harness.LaunchTimeout : Harness.Timeout;
main_log.WriteLine($"{timeoutType} timed out after {timeoutValue}");
await Process_Extensions.KillTreeAsync(pid, main_log, true);
}
else
{
main_log.WriteLine("Could not find pid in mtouch output.");
}
}
listener.Cancel();
// cleanup after us
if (EnsureCleanSimulatorState)
{
await SimDevice.KillEverythingAsync(main_log);
}
foreach (var log in systemLogs)
{
log.StopCapture();
}
}
else
{
main_log.WriteLine("*** Executing {0}/{1} on device '{2}' ***", appName, mode, device_name);
if (mode == "watchos")
{
args.Append(" --attach-native-debugger"); // this prevents the watch from backgrounding the app.
}
else
{
args.Append(" --wait-for-exit");
}
AddDeviceName(args);
device_system_log = Logs.CreateStream(LogDirectory, $"device-{device_name}-{DateTime.Now:yyyyMMdd_HHmmss}.log", "Device log");
var logdev = new DeviceLogCapturer()
{
Harness = Harness,
Log = device_system_log,
DeviceName = device_name,
};
logdev.StartCapture();
await crash_reports.StartCaptureAsync();
main_log.WriteLine("Starting test run");
var result = await ProcessHelper.ExecuteCommandAsync(Harness.MlaunchPath, args.ToString(), main_log, TimeSpan.FromMinutes(Harness.Timeout), cancellation_token : cancellation_source.Token);
if (result.TimedOut)
{
timed_out = true;
success = false;
main_log.WriteLine("Test run timed out after {0} minute(s).", Harness.Timeout);
}
else if (result.Succeeded)
{
main_log.WriteLine("Test run completed");
success = true;
}
else
{
main_log.WriteLine("Test run failed");
success = false;
}
logdev.StopCapture();
// Upload the system log
if (File.Exists(device_system_log.FullPath))
{
main_log.WriteLine("A capture of the device log is: {0}", device_system_log.FullPath);
Harness.LogWrench("@MonkeyWrench: AddFile: {0}", device_system_log.FullPath);
}
}
listener.Dispose();
// check the final status
var crashed = false;
if (File.Exists(listener_log.FullPath))
{
Harness.LogWrench("@MonkeyWrench: AddFile: {0}", listener_log.FullPath);
success = TestsSucceeded(listener_log, timed_out, crashed);
}
else if (timed_out)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} never launched</i></b><br/>", mode);
main_log.WriteLine("Test run never launched");
success = false;
}
else if (launch_failure)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} failed to launch</i></b><br/>", mode);
main_log.WriteLine("Test run failed to launch");
success = false;
}
else
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} crashed at startup (no log)</i></b><br/>", mode);
main_log.WriteLine("Test run crashed before it started (no log file produced)");
crashed = true;
success = false;
}
if (!success.HasValue)
{
success = false;
}
await crash_reports.EndCaptureAsync(TimeSpan.FromSeconds(success.Value ? 0 : 5));
if (timed_out)
{
Result = TestExecutingResult.TimedOut;
}
else if (crashed)
{
Result = TestExecutingResult.Crashed;
}
else if (success.Value)
{
Result = TestExecutingResult.Succeeded;
}
else
{
Result = TestExecutingResult.Failed;
}
// Check crash reports to see if any of them explains why the test run crashed.
if (!success.Value)
{
int pid = 0;
string crash_reason = null;
foreach (var crash in crash_reports.Logs)
{
try {
if (pid == 0)
{
// Find the pid
using (var log_reader = main_log.GetReader()) {
string line;
while ((line = log_reader.ReadLine()) != null)
{
const string str = "was launched with pid '";
var idx = line.IndexOf(str, StringComparison.Ordinal);
if (idx > 0)
{
idx += str.Length;
var next_idx = line.IndexOf('\'', idx);
if (next_idx > idx)
{
int.TryParse(line.Substring(idx, next_idx - idx), out pid);
}
}
if (pid != 0)
{
break;
}
}
}
}
using (var crash_reader = crash.GetReader()) {
var text = crash_reader.ReadToEnd();
var reader = System.Runtime.Serialization.Json.JsonReaderWriterFactory.CreateJsonReader(Encoding.UTF8.GetBytes(text), new XmlDictionaryReaderQuotas());
var doc = new XmlDocument();
doc.Load(reader);
foreach (XmlNode node in doc.SelectNodes($"/root/processes/item[pid = '" + pid + "']"))
{
Console.WriteLine(node?.InnerXml);
Console.WriteLine(node?.SelectSingleNode("reason")?.InnerText);
crash_reason = node?.SelectSingleNode("reason")?.InnerText;
}
}
if (crash_reason != null)
{
break;
}
} catch (Exception e) {
Harness.Log(2, "Failed to process crash report '{1}': {0}", e.Message, crash.Description);
}
}
if (!string.IsNullOrEmpty(crash_reason))
{
if (crash_reason == "per-process-limit")
{
FailureMessage = "Killed due to using too much memory (per-process-limit).";
}
else
{
FailureMessage = $"Killed by the OS ({crash_reason})";
}
}
}
return(success.Value ? 0 : 1);
}
public async Task <int> RunAsync()
{
CrashReportSnapshot crash_reports;
LogStream device_system_log = null;
LogStream listener_log = null;
Log run_log = main_log;
Initialize();
crash_reports = new CrashReportSnapshot()
{
Device = !isSimulator, Harness = Harness, Log = main_log, Logs = Logs, LogDirectory = LogDirectory
};
var args = new StringBuilder();
if (!string.IsNullOrEmpty(Harness.XcodeRoot))
{
args.Append(" --sdkroot ").Append(Harness.XcodeRoot);
}
for (int i = -1; i < Harness.Verbosity; i++)
{
args.Append(" -v ");
}
args.Append(" -argument=-connection-mode -argument=none"); // This will prevent the app from trying to connect to any IDEs
args.Append(" -argument=-app-arg:-autostart");
args.Append(" -setenv=NUNIT_AUTOSTART=true");
args.Append(" -argument=-app-arg:-autoexit");
args.Append(" -setenv=NUNIT_AUTOEXIT=true");
args.Append(" -argument=-app-arg:-enablenetwork");
args.Append(" -setenv=NUNIT_ENABLE_NETWORK=true");
if (isSimulator)
{
args.Append(" -argument=-app-arg:-hostname:127.0.0.1");
args.Append(" -setenv=NUNIT_HOSTNAME=127.0.0.1");
}
else
{
var ips = new StringBuilder();
var ipAddresses = System.Net.Dns.GetHostEntry(System.Net.Dns.GetHostName()).AddressList;
for (int i = 0; i < ipAddresses.Length; i++)
{
if (i > 0)
{
ips.Append(',');
}
ips.Append(ipAddresses [i].ToString());
}
args.AppendFormat(" -argument=-app-arg:-hostname:{0}", ips.ToString());
args.AppendFormat(" -setenv=NUNIT_HOSTNAME={0}", ips.ToString());
}
var transport = mode == "watchos" ? "HTTP" : "TCP";
args.AppendFormat(" -argument=-app-arg:-transport:{0}", transport);
args.AppendFormat(" -setenv=NUNIT_TRANSPORT={0}", transport);
SimpleListener listener;
switch (transport)
{
case "HTTP":
listener = new SimpleHttpListener();
break;
case "TCP":
listener = new SimpleTcpListener();
break;
default:
throw new NotImplementedException();
}
listener_log = Logs.CreateStream(LogDirectory, string.Format("test-{0:yyyyMMdd_HHmmss}.log", DateTime.Now), "Test log");
listener.TestLog = listener_log;
listener.Log = main_log;
listener.AutoExit = true;
listener.Address = System.Net.IPAddress.Any;
listener.Initialize();
args.AppendFormat(" -argument=-app-arg:-hostport:{0}", listener.Port);
args.AppendFormat(" -setenv=NUNIT_HOSTPORT={0}", listener.Port);
foreach (var kvp in Harness.EnvironmentVariables)
{
args.AppendFormat(" -setenv={0}={1}", kvp.Key, kvp.Value);
}
bool?success = null;
bool timed_out = false;
if (isSimulator)
{
FindSimulator();
var systemLogs = new List <CaptureLog> ();
foreach (var sim in simulators)
{
// Upload the system log
main_log.WriteLine("System log for the '{1}' simulator is: {0}", sim.SystemLog, sim.Name);
bool isCompanion = sim != simulator;
var log = new CaptureLog(sim.SystemLog)
{
Path = Path.Combine(LogDirectory, sim.UDID + ".log"),
Description = isCompanion ? "System log (companion)" : "System log",
};
log.StartCapture();
Logs.Add(log);
systemLogs.Add(log);
Harness.LogWrench("@MonkeyWrench: AddFile: {0}", log.Path);
}
main_log.WriteLine("*** Executing {0}/{1} in the simulator ***", appName, mode);
if (EnsureCleanSimulatorState)
{
foreach (var sim in simulators)
{
await sim.PrepareSimulatorAsync(main_log, bundle_identifier);
}
}
args.Append(" --launchsim");
args.AppendFormat(" \"{0}\" ", launchAppPath);
args.Append(" --device=:v2:udid=").Append(simulator.UDID).Append(" ");
await crash_reports.StartCaptureAsync();
listener.StartAsync();
main_log.WriteLine("Starting test run");
var cancellation_source = new CancellationTokenSource();
ThreadPool.QueueUserWorkItem((v) => {
if (!listener.WaitForConnection(TimeSpan.FromMinutes(Harness.LaunchTimeout)))
{
cancellation_source.Cancel();
main_log.WriteLine("Test launch timed out after {0} minute(s).", Harness.LaunchTimeout);
timed_out = true;
}
else
{
main_log.WriteLine("Test run started");
}
});
var result = await ProcessHelper.ExecuteCommandAsync(Harness.MlaunchPath, args.ToString(), run_log, TimeSpan.FromMinutes(Harness.Timeout), cancellation_token : cancellation_source.Token);
if (result.TimedOut)
{
timed_out = true;
success = false;
main_log.WriteLine("Test run timed out after {0} minute(s).", Harness.Timeout);
}
else if (result.Succeeded)
{
main_log.WriteLine("Test run completed");
success = true;
}
else
{
main_log.WriteLine("Test run failed");
success = false;
}
if (!success.Value)
{
// find pid
var pid = -1;
using (var reader = run_log.GetReader()) {
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
if (line.StartsWith("Application launched. PID = ", StringComparison.Ordinal))
{
var pidstr = line.Substring("Application launched. PID = ".Length);
if (!int.TryParse(pidstr, out pid))
{
main_log.WriteLine("Could not parse pid: {0}", pidstr);
}
}
else if (line.Contains("Xamarin.Hosting: Launched ") && line.Contains(" with pid "))
{
var pidstr = line.Substring(line.LastIndexOf(' '));
if (!int.TryParse(pidstr, out pid))
{
main_log.WriteLine("Could not parse pid: {0}", pidstr);
}
}
}
}
if (pid > 0)
{
var launchTimedout = cancellation_source.IsCancellationRequested;
await KillPidAsync(main_log, pid, TimeSpan.FromSeconds(5), TimeSpan.FromMinutes(launchTimedout ? Harness.LaunchTimeout : Harness.Timeout), launchTimedout? "Launch" : "Completion");
}
else
{
main_log.WriteLine("Could not find pid in mtouch output.");
}
}
listener.Cancel();
// cleanup after us
if (EnsureCleanSimulatorState)
{
await SimDevice.KillEverythingAsync(main_log);
}
foreach (var log in systemLogs)
{
log.StopCapture();
}
}
else
{
FindDevice();
main_log.WriteLine("*** Executing {0}/{1} on device ***", appName, mode);
args.Append(" --launchdev");
args.AppendFormat(" \"{0}\" ", launchAppPath);
var waits_for_exit = false;
if (mode == "watchos")
{
args.Append(" --attach-native-debugger"); // this prevents the watch from backgrounding the app.
waits_for_exit = true;
}
AddDeviceName(args);
device_system_log = Logs.CreateStream(LogDirectory, "device.log", "Device log");
var logdev = new DeviceLogCapturer()
{
Harness = Harness,
Log = device_system_log,
DeviceName = device_name,
};
logdev.StartCapture();
await crash_reports.StartCaptureAsync();
listener.StartAsync();
main_log.WriteLine("Starting test run");
double launch_timeout = waits_for_exit ? Harness.Timeout : 1;
double listener_timeout = waits_for_exit ? 0.2 : Harness.Timeout;
await ProcessHelper.ExecuteCommandAsync(Harness.MlaunchPath, args.ToString(), main_log, TimeSpan.FromMinutes(launch_timeout));
if (listener.WaitForCompletion(TimeSpan.FromMinutes(listener_timeout)))
{
main_log.WriteLine("Test run completed");
}
else
{
main_log.WriteLine("Test run did not complete in {0} minutes.", Harness.Timeout);
listener.Cancel();
success = false;
timed_out = true;
}
logdev.StopCapture();
// Upload the system log
if (File.Exists(device_system_log.FullPath))
{
main_log.WriteLine("A capture of the device log is: {0}", device_system_log.FullPath);
if (Harness.InWrench)
{
Harness.LogWrench("@MonkeyWrench: AddFile: {0}", device_system_log.FullPath);
}
}
}
listener.Dispose();
// check the final status
var crashed = false;
if (File.Exists(listener_log.FullPath))
{
Harness.LogWrench("@MonkeyWrench: AddFile: {0}", listener_log.FullPath);
string log;
using (var reader = listener_log.GetReader())
log = reader.ReadToEnd();
if (log.Contains("Tests run"))
{
var tests_run = string.Empty;
var log_lines = log.Split('\n');
var failed = false;
foreach (var line in log_lines)
{
if (line.Contains("Tests run:"))
{
Console.WriteLine(line);
tests_run = line.Replace("Tests run: ", "");
break;
}
else if (line.Contains("FAIL"))
{
Console.WriteLine(line);
failed = true;
}
}
if (failed)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b>{0} failed: {1}</b><br/>", mode, tests_run);
main_log.WriteLine("Test run failed");
success = false;
}
else
{
Harness.LogWrench("@MonkeyWrench: AddSummary: {0} succeeded: {1}<br/>", mode, tests_run);
main_log.WriteLine("Test run succeeded");
success = true;
}
}
else if (timed_out)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} timed out</i></b><br/>", mode);
}
else
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} crashed</i></b><br/>", mode);
main_log.WriteLine("Test run crashed");
crashed = true;
}
}
else if (timed_out)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} never launched</i></b><br/>", mode);
main_log.WriteLine("Test run never launched");
}
else
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} crashed at startup (no log)</i></b><br/>", mode);
main_log.WriteLine("Test run crashed before it started (no log file produced)");
crashed = true;
}
if (!success.HasValue)
{
success = false;
}
await crash_reports.EndCaptureAsync(TimeSpan.FromSeconds(success.Value ? 0 : 5));
if (timed_out)
{
Result = TestExecutingResult.TimedOut;
}
else if (crashed)
{
Result = TestExecutingResult.Crashed;
}
else if (success.Value)
{
Result = TestExecutingResult.Succeeded;
}
else
{
Result = TestExecutingResult.Failed;
}
return(success.Value ? 0 : 1);
}
public static Task KillEverythingAsync(Log log)
{
var to_kill = new string [] { "iPhone Simulator", "iOS Simulator", "Simulator", "Simulator (Watch)", "com.apple.CoreSimulator.CoreSimulatorService" };
return(ProcessHelper.ExecuteCommandAsync("killall", "-9 " + string.Join(" ", to_kill.Select((v) => Harness.Quote(v)).ToArray()), log, TimeSpan.FromSeconds(10)));
}
public async Task AgreeToPromptsAsync(Log log, params string[] bundle_identifiers)
{
if (bundle_identifiers == null || bundle_identifiers.Length == 0)
{
log.WriteLine("No bundle identifiers given when requested permission editing.");
return;
}
var TCC_db = Path.Combine(DataPath, "data", "Library", "TCC", "TCC.db");
var sim_services = new string [] {
"kTCCServiceAddressBook",
"kTCCServiceCalendar",
"kTCCServicePhotos",
"kTCCServiceMediaLibrary",
"kTCCServiceUbiquity",
"kTCCServiceWillow"
};
var failure = false;
var tcc_edit_timeout = 30;
var watch = new Stopwatch();
watch.Start();
do
{
if (failure)
{
log.WriteLine("Failed to edit TCC.db, trying again in 1 second... ", (int)(tcc_edit_timeout - watch.Elapsed.TotalSeconds));
await Task.Delay(TimeSpan.FromSeconds(1));
}
failure = false;
foreach (var bundle_identifier in bundle_identifiers)
{
var sql = new System.Text.StringBuilder();
sql.Append(StringUtils.Quote(TCC_db));
sql.Append(" \"");
foreach (var service in sim_services)
{
switch (TCCFormat)
{
case 1:
// CREATE TABLE access (service TEXT NOT NULL, client TEXT NOT NULL, client_type INTEGER NOT NULL, allowed INTEGER NOT NULL, prompt_count INTEGER NOT NULL, csreq BLOB, CONSTRAINT key PRIMARY KEY (service, client, client_type));
sql.AppendFormat("INSERT INTO access VALUES('{0}','{1}',0,1,0,NULL);", service, bundle_identifier);
sql.AppendFormat("INSERT INTO access VALUES('{0}','{1}',0,1,0,NULL);", service, bundle_identifier + ".watchkitapp");
break;
case 2:
// CREATE TABLE access (service TEXT NOT NULL, client TEXT NOT NULL, client_type INTEGER NOT NULL, allowed INTEGER NOT NULL, prompt_count INTEGER NOT NULL, csreq BLOB, policy_id INTEGER, PRIMARY KEY (service, client, client_type), FOREIGN KEY (policy_id) REFERENCES policies(id) ON DELETE CASCADE ON UPDATE CASCADE);
sql.AppendFormat("INSERT INTO access VALUES('{0}','{1}',0,1,0,NULL,NULL);", service, bundle_identifier);
sql.AppendFormat("INSERT INTO access VALUES('{0}','{1}',0,1,0,NULL,NULL);", service, bundle_identifier + ".watchkitapp");
break;
case 3: // Xcode 10+
// CREATE TABLE access ( service TEXT NOT NULL, client TEXT NOT NULL, client_type INTEGER NOT NULL, allowed INTEGER NOT NULL, prompt_count INTEGER NOT NULL, csreq BLOB, policy_id INTEGER, indirect_object_identifier_type INTEGER, indirect_object_identifier TEXT, indirect_object_code_identity BLOB, flags INTEGER, last_modified INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER)), PRIMARY KEY (service, client, client_type, indirect_object_identifier), FOREIGN KEY (policy_id) REFERENCES policies(id) ON DELETE CASCADE ON UPDATE CASCADE)
sql.AppendFormat("INSERT OR REPLACE INTO access VALUES('{0}','{1}',0,1,0,NULL,NULL,NULL,'UNUSED',NULL,NULL,1);", service, bundle_identifier);
sql.AppendFormat("INSERT OR REPLACE INTO access VALUES('{0}','{1}',0,1,0,NULL,NULL,NULL,'UNUSED',NULL,NULL,1);", service, bundle_identifier + ".watchkitapp");
break;
default:
throw new NotImplementedException();
}
}
sql.Append("\"");
var rv = await ProcessHelper.ExecuteCommandAsync("sqlite3", sql.ToString(), log, TimeSpan.FromSeconds(5));
if (!rv.Succeeded)
{
failure = true;
break;
}
}
} while (failure && watch.Elapsed.TotalSeconds <= tcc_edit_timeout);
if (failure)
{
log.WriteLine("Failed to edit TCC.db, the test run might hang due to permission request dialogs");
}
else
{
log.WriteLine("Successfully edited TCC.db");
}
log.WriteLine("Current TCC database contents:");
await ProcessHelper.ExecuteCommandAsync("sqlite3", $"{StringUtils.Quote (TCC_db)} .dump", log, TimeSpan.FromSeconds(5));
}
public static async Task <ProcessExecutionResult> RunAsync(this Process process, Log log, TextWriter StdoutStream, TextWriter StderrStream, TimeSpan?timeout = null, Dictionary <string, string> environment_variables = null, CancellationToken?cancellation_token = null, bool?diagnostics = null)
{
var stdout_completion = new TaskCompletionSource <bool> ();
var stderr_completion = new TaskCompletionSource <bool> ();
var rv = new ProcessExecutionResult();
process.StartInfo.RedirectStandardError = true;
process.StartInfo.RedirectStandardOutput = true;
process.StartInfo.UseShellExecute = false;
if (environment_variables != null)
{
foreach (var kvp in environment_variables)
{
process.StartInfo.EnvironmentVariables [kvp.Key] = kvp.Value;
}
}
process.OutputDataReceived += (object sender, DataReceivedEventArgs e) =>
{
if (e.Data != null)
{
lock (StdoutStream) {
StdoutStream.WriteLine(e.Data);
StdoutStream.Flush();
}
}
else
{
stdout_completion.TrySetResult(true);
}
};
process.ErrorDataReceived += (object sender, DataReceivedEventArgs e) =>
{
if (e.Data != null)
{
lock (StderrStream) {
StderrStream.WriteLine(e.Data);
StderrStream.Flush();
}
}
else
{
stderr_completion.TrySetResult(true);
}
};
var sb = new StringBuilder();
if (process.StartInfo.EnvironmentVariables != null)
{
var currentEnvironment = Environment.GetEnvironmentVariables().Cast <System.Collections.DictionaryEntry> ().ToDictionary((v) => (string)v.Key, (v) => (string)v.Value, StringComparer.Ordinal);
var processEnvironment = process.StartInfo.EnvironmentVariables.Cast <System.Collections.DictionaryEntry> ().ToDictionary((v) => (string)v.Key, (v) => (string)v.Value, StringComparer.Ordinal);
var allKeys = currentEnvironment.Keys.Union(processEnvironment.Keys).Distinct();
foreach (var key in allKeys)
{
string a = null, b = null;
currentEnvironment.TryGetValue(key, out a);
processEnvironment.TryGetValue(key, out b);
if (a != b)
{
sb.Append($"{key}={StringUtils.Quote (b)} ");
}
}
}
sb.Append($"{StringUtils.Quote (process.StartInfo.FileName)} {process.StartInfo.Arguments}");
log.WriteLine(sb);
process.Start();
var pid = process.Id;
process.BeginErrorReadLine();
process.BeginOutputReadLine();
cancellation_token?.Register(() => {
var hasExited = false;
try {
hasExited = process.HasExited;
} catch {
// Process.HasExited can sometimes throw exceptions, so
// just ignore those and to be safe treat it as the
// process didn't exit (the safe option being to not leave
// processes behind).
}
if (!hasExited)
{
StderrStream.WriteLine($"Execution of {pid} was cancelled.");
ProcessHelper.kill(pid, 9);
}
});
if (timeout.HasValue)
{
if (!await process.WaitForExitAsync(timeout.Value))
{
await process.KillTreeAsync(log, diagnostics ?? true);
rv.TimedOut = true;
lock (StderrStream)
log.WriteLine($"{pid} Execution timed out after {timeout.Value.TotalSeconds} seconds and the process was killed.");
}
}
await process.WaitForExitAsync();
Task.WaitAll(new Task [] { stderr_completion.Task, stdout_completion.Task }, TimeSpan.FromSeconds(1));
try {
rv.ExitCode = process.ExitCode;
} catch (Exception e) {
rv.ExitCode = 12345678;
log.WriteLine($"Failed to get ExitCode: {e}");
}
return(rv);
}
public static async Task <ProcessExecutionResult> RunAsync(this Process process, Log log, TextWriter StdoutStream, TextWriter StderrStream, TimeSpan?timeout = null, Dictionary <string, string> environment_variables = null, CancellationToken?cancellation_token = null)
{
var stdout_completion = new TaskCompletionSource <bool> ();
var stderr_completion = new TaskCompletionSource <bool> ();
var exit_completion = new TaskCompletionSource <bool> ();
var rv = new ProcessExecutionResult();
process.StartInfo.RedirectStandardError = true;
process.StartInfo.RedirectStandardOutput = true;
process.StartInfo.UseShellExecute = false;
if (environment_variables != null)
{
foreach (var kvp in environment_variables)
{
process.StartInfo.EnvironmentVariables [kvp.Key] = kvp.Value;
}
}
process.OutputDataReceived += (object sender, DataReceivedEventArgs e) =>
{
if (e.Data != null)
{
lock (StdoutStream) {
StdoutStream.WriteLine(e.Data);
StdoutStream.Flush();
}
}
else
{
stdout_completion.TrySetResult(true);
}
};
process.ErrorDataReceived += (object sender, DataReceivedEventArgs e) =>
{
if (e.Data != null)
{
lock (StderrStream) {
StderrStream.WriteLine(e.Data);
StderrStream.Flush();
}
}
else
{
stderr_completion.TrySetResult(true);
}
};
var sb = new StringBuilder();
if (process.StartInfo.EnvironmentVariables != null)
{
var currentEnvironment = Environment.GetEnvironmentVariables().Cast <System.Collections.DictionaryEntry> ().ToDictionary((v) => (string)v.Key, (v) => (string)v.Value, StringComparer.Ordinal);
var processEnvironment = process.StartInfo.EnvironmentVariables.Cast <System.Collections.DictionaryEntry> ().ToDictionary((v) => (string)v.Key, (v) => (string)v.Value, StringComparer.Ordinal);
var allKeys = currentEnvironment.Keys.Union(processEnvironment.Keys).Distinct();
foreach (var key in allKeys)
{
string a = null, b = null;
currentEnvironment.TryGetValue(key, out a);
processEnvironment.TryGetValue(key, out b);
if (a != b)
{
sb.Append($"{key}={StringUtils.Quote (b)} ");
}
}
}
sb.Append($"{StringUtils.Quote (process.StartInfo.FileName)} {process.StartInfo.Arguments}");
log.WriteLine(sb);
process.Start();
process.BeginErrorReadLine();
process.BeginOutputReadLine();
cancellation_token?.Register(() => {
if (!exit_completion.Task.IsCompleted)
{
StderrStream.WriteLine($"Execution was cancelled.");
ProcessHelper.kill(process.Id, 9);
}
});
new Thread(() =>
{
if (timeout.HasValue)
{
if (!process.WaitForExit((int)timeout.Value.TotalMilliseconds))
{
process.KillTreeAsync(log, true).Wait();
rv.TimedOut = true;
lock (StderrStream)
log.WriteLine($"Execution timed out after {timeout.Value.TotalSeconds} seconds and the process was killed.");
}
}
process.WaitForExit();
exit_completion.TrySetResult(true);
Task.WaitAll(new Task [] { stderr_completion.Task, stdout_completion.Task }, TimeSpan.FromSeconds(1));
stderr_completion.TrySetResult(false);
stdout_completion.TrySetResult(false);
})
{
IsBackground = true,
}.Start();
await Task.WhenAll(stderr_completion.Task, stdout_completion.Task, exit_completion.Task);
try {
rv.ExitCode = process.ExitCode;
} catch (Exception e) {
rv.ExitCode = 12345678;
log.WriteLine($"Failed to get ExitCode: {e}");
}
return(rv);
}
