public GitHub(Harness harness, IProcessManager processManager)
{
if (harness == null)
{
throw new ArgumentNullException(nameof(harness));
}
if (processManager == null)
{
throw new ArgumentNullException(nameof(processManager));
}
this.harness = harness;
this.processManager = processManager;
}
public static IEnumerable <string> GetLabels(Harness harness, int pull_request)
{
var info = DownloadPullRequestInfo(harness, pull_request);
using (var reader = JsonReaderWriterFactory.CreateJsonReader(info, new XmlDictionaryReaderQuotas())) {
var doc = new XmlDocument();
doc.Load(reader);
var rv = new List <string> ();
foreach (XmlNode node in doc.SelectNodes("/root/labels/item/name"))
{
rv.Add(node.InnerText);
}
return(rv);
}
}
void CreateWatchOSLibraryProject()
{
var csproj = inputProject;
csproj.SetProjectTypeGuids("{FC940695-DFE0-4552-9F25-99AF4A5619A1};" + LanguageGuid);
csproj.SetOutputPath("bin\\$(Platform)\\$(Configuration)" + Suffix);
csproj.SetIntermediateOutputPath("obj\\$(Platform)\\$(Configuration)" + Suffix);
csproj.RemoveTargetFrameworkIdentifier();
csproj.SetPlatformAssembly("Xamarin.WatchOS");
csproj.SetImport(IsBindingProject ? BindingsImports : Imports);
csproj.AddAdditionalDefines("XAMCORE_2_0;XAMCORE_3_0;MONOTOUCH_WATCH;");
csproj.FixProjectReferences(Suffix);
csproj.SetExtraLinkerDefs("extra-linker-defs" + ExtraLinkerDefsSuffix + ".xml");
csproj.FixTestLibrariesReferences(Platform);
Harness.Save(csproj, WatchOSProjectPath);
WatchOSGuid = csproj.GetProjectGuid();
}
public static string GetPullRequestTargetBranch(Harness harness, int pull_request)
{
if (pull_request <= 0)
{
return(string.Empty);
}
var info = DownloadPullRequestInfo(harness, pull_request);
if (info.Length == 0)
{
return(string.Empty);
}
using (var reader = JsonReaderWriterFactory.CreateJsonReader(info, new XmlDictionaryReaderQuotas())) {
var doc = new XmlDocument();
doc.Load(reader);
return(doc.SelectSingleNode("/root/base/ref").InnerText);
}
}
public void Convert()
{
var inputProject = new XmlDocument();
var xml = File.ReadAllText(TemplatePath);
inputProject.LoadXmlWithoutNetworkAccess(xml);
inputProject.SetOutputPath("bin\\$(Platform)\\$(Configuration)" + FlavorSuffix);
inputProject.SetIntermediateOutputPath("obj\\$(Platform)\\$(Configuration)" + FlavorSuffix);
inputProject.SetAssemblyName(inputProject.GetAssemblyName() + FlavorSuffix);
var template_info_plist = Path.Combine(Path.GetDirectoryName(TemplatePath), inputProject.GetInfoPListInclude());
var target_info_plist = Path.Combine(Path.GetDirectoryName(template_info_plist), "Info" + TemplateSuffix + FlavorSuffix + ".plist");
SetInfoPListMinimumOSVersion(template_info_plist, target_info_plist);
inputProject.FixInfoPListInclude(FlavorSuffix, newName: Path.GetFileName(target_info_plist));
AddProjectDefines(inputProject);
Harness.Save(inputProject, ProjectPath);
}
protected override void ExecuteInternal()
{
if (MonoNativeInfo == null)
{
return;
}
MonoNativeInfo.AddProjectDefines(inputProject);
inputProject.AddAdditionalDefines("MONO_NATIVE_IOS");
inputProject.FixInfoPListInclude(Suffix);
inputProject.SetExtraLinkerDefs("extra-linker-defs" + ExtraLinkerDefsSuffix + ".xml");
Harness.Save(inputProject, ProjectPath);
XmlDocument info_plist = new XmlDocument();
var target_info_plist = Path.Combine(TargetDirectory, "Info" + Suffix + ".plist");
info_plist.LoadWithoutNetworkAccess(Path.Combine(TargetDirectory, "Info.plist"));
info_plist.SetMinimumOSVersion(GetMinimumOSVersion(info_plist.GetMinimumOSVersion()));
Harness.Save(info_plist, target_info_plist);
}
static byte[] DownloadPullRequestInfo(Harness harness, int pull_request)
{
var path = Path.Combine(harness.LogDirectory, "pr" + pull_request + ".log");
if (!File.Exists(path))
{
Directory.CreateDirectory(harness.LogDirectory);
using (var client = CreateClient()) {
byte [] data;
try {
data = client.DownloadData($"https://api.github.com/repos/xamarin/xamarin-macios/pulls/{pull_request}");
File.WriteAllBytes(path, data);
return(data);
} catch (WebException we) {
harness.Log("Could not load pull request info: {0}\n{1}", we, new StreamReader(we.Response.GetResponseStream()).ReadToEnd());
File.WriteAllText(path, string.Empty);
return(new byte [0]);
}
}
}
return(File.ReadAllBytes(path));
}
public static IEnumerable <string> GetModifiedFiles(Harness harness, int pull_request)
{
var path = Path.Combine(harness.LogDirectory, "pr" + pull_request + "-files.log");
if (!File.Exists(path))
{
var rv = GetModifiedFilesLocally(harness, pull_request);
if (rv == null || rv.Count() == 0)
{
rv = GetModifiedFilesRemotely(harness, pull_request);
if (rv == null)
{
rv = new string [] { }
}
;
}
File.WriteAllLines(path, rv.ToArray());
return(rv);
}
return(File.ReadAllLines(path));
}
static IEnumerable <string> GetModifiedFilesRemotely(Harness harness, int pull_request)
{
var path = Path.Combine(harness.LogDirectory, "pr" + pull_request + "-remote-files.log");
if (!File.Exists(path))
{
Directory.CreateDirectory(harness.LogDirectory);
using (var client = CreateClient()) {
var rv = new List <string> ();
var url = $"https://api.github.com/repos/xamarin/xamarin-macios/pulls/{pull_request}/files?per_page=100"; // 100 items per page is max
do
{
byte [] data;
try {
data = client.DownloadData(url);
} catch (WebException we) {
harness.Log("Could not load pull request files: {0}\n{1}", we, new StreamReader(we.Response.GetResponseStream()).ReadToEnd());
File.WriteAllText(path, string.Empty);
return(new string [] { });
}
var reader = JsonReaderWriterFactory.CreateJsonReader(data, new XmlDictionaryReaderQuotas());
var doc = new XmlDocument();
doc.Load(reader);
foreach (XmlNode node in doc.SelectNodes("/root/item/filename"))
{
rv.Add(node.InnerText);
}
url = null;
var link = client.ResponseHeaders ["Link"];
try {
if (link != null)
{
var ltIdx = link.IndexOf('<');
var gtIdx = link.IndexOf('>', ltIdx + 1);
while (ltIdx >= 0 && gtIdx > ltIdx)
{
var linkUrl = link.Substring(ltIdx + 1, gtIdx - ltIdx - 1);
if (link [gtIdx + 1] != ';')
{
break;
}
var commaIdx = link.IndexOf(',', gtIdx + 1);
string rel;
if (commaIdx != -1)
{
rel = link.Substring(gtIdx + 3, commaIdx - gtIdx - 3);
}
else
{
rel = link.Substring(gtIdx + 3);
}
if (rel == "rel=\"next\"")
{
url = linkUrl;
break;
}
if (commaIdx == -1)
{
break;
}
ltIdx = link.IndexOf('<', commaIdx);
gtIdx = link.IndexOf('>', ltIdx + 1);
}
}
} catch (Exception e) {
harness.Log("Could not paginate github response: {0}: {1}", link, e.Message);
}
} while (url != null);
File.WriteAllLines(path, rv.ToArray());
return(rv);
}
}
return(File.ReadAllLines(path));
}
public async Task StartCaptureAsync()
{
InitialSet = await Harness.CreateCrashReportsSnapshotAsync(Log, !Device, DeviceName);
}
public async Task EndCaptureAsync(TimeSpan timeout)
{
// Check for crash reports
var crash_report_search_done = false;
var crash_report_search_timeout = timeout.TotalSeconds;
var watch = new Stopwatch();
watch.Start();
do
{
var end_crashes = await Harness.CreateCrashReportsSnapshotAsync(Log, !Device, DeviceName);
end_crashes.ExceptWith(InitialSet);
Reports = end_crashes;
if (end_crashes.Count > 0)
{
Log.WriteLine("Found {0} new crash report(s)", end_crashes.Count);
List <ILogFile> crash_reports;
if (!Device)
{
crash_reports = new List <ILogFile> (end_crashes.Count);
foreach (var path in end_crashes)
{
Logs.AddFile(path, $"Crash report: {Path.GetFileName (path)}");
}
}
else
{
// Download crash reports from the device. We put them in the project directory so that they're automatically deleted on wrench
// (if we put them in /tmp, they'd never be deleted).
var downloaded_crash_reports = new List <ILogFile> ();
foreach (var file in end_crashes)
{
var name = Path.GetFileName(file);
var crash_report_target = Logs.Create(name, $"Crash report: {name}", timestamp: false);
var sb = new List <string> ();
sb.Add($"--download-crash-report={file}");
sb.Add($"--download-crash-report-to={crash_report_target.Path}");
sb.Add("--sdkroot");
sb.Add(Harness.XcodeRoot);
if (!string.IsNullOrEmpty(DeviceName))
{
sb.Add("--devname");
sb.Add(DeviceName);
}
var result = await ProcessHelper.ExecuteCommandAsync(Harness.MlaunchPath, sb, Log, TimeSpan.FromMinutes(1));
if (result.Succeeded)
{
Log.WriteLine("Downloaded crash report {0} to {1}", file, crash_report_target.Path);
crash_report_target = await Harness.SymbolicateCrashReportAsync(Logs, Log, crash_report_target);
downloaded_crash_reports.Add(crash_report_target);
}
else
{
Log.WriteLine("Could not download crash report {0}", file);
}
}
crash_reports = downloaded_crash_reports;
}
foreach (var cp in crash_reports)
{
Harness.LogWrench("@MonkeyWrench: AddFile: {0}", cp.Path);
Log.WriteLine(" {0}", cp.Path);
}
crash_report_search_done = true;
}
else
{
if (watch.Elapsed.TotalSeconds > crash_report_search_timeout)
{
crash_report_search_done = true;
}
else
{
Log.WriteLine("No crash reports, waiting a second to see if the crash report service just didn't complete in time ({0})", (int)(crash_report_search_timeout - watch.Elapsed.TotalSeconds));
Thread.Sleep(TimeSpan.FromSeconds(1));
}
}
} while (!crash_report_search_done);
}
public static void CreateSolution(Harness harness, IEnumerable <Target> targets, Target exeTarget, string infix)
{
var folders = new StringBuilder();
var srcDirectory = Path.Combine(Harness.RootDirectory, "..", "src");
var sln_path = exeTarget == null?Path.Combine(Harness.RootDirectory, "tests-" + infix + ".sln") : Path.Combine(Path.GetDirectoryName(exeTarget.ProjectPath), Path.GetFileNameWithoutExtension(exeTarget.ProjectPath) + ".sln");
using (var writer = new StringWriter()) {
writer.WriteLine();
writer.WriteLine("Microsoft Visual Studio Solution File, Format Version 11.00");
writer.WriteLine("# Visual Studio 2010");
foreach (var target in targets)
{
var relatedProjects = target.GetRelatedProjects();
var hasRelatedProjects = relatedProjects != null;
var folderGuid = string.Empty;
var useFolders = hasRelatedProjects && target.IsExe && exeTarget == null;
if (hasRelatedProjects && target.IsExe)
{
if (exeTarget == null)
{
CreateSolution(harness, targets, target, infix); // create a solution for just this test project as well
}
else if (exeTarget != target)
{
continue;
}
}
if (useFolders)
{
folderGuid = Guid.NewGuid().ToString().ToUpperInvariant();
writer.WriteLine("Project(\"{{2150E333-8FDC-42A3-9474-1A3956D46DE8}}\") = \"{0}\", \"{0}\", \"{{{1}}}\"", target.Name, folderGuid);
writer.WriteLine("EndProject");
}
writer.WriteLine("Project(\"{3}\") = \"{0}\", \"{1}\", \"{2}\"", Path.GetFileNameWithoutExtension(target.ProjectPath), FixProjectPath(sln_path, Path.GetFullPath(target.ProjectPath)), target.ProjectGuid, target.LanguageGuid);
writer.WriteLine("EndProject");
if (hasRelatedProjects && target.IsExe)
{
foreach (var rp in relatedProjects)
{
writer.WriteLine("Project(\"{3}\") = \"{0}\", \"{1}\", \"{2}\"", Path.GetFileNameWithoutExtension(rp.ProjectPath), FixProjectPath(sln_path, Path.GetFullPath(rp.ProjectPath)), rp.Guid, target.LanguageGuid);
writer.WriteLine("EndProject");
}
}
if (useFolders)
{
folders.AppendFormat("\t\t{0} = {{{1}}}\n", target.ProjectGuid, folderGuid);
foreach (var rp in relatedProjects)
{
folders.AppendFormat("\t\t{0} = {{{1}}}\n", rp.Guid, folderGuid);
}
}
}
// Add reference to MonoTouch.NUnitLite project
string configuration;
var proj_path = Path.GetFullPath(Path.Combine(srcDirectory, "MonoTouch.NUnitLite." + infix + ".csproj"));
if (!File.Exists(proj_path))
{
proj_path = Path.GetFullPath(Path.Combine(srcDirectory, "MonoTouch.NUnitLite.csproj"));
}
AddProjectToSolution(harness, sln_path, writer, proj_path, out configuration);
writer.WriteLine("Global");
writer.WriteLine("\tGlobalSection(SolutionConfigurationPlatforms) = preSolution");
writer.WriteLine("\t\tDebug|iPhoneSimulator = Debug|iPhoneSimulator");
writer.WriteLine("\t\tRelease|iPhoneSimulator = Release|iPhoneSimulator");
writer.WriteLine("\t\tDebug|iPhone = Debug|iPhone");
writer.WriteLine("\t\tRelease|iPhone = Release|iPhone");
writer.WriteLine("\t\tRelease-bitcode|iPhone = Release-bitcode|iPhone");
writer.WriteLine("\t\tDebug|Any CPU = Debug|Any CPU");
writer.WriteLine("\t\tRelease|Any CPU = Release|Any CPU");
writer.WriteLine("\tEndGlobalSection");
writer.WriteLine("\tGlobalSection(ProjectConfigurationPlatforms) = postSolution");
var exePlatforms = new string[] { "iPhone", "iPhoneSimulator" };
var configurations = new string[] { "Debug", "Release", "Release-bitcode" };
foreach (var target in targets)
{
if (target.IsExe && exeTarget != null && target != exeTarget)
{
continue;
}
foreach (var conf in configurations)
{
if (target.IsExe)
{
foreach (var platform in exePlatforms)
{
writer.WriteLine("\t\t{0}.{1}|{2}.ActiveCfg = {1}|{2}", target.ProjectGuid, conf, platform);
writer.WriteLine("\t\t{0}.{1}|{2}.Build.0 = {1}|{2}", target.ProjectGuid, conf, platform);
}
}
else
{
foreach (var platform in new string[] { "Any CPU", "iPhone", "iPhoneSimulator" })
{
writer.WriteLine("\t\t{0}.{1}|{2}.ActiveCfg = {1}|Any CPU", target.ProjectGuid, conf, platform);
writer.WriteLine("\t\t{0}.{1}|{2}.Build.0 = {1}|Any CPU", target.ProjectGuid, conf, platform);
}
}
}
if (target.IsExe)
{
var relatedProjects = target.GetRelatedProjects();
if (relatedProjects != null)
{
foreach (var rp in relatedProjects)
{
foreach (var conf in configurations)
{
foreach (var platform in exePlatforms)
{
writer.WriteLine("\t\t{0}.{1}|{2}.ActiveCfg = {1}|{2}", rp.Guid, conf, platform);
writer.WriteLine("\t\t{0}.{1}|{2}.Build.0 = {1}|{2}", rp.Guid, conf, platform);
}
}
}
}
}
}
writer.Write(configuration);
writer.WriteLine("\tEndGlobalSection");
if (folders.Length > 0)
{
writer.WriteLine("\tGlobalSection(NestedProjects) = preSolution");
writer.Write(folders.ToString());
writer.WriteLine("\tEndGlobalSection");
}
writer.WriteLine("EndGlobal");
harness.Save(writer, sln_path);
}
}
public static void CreateSolution(Harness harness, IEnumerable <Target> targets, string infix)
{
CreateSolution(harness, targets, null, infix);
}
public MacMonoNativeInfo(Harness harness, MonoNativeFlavor flavor)
: base(harness, flavor)
{
}
static IEnumerable <string> GetModifiedFilesLocally(IProcessManager processManager, Harness harness, int pull_request)
{
var base_commit = $"origin/pr/{pull_request}/merge^";
var head_commit = $"origin/pr/{pull_request}/merge";
harness.Log("Fetching modified files for commit range {0}..{1}", base_commit, head_commit);
if (string.IsNullOrEmpty(head_commit) || string.IsNullOrEmpty(base_commit))
{
return(null);
}
using (var git = new Process()) {
git.StartInfo.FileName = "git";
git.StartInfo.Arguments = $"diff-tree --no-commit-id --name-only -r {base_commit}..{head_commit}";
var output = new MemoryLog();
var rv = processManager.RunAsync(git, harness.HarnessLog, stdoutLog: output, stderrLog: output).Result;
if (rv.Succeeded)
{
return(output.ToString().Split(new char [] { '\n' }, StringSplitOptions.RemoveEmptyEntries));
}
harness.Log("Could not fetch commit range:");
harness.Log(output.ToString());
return(null);
}
}
public MonoNativeInfo(Harness harness, MonoNativeFlavor flavor)
{
Harness = harness;
Flavor = flavor;
}
void CreateWatchOSExtensionProject()
{
var csproj = inputProject;
var suffix = Suffix + "-extension";
// Remove unused configurations
csproj.DeleteConfiguration("iPhone", "Release-bitcode");
csproj.DeleteConfiguration("iPhone", "Release64");
csproj.DeleteConfiguration("iPhone", "Debug64");
csproj.FixArchitectures("i386", "ARMv7k", "iPhone", "Release32");
csproj.FixArchitectures("i386", "ARMv7k", "iPhone", "Debug32");
// add Release64_32 and set the correct architecture
csproj.CloneConfiguration("iPhone", "Release", "Release64_32");
csproj.FixArchitectures("i386", "ARM64_32", "iPhone", "Release64_32");
// add Debug64_32 and set the correct architecture
csproj.CloneConfiguration("iPhone", "Debug", "Debug64_32");
csproj.FixArchitectures("i386", "ARM64_32", "iPhone", "Debug64_32");
csproj.FixArchitectures(SimulatorArchitectures, DeviceArchitectures, "iPhoneSimulator", "Debug");
csproj.FixArchitectures(SimulatorArchitectures, DeviceArchitectures, "iPhoneSimulator", "Release");
csproj.FixArchitectures(SimulatorArchitectures, DeviceArchitectures, "iPhone", "Debug");
csproj.FixArchitectures(SimulatorArchitectures, DeviceArchitectures, "iPhone", "Release");
csproj.SetProjectTypeGuids("{1E2E965C-F6D2-49ED-B86E-418A60C69EEF};" + LanguageGuid);
csproj.SetOutputPath("bin\\$(Platform)\\$(Configuration)" + suffix);
csproj.SetIntermediateOutputPath("obj\\$(Platform)\\$(Configuration)" + suffix);
csproj.RemoveTargetFrameworkIdentifier();
csproj.SetPlatformAssembly("Xamarin.WatchOS");
csproj.SetImport(IsFSharp ? "$(MSBuildExtensionsPath)\\Xamarin\\WatchOS\\Xamarin.WatchOS.AppExtension.FSharp.targets" : "$(MSBuildExtensionsPath)\\Xamarin\\WatchOS\\Xamarin.WatchOS.AppExtension.CSharp.targets");
csproj.FixProjectReferences("-watchos");
csproj.FixInfoPListInclude(suffix);
csproj.SetOutputType("Library");
csproj.AddAdditionalDefines("BITCODE", "iPhone", "Release");
csproj.AddAdditionalDefines("XAMCORE_2_0;XAMCORE_3_0;FEATURE_NO_BSD_SOCKETS;MONOTOUCH_WATCH;");
csproj.RemoveReferences("OpenTK-1.0");
var ext = IsFSharp ? "fs" : "cs";
csproj.AddCompileInclude("InterfaceController." + ext, Path.Combine(Harness.WatchOSExtensionTemplate, "InterfaceController." + ext));
csproj.SetExtraLinkerDefs("extra-linker-defs" + ExtraLinkerDefsSuffix + ".xml");
csproj.SetMtouchUseBitcode(true, "iPhone", "Release");
csproj.SetMtouchUseLlvm(true, "iPhone", "Release");
if (MonoNativeInfo != null)
{
csproj.AddAdditionalDefines("MONO_NATIVE_WATCH");
MonoNativeHelper.AddProjectDefines(csproj, MonoNativeInfo.Flavor);
MonoNativeHelper.RemoveSymlinkMode(csproj);
}
// Not linking a watch extensions requires passing -Os to the native compiler.
// https://github.com/mono/mono/issues/9867
var configurations = new string [] { "Debug", "Debug32", "Debug64_32", "Release", "Release32", "Release64_32" };
foreach (var c in configurations)
{
var flags = "-fembed-bitcode-marker";
if (csproj.GetMtouchLink("iPhone", c) == "None")
{
flags += " -Os";
}
csproj.AddExtraMtouchArgs($"--gcc_flags='{flags}'", "iPhone", c);
}
Harness.Save(csproj, WatchOSExtensionProjectPath);
WatchOSExtensionGuid = csproj.GetProjectGuid();
XmlDocument info_plist = new XmlDocument();
var target_info_plist = Path.Combine(TargetDirectory, $"Info{Suffix}-extension.plist");
info_plist.LoadWithoutNetworkAccess(Path.Combine(TargetDirectory, "Info.plist"));
BundleIdentifier = info_plist.GetCFBundleIdentifier() + "_watch";
if (BundleIdentifier.Length >= 58)
{
BundleIdentifier = BundleIdentifier.Substring(0, 57); // If the main app's bundle id is 58 characters (or sometimes more), then the watch extension crashes at launch. radar #29847128.
}
info_plist.SetCFBundleIdentifier(BundleIdentifier + ".watchkitapp.watchkitextension");
info_plist.SetMinimumOSVersion(GetMinimumOSVersion("2.0"));
info_plist.SetUIDeviceFamily(4);
info_plist.AddPListStringValue("RemoteInterfacePrincipleClass", "InterfaceController");
info_plist.AddPListKeyValuePair("NSExtension", "dict", string.Format(
@"
<key>NSExtensionAttributes</key>
<dict>
<key>WKAppBundleIdentifier</key>
<string>{0}.watchkitapp</string>
</dict>
<key>NSExtensionPointIdentifier</key>
<string>com.apple.watchkit</string>
", BundleIdentifier));
if (!info_plist.ContainsKey("NSAppTransportSecurity"))
{
info_plist.AddPListKeyValuePair("NSAppTransportSecurity", "dict",
@"
<key>NSAllowsArbitraryLoads</key>
<true/>
" );
}
Harness.Save(info_plist, target_info_plist);
}
