public static BuildTargetResult CheckPackageCache(BuildTargetContext c) { var ciBuild = string.Equals(Environment.GetEnvironmentVariable("CI_BUILD"), "1", StringComparison.Ordinal); // Always set the package cache location local to the build Environment.SetEnvironmentVariable("NUGET_PACKAGES", Dirs.NuGetPackages); CleanNuGetTempCache(); // Determine cache expiration time var cacheExpiration = 7 * 24; // cache expiration in hours var cacheExpirationStr = Environment.GetEnvironmentVariable("NUGET_PACKAGES_CACHE_TIME_LIMIT"); if (!string.IsNullOrEmpty(cacheExpirationStr)) { cacheExpiration = int.Parse(cacheExpirationStr); } if (ciBuild) { var cacheTimeFile = Path.Combine(Dirs.NuGetPackages, "packageCacheTime.txt"); DateTime?cacheTime = null; try { // Read the cache file if (File.Exists(cacheTimeFile)) { var content = File.ReadAllText(cacheTimeFile); if (!string.IsNullOrEmpty(content)) { cacheTime = DateTime.ParseExact("O", content, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal); } } } catch (Exception ex) { c.Warn($"Error reading NuGet cache time file, leaving the cache alone"); c.Warn($"Error Detail: {ex.ToString()}"); } if (cacheTime == null || (cacheTime.Value.AddHours(cacheExpiration) < DateTime.UtcNow)) { // Cache has expired or the status is unknown, clear it and write the file c.Info("Clearing NuGet cache"); Rmdir(Dirs.NuGetPackages); Mkdirp(Dirs.NuGetPackages); File.WriteAllText(cacheTimeFile, DateTime.UtcNow.ToString("O")); } } return(c.Success()); }
private static BuildTargetResult CopyAppDeps(BuildTargetContext c, string outputDir) { var appDepOutputDir = Path.Combine(outputDir, "appdepsdk"); Rmdir(appDepOutputDir); Mkdirp(appDepOutputDir); // Find toolchain package string packageId; if (CurrentPlatform.IsWindows) { if (CurrentArchitecture.Isx86) { // https://github.com/dotnet/cli/issues/1550 c.Warn("Native compilation is not yet working on Windows x86"); return(c.Success()); } packageId = "toolchain.win7-x64.Microsoft.DotNet.AppDep"; } else if (CurrentPlatform.IsUbuntu) { packageId = "toolchain.ubuntu.14.04-x64.Microsoft.DotNet.AppDep"; } else if (CurrentPlatform.IsCentOS || CurrentPlatform.IsRHEL || CurrentPlatform.IsDebian) { c.Warn($"Native compilation is not yet working on {CurrentPlatform.Current}"); return(c.Success()); } else if (CurrentPlatform.IsOSX) { packageId = "toolchain.osx.10.10-x64.Microsoft.DotNet.AppDep"; } else { return(c.Failed("Unsupported OS Platform")); } var appDepPath = Path.Combine( Dirs.NuGetPackages, packageId, AppDepSdkVersion); CopyRecursive(appDepPath, appDepOutputDir, overwrite: true); return(c.Success()); }
public static BuildTargetResult CopySharedHostLayout(BuildTargetContext c) { var sharedHostRoot = Path.Combine(Dirs.Output, "obj", "sharedHost"); if (Directory.Exists(sharedHostRoot)) { Utils.DeleteDirectory(sharedHostRoot); } Directory.CreateDirectory(sharedHostRoot); foreach (var file in Directory.GetFiles(Dirs.SharedFrameworkPublish, "*", SearchOption.TopDirectoryOnly)) { var destFile = file.Replace(Dirs.SharedFrameworkPublish, sharedHostRoot); File.Copy(file, destFile, true); c.Warn(destFile); } FixPermissions(sharedHostRoot); File.Copy( Path.Combine(Dirs.RepoRoot, "resources", "ThirdPartyNotices.txt"), Path.Combine(sharedHostRoot, "ThirdPartyNotices.txt")); File.Copy( Path.Combine(Dirs.RepoRoot, "resources", "LICENSE.txt"), Path.Combine(sharedHostRoot, "LICENSE.txt")); c.BuildContext["SharedHostPublishRoot"] = sharedHostRoot; return(c.Success()); }
public static BuildTargetResult CrossgenSharedFx(BuildTargetContext c, string pathToAssemblies) { // Check if we need to skip crossgen if (string.Equals(Environment.GetEnvironmentVariable("DONT_CROSSGEN_SHAREDFRAMEWORK"), "1")) { c.Warn("Skipping crossgen for SharedFx because DONT_CROSSGEN_SHAREDFRAMEWORK is set to 1"); return(c.Success()); } foreach (var file in Directory.GetFiles(pathToAssemblies)) { string fileName = Path.GetFileName(file); if (fileName == "mscorlib.dll" || fileName == "mscorlib.ni.dll" || !HasMetadata(file)) { continue; } string tempPathName = Path.ChangeExtension(file, "readytorun"); // This is not always correct. The version of crossgen we need to pick up is whatever one was restored as part // of the Microsoft.NETCore.Runtime.CoreCLR package that is part of the shared library. For now, the version hardcoded // in CompileTargets and the one in the shared library project.json match and are updated in lock step, but long term // we need to be able to look at the project.lock.json file and figure out what version of Microsoft.NETCore.Runtime.CoreCLR // was used, and then select that version. ExecSilent(Crossgen.GetCrossgenPathForVersion(CompileTargets.CoreCLRVersion), "-readytorun", "-in", file, "-out", tempPathName, "-platform_assemblies_paths", pathToAssemblies); File.Delete(file); File.Move(tempPathName, file); } return(c.Success()); }
public static BuildTargetResult ReplaceProjectJson(BuildTargetContext c) { List <DependencyInfo> dependencyInfos = c.GetDependencyInfos(); const string noUpdateFileName = ".noautoupdate"; IEnumerable <string> projectJsonFiles = Enumerable.Union( Directory.GetFiles(Dirs.RepoRoot, "project.json", SearchOption.AllDirectories), Directory.GetFiles(Path.Combine(Dirs.RepoRoot, @"src\dotnet\commands\dotnet-new"), "project.json.template", SearchOption.AllDirectories)) .Where(p => !File.Exists(Path.Combine(Path.GetDirectoryName(p), noUpdateFileName)) && !Path.GetDirectoryName(p).EndsWith("CSharp_Web", StringComparison.Ordinal)); JObject projectRoot; foreach (string projectJsonFile in projectJsonFiles) { try { projectRoot = ReadProject(projectJsonFile); } catch (Exception e) { c.Warn($"Non-fatal exception occurred reading '{projectJsonFile}'. Skipping file. Exception: {e}. "); continue; } if (projectRoot == null) { c.Warn($"A non valid JSON file was encountered '{projectJsonFile}'. Skipping file."); continue; } bool changedAnyPackage = FindAllDependencyProperties(projectRoot) .Select(dependencyProperty => ReplaceDependencyVersion(dependencyProperty, dependencyInfos)) .ToArray() .Any(shouldWrite => shouldWrite); if (changedAnyPackage) { c.Info($"Writing changes to {projectJsonFile}"); WriteProject(projectRoot, projectJsonFile); } } return(c.Success()); }
public static BuildTargetResult ReplaceProjectJson(BuildTargetContext c) { List <DependencyInfo> dependencyInfos = c.GetDependencyInfos(); var projectJsonFiles = new List <string> { Path.Combine(Dirs.PkgProjects, "Microsoft.NETCore.App", "project.json.template"), Path.Combine(Dirs.PkgProjects, "Microsoft.NETCore.UniversalWindowsPlatform", "project.json.template"), Path.Combine(Dirs.PkgDeps, "project.json") }; //projectJsonFiles.AddRange(Directory.GetFiles(Dirs.RepoRoot, "project.json", SearchOption.AllDirectories)); JObject projectRoot; foreach (string projectJsonFile in projectJsonFiles) { try { projectRoot = ReadProject(projectJsonFile); } catch (Exception e) { c.Warn($"Non-fatal exception occurred reading '{projectJsonFile}'. Skipping file. Exception: {e}. "); continue; } if (projectRoot == null) { c.Warn($"A non valid JSON file was encountered '{projectJsonFile}'. Skipping file."); continue; } bool changedAnyPackage = FindAllDependencyProperties(projectRoot) .Select(dependencyProperty => ReplaceDependencyVersion(dependencyProperty, dependencyInfos)) .ToArray() .Any(shouldWrite => shouldWrite); if (changedAnyPackage) { c.Info($"Writing changes to {projectJsonFile}"); WriteProject(projectRoot, projectJsonFile); } } return(c.Success()); }
public void CrossgenDirectory(BuildTargetContext c, string pathToAssemblies) { // Check if we need to skip crossgen if (string.Equals(Environment.GetEnvironmentVariable("DISABLE_CROSSGEN"), "1")) { c.Warn("Skipping crossgen for because DISABLE_CROSSGEN is set to 1"); return; } string sharedFxPath = c.BuildContext.Get <string>("SharedFrameworkPath"); // HACK // The input directory can be a portable FAT app (example the CLI itself). // In that case there can be RID specific managed dependencies which are not right next to the app binary (example System.Diagnostics.TraceSource). // We need those dependencies during crossgen. For now we just pass all subdirectories of the input directory as input to crossgen. // The right fix - // If the assembly has deps.json then parse the json file to get all the dependencies, pass these dependencies as input to crossgen. // else pass the current directory of assembly as input to crossgen. var addtionalPaths = Directory.GetDirectories(pathToAssemblies, "*", SearchOption.AllDirectories).ToList(); var paths = new List <string>() { sharedFxPath, pathToAssemblies }; paths.AddRange(addtionalPaths); var platformAssembliesPaths = string.Join(Path.PathSeparator.ToString(), paths.Distinct()); var env = new Dictionary <string, string>() { // disable partial ngen { "COMPlus_PartialNGen", "0" } }; foreach (var file in Directory.GetFiles(pathToAssemblies)) { string fileName = Path.GetFileName(file); if (s_excludedLibraries.Any(lib => String.Equals(lib, fileName, StringComparison.OrdinalIgnoreCase)) || !PEUtils.HasMetadata(file)) { continue; } string tempPathName = Path.ChangeExtension(file, "readytorun"); IList <string> crossgenArgs = new List <string> { "-readytorun", "-in", file, "-out", tempPathName, "-JITPath", GetLibCLRJitPathForVersion(), "-platform_assemblies_paths", platformAssembliesPaths }; ExecSilent(_crossGenPath, crossgenArgs, env); File.Copy(tempPathName, file, overwrite: true); File.Delete(tempPathName); } }
public static BuildTargetResult ReplaceProjectJson(BuildTargetContext c) { List <DependencyInfo> dependencyInfos = c.GetDependencyInfos(); IEnumerable <string> projectJsonFiles = Directory.GetFiles(Dirs.RepoRoot, "project.json", SearchOption.AllDirectories); JObject projectRoot; foreach (string projectJsonFile in projectJsonFiles) { try { projectRoot = ReadProject(projectJsonFile); } catch (Exception e) { c.Warn($"Non-fatal exception occurred reading '{projectJsonFile}'. Skipping file. Exception: {e}. "); continue; } if (projectRoot == null) { c.Warn($"A non valid JSON file was encountered '{projectJsonFile}'. Skipping file."); continue; } bool changedAnyPackage = FindAllDependencyProperties(projectRoot) .Select(dependencyProperty => ReplaceDependencyVersion(dependencyProperty, dependencyInfos)) .ToArray() .Any(shouldWrite => shouldWrite); if (changedAnyPackage) { c.Info($"Writing changes to {projectJsonFile}"); WriteProject(projectRoot, projectJsonFile); } } return(c.Success()); }
private static BuildTargetResult CopyAppDeps(BuildTargetContext c, string outputDir) { var appDepOutputDir = Path.Combine(outputDir, "appdepsdk"); Rmdir(appDepOutputDir); Mkdirp(appDepOutputDir); // Find toolchain package string packageId; if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { packageId = "toolchain.win7-x64.Microsoft.DotNet.AppDep"; } else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { var osname = PlatformServices.Default.Runtime.OperatingSystem; if (string.Equals(osname, "ubuntu", StringComparison.OrdinalIgnoreCase)) { packageId = "toolchain.ubuntu.14.04-x64.Microsoft.DotNet.AppDep"; } else if (string.Equals(osname, "centos", StringComparison.OrdinalIgnoreCase)) { c.Warn("Native compilation is not yet working on CentOS"); return(c.Success()); } else { return(c.Failed($"Unknown Linux Distro: {osname}")); } } else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { packageId = "toolchain.osx.10.10-x64.Microsoft.DotNet.AppDep"; } else { return(c.Failed("Unsupported OS Platform")); } var appDepPath = Path.Combine( Dirs.NuGetPackages, packageId, AppDepSdkVersion); CopyRecursive(appDepPath, appDepOutputDir, overwrite: true); return(c.Success()); }
public static BuildTargetResult CheckPackageCache(BuildTargetContext c) { var ciBuild = string.Equals(Environment.GetEnvironmentVariable("CI_BUILD"), "1", StringComparison.Ordinal); if (ciBuild) { // On CI, HOME is redirected under the repo, which gets deleted after every build. // So make NUGET_PACKAGES outside of the repo. var nugetPackages = Path.GetFullPath(Path.Combine(c.BuildContext.BuildDirectory, "..", ".nuget", "packages")); Environment.SetEnvironmentVariable("NUGET_PACKAGES", nugetPackages); Dirs.NuGetPackages = nugetPackages; } // Set the package cache location in NUGET_PACKAGES just to be safe if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("NUGET_PACKAGES"))) { Environment.SetEnvironmentVariable("NUGET_PACKAGES", Dirs.NuGetPackages); } CleanNuGetTempCache(); // Determine cache expiration time var cacheExpiration = 7 * 24; // cache expiration in hours var cacheExpirationStr = Environment.GetEnvironmentVariable("NUGET_PACKAGES_CACHE_TIME_LIMIT"); if (!string.IsNullOrEmpty(cacheExpirationStr)) { cacheExpiration = int.Parse(cacheExpirationStr); } if (ciBuild) { var cacheTimeFile = Path.Combine(Dirs.NuGetPackages, "packageCacheTime.txt"); DateTime?cacheTime = null; try { // Read the cache file if (File.Exists(cacheTimeFile)) { var content = File.ReadAllText(cacheTimeFile); if (!string.IsNullOrEmpty(content)) { cacheTime = DateTime.ParseExact("O", content, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal); } } } catch (Exception ex) { c.Warn($"Error reading NuGet cache time file, leaving the cache alone"); c.Warn($"Error Detail: {ex.ToString()}"); } if (cacheTime == null || (cacheTime.Value.AddHours(cacheExpiration) < DateTime.UtcNow)) { // Cache has expired or the status is unknown, clear it and write the file c.Info("Clearing NuGet cache"); Rmdir(Dirs.NuGetPackages); Mkdirp(Dirs.NuGetPackages); File.WriteAllText(cacheTimeFile, DateTime.UtcNow.ToString("O")); } } return(c.Success()); }
public static BuildTargetResult CommitChanges(BuildTargetContext c) { CommandResult statusResult = Cmd("git", "status", "--porcelain") .CaptureStdOut() .Execute(); statusResult.EnsureSuccessful(); bool hasModifiedFiles = !string.IsNullOrWhiteSpace(statusResult.StdOut); bool hasUpdatedDependencies = c.GetDependencyInfos().Where(d => d.IsUpdated).Any(); if (hasModifiedFiles != hasUpdatedDependencies) { return(c.Failed($"'git status' does not match DependencyInfo information. Git has modified files: {hasModifiedFiles}. DependencyInfo is updated: {hasUpdatedDependencies}.")); } if (!hasUpdatedDependencies) { c.Warn("Dependencies are currently up to date"); return(c.Success()); } string userName = s_config.UserName; string email = s_config.Email; string commitMessage = GetCommitMessage(c); Cmd("git", "commit", "-a", "-m", commitMessage, "--author", $"{userName} <{email}>") .EnvironmentVariable("GIT_COMMITTER_NAME", userName) .EnvironmentVariable("GIT_COMMITTER_EMAIL", email) .Execute() .EnsureSuccessful(); string remoteUrl = $"github.com/{s_config.GitHubOriginOwner}/{s_config.GitHubProject}.git"; string remoteBranchName = $"UpdateDependencies{DateTime.UtcNow.ToString("yyyyMMddhhmmss")}"; string refSpec = $"HEAD:refs/heads/{remoteBranchName}"; string logMessage = $"git push https://{remoteUrl} {refSpec}"; BuildReporter.BeginSection("EXEC", logMessage); CommandResult pushResult = Cmd("git", "push", $"https://{userName}:{s_config.Password}@{remoteUrl}", refSpec) .QuietBuildReporter() // we don't want secrets showing up in our logs .CaptureStdErr() // git push will write to StdErr upon success, disable that .CaptureStdOut() .Execute(); var message = logMessage + $" exited with {pushResult.ExitCode}"; if (pushResult.ExitCode == 0) { BuildReporter.EndSection("EXEC", message.Green(), success: true); } else { BuildReporter.EndSection("EXEC", message.Red().Bold(), success: false); } pushResult.EnsureSuccessful(suppressOutput: true); c.SetRemoteBranchName(remoteBranchName); return(c.Success()); }
private static BuildTargetResult Crossgen(BuildTargetContext c, string outputDir) { // Check if we need to skip crossgen if (string.Equals(Environment.GetEnvironmentVariable("DOTNET_BUILD_SKIP_CROSSGEN"), "1")) { c.Warn("Skipping crossgen because DOTNET_BUILD_SKIP_CROSSGEN is set"); return(c.Success()); } // Find crossgen string packageId; if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { packageId = "runtime.win7-x64.Microsoft.NETCore.Runtime.CoreCLR"; } else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { var osname = PlatformServices.Default.Runtime.OperatingSystem; if (string.Equals(osname, "ubuntu", StringComparison.OrdinalIgnoreCase)) { packageId = "runtime.ubuntu.14.04-x64.Microsoft.NETCore.Runtime.CoreCLR"; } else if (string.Equals(osname, "centos", StringComparison.OrdinalIgnoreCase)) { // CentOS runtime is in the runtime.rhel.7-x64... package. packageId = "runtime.rhel.7-x64.Microsoft.NETCore.Runtime.CoreCLR"; } else { return(c.Failed($"Unknown Linux Distro: {osname}")); } } else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { packageId = "runtime.osx.10.10-x64.Microsoft.NETCore.Runtime.CoreCLR"; } else { return(c.Failed("Unsupported OS Platform")); } var crossGenExePath = Path.Combine( Dirs.NuGetPackages, packageId, CoreCLRVersion, "tools", $"crossgen{Constants.ExeSuffix}"); // We have to copy crossgen next to mscorlib var crossgen = Path.Combine(outputDir, $"crossgen{Constants.ExeSuffix}"); File.Copy(crossGenExePath, crossgen, overwrite: true); Chmod(crossgen, "a+x"); // And if we have mscorlib.ni.dll, we need to rename it to mscorlib.dll if (File.Exists(Path.Combine(outputDir, "mscorlib.ni.dll"))) { File.Copy(Path.Combine(outputDir, "mscorlib.ni.dll"), Path.Combine(outputDir, "mscorlib.dll"), overwrite: true); } foreach (var assemblyToCrossgen in AssembliesToCrossGen) { c.Info($"Crossgenning {assemblyToCrossgen}"); ExecInSilent(outputDir, crossgen, "-nologo", "-platform_assemblies_paths", outputDir, assemblyToCrossgen); } c.Info("Crossgen complete"); // Check if csc/vbc.ni.exe exists, and overwrite the dll with it just in case if (File.Exists(Path.Combine(outputDir, "csc.ni.exe")) && !File.Exists(Path.Combine(outputDir, "csc.ni.dll"))) { File.Move(Path.Combine(outputDir, "csc.ni.exe"), Path.Combine(outputDir, "csc.ni.dll")); } if (File.Exists(Path.Combine(outputDir, "vbc.ni.exe")) && !File.Exists(Path.Combine(outputDir, "vbc.ni.dll"))) { File.Move(Path.Combine(outputDir, "vbc.ni.exe"), Path.Combine(outputDir, "vbc.ni.dll")); } return(c.Success()); }