public static BuildTargetResult CheckPackageCache(BuildTargetContext c)
{
var ciBuild = string.Equals(Environment.GetEnvironmentVariable("CI_BUILD"), "1", StringComparison.Ordinal);
// Always set the package cache location local to the build
Environment.SetEnvironmentVariable("NUGET_PACKAGES", Dirs.NuGetPackages);
CleanNuGetTempCache();
// Determine cache expiration time
var cacheExpiration = 7 * 24; // cache expiration in hours
var cacheExpirationStr = Environment.GetEnvironmentVariable("NUGET_PACKAGES_CACHE_TIME_LIMIT");
if (!string.IsNullOrEmpty(cacheExpirationStr))
{
cacheExpiration = int.Parse(cacheExpirationStr);
}
if (ciBuild)
{
var cacheTimeFile = Path.Combine(Dirs.NuGetPackages, "packageCacheTime.txt");
DateTime?cacheTime = null;
try
{
// Read the cache file
if (File.Exists(cacheTimeFile))
{
var content = File.ReadAllText(cacheTimeFile);
if (!string.IsNullOrEmpty(content))
{
cacheTime = DateTime.ParseExact("O", content, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal);
}
}
}
catch (Exception ex)
{
c.Warn($"Error reading NuGet cache time file, leaving the cache alone");
c.Warn($"Error Detail: {ex.ToString()}");
}
if (cacheTime == null || (cacheTime.Value.AddHours(cacheExpiration) < DateTime.UtcNow))
{
// Cache has expired or the status is unknown, clear it and write the file
c.Info("Clearing NuGet cache");
Rmdir(Dirs.NuGetPackages);
Mkdirp(Dirs.NuGetPackages);
File.WriteAllText(cacheTimeFile, DateTime.UtcNow.ToString("O"));
}
}
return(c.Success());
}
private static BuildTargetResult CopyAppDeps(BuildTargetContext c, string outputDir)
{
var appDepOutputDir = Path.Combine(outputDir, "appdepsdk");
Rmdir(appDepOutputDir);
Mkdirp(appDepOutputDir);
// Find toolchain package
string packageId;
if (CurrentPlatform.IsWindows)
{
if (CurrentArchitecture.Isx86)
{
// https://github.com/dotnet/cli/issues/1550
c.Warn("Native compilation is not yet working on Windows x86");
return(c.Success());
}
packageId = "toolchain.win7-x64.Microsoft.DotNet.AppDep";
}
else if (CurrentPlatform.IsUbuntu)
{
packageId = "toolchain.ubuntu.14.04-x64.Microsoft.DotNet.AppDep";
}
else if (CurrentPlatform.IsCentOS || CurrentPlatform.IsRHEL || CurrentPlatform.IsDebian)
{
c.Warn($"Native compilation is not yet working on {CurrentPlatform.Current}");
return(c.Success());
}
else if (CurrentPlatform.IsOSX)
{
packageId = "toolchain.osx.10.10-x64.Microsoft.DotNet.AppDep";
}
else
{
return(c.Failed("Unsupported OS Platform"));
}
var appDepPath = Path.Combine(
Dirs.NuGetPackages,
packageId,
AppDepSdkVersion);
CopyRecursive(appDepPath, appDepOutputDir, overwrite: true);
return(c.Success());
}
public static BuildTargetResult CopySharedHostLayout(BuildTargetContext c)
{
var sharedHostRoot = Path.Combine(Dirs.Output, "obj", "sharedHost");
if (Directory.Exists(sharedHostRoot))
{
Utils.DeleteDirectory(sharedHostRoot);
}
Directory.CreateDirectory(sharedHostRoot);
foreach (var file in Directory.GetFiles(Dirs.SharedFrameworkPublish, "*", SearchOption.TopDirectoryOnly))
{
var destFile = file.Replace(Dirs.SharedFrameworkPublish, sharedHostRoot);
File.Copy(file, destFile, true);
c.Warn(destFile);
}
FixPermissions(sharedHostRoot);
File.Copy(
Path.Combine(Dirs.RepoRoot, "resources", "ThirdPartyNotices.txt"),
Path.Combine(sharedHostRoot, "ThirdPartyNotices.txt"));
File.Copy(
Path.Combine(Dirs.RepoRoot, "resources", "LICENSE.txt"),
Path.Combine(sharedHostRoot, "LICENSE.txt"));
c.BuildContext["SharedHostPublishRoot"] = sharedHostRoot;
return(c.Success());
}
public static BuildTargetResult CrossgenSharedFx(BuildTargetContext c, string pathToAssemblies)
{
// Check if we need to skip crossgen
if (string.Equals(Environment.GetEnvironmentVariable("DONT_CROSSGEN_SHAREDFRAMEWORK"), "1"))
{
c.Warn("Skipping crossgen for SharedFx because DONT_CROSSGEN_SHAREDFRAMEWORK is set to 1");
return(c.Success());
}
foreach (var file in Directory.GetFiles(pathToAssemblies))
{
string fileName = Path.GetFileName(file);
if (fileName == "mscorlib.dll" || fileName == "mscorlib.ni.dll" || !HasMetadata(file))
{
continue;
}
string tempPathName = Path.ChangeExtension(file, "readytorun");
// This is not always correct. The version of crossgen we need to pick up is whatever one was restored as part
// of the Microsoft.NETCore.Runtime.CoreCLR package that is part of the shared library. For now, the version hardcoded
// in CompileTargets and the one in the shared library project.json match and are updated in lock step, but long term
// we need to be able to look at the project.lock.json file and figure out what version of Microsoft.NETCore.Runtime.CoreCLR
// was used, and then select that version.
ExecSilent(Crossgen.GetCrossgenPathForVersion(CompileTargets.CoreCLRVersion),
"-readytorun", "-in", file, "-out", tempPathName, "-platform_assemblies_paths", pathToAssemblies);
File.Delete(file);
File.Move(tempPathName, file);
}
return(c.Success());
}
public static BuildTargetResult ReplaceProjectJson(BuildTargetContext c)
{
List <DependencyInfo> dependencyInfos = c.GetDependencyInfos();
const string noUpdateFileName = ".noautoupdate";
IEnumerable <string> projectJsonFiles = Enumerable.Union(
Directory.GetFiles(Dirs.RepoRoot, "project.json", SearchOption.AllDirectories),
Directory.GetFiles(Path.Combine(Dirs.RepoRoot, @"src\dotnet\commands\dotnet-new"), "project.json.template", SearchOption.AllDirectories))
.Where(p => !File.Exists(Path.Combine(Path.GetDirectoryName(p), noUpdateFileName)) &&
!Path.GetDirectoryName(p).EndsWith("CSharp_Web", StringComparison.Ordinal));
JObject projectRoot;
foreach (string projectJsonFile in projectJsonFiles)
{
try
{
projectRoot = ReadProject(projectJsonFile);
}
catch (Exception e)
{
c.Warn($"Non-fatal exception occurred reading '{projectJsonFile}'. Skipping file. Exception: {e}. ");
continue;
}
if (projectRoot == null)
{
c.Warn($"A non valid JSON file was encountered '{projectJsonFile}'. Skipping file.");
continue;
}
bool changedAnyPackage = FindAllDependencyProperties(projectRoot)
.Select(dependencyProperty => ReplaceDependencyVersion(dependencyProperty, dependencyInfos))
.ToArray()
.Any(shouldWrite => shouldWrite);
if (changedAnyPackage)
{
c.Info($"Writing changes to {projectJsonFile}");
WriteProject(projectRoot, projectJsonFile);
}
}
return(c.Success());
}
public static BuildTargetResult ReplaceProjectJson(BuildTargetContext c)
{
List <DependencyInfo> dependencyInfos = c.GetDependencyInfos();
var projectJsonFiles = new List <string>
{
Path.Combine(Dirs.PkgProjects, "Microsoft.NETCore.App", "project.json.template"),
Path.Combine(Dirs.PkgProjects, "Microsoft.NETCore.UniversalWindowsPlatform", "project.json.template"),
Path.Combine(Dirs.PkgDeps, "project.json")
};
//projectJsonFiles.AddRange(Directory.GetFiles(Dirs.RepoRoot, "project.json", SearchOption.AllDirectories));
JObject projectRoot;
foreach (string projectJsonFile in projectJsonFiles)
{
try
{
projectRoot = ReadProject(projectJsonFile);
}
catch (Exception e)
{
c.Warn($"Non-fatal exception occurred reading '{projectJsonFile}'. Skipping file. Exception: {e}. ");
continue;
}
if (projectRoot == null)
{
c.Warn($"A non valid JSON file was encountered '{projectJsonFile}'. Skipping file.");
continue;
}
bool changedAnyPackage = FindAllDependencyProperties(projectRoot)
.Select(dependencyProperty => ReplaceDependencyVersion(dependencyProperty, dependencyInfos))
.ToArray()
.Any(shouldWrite => shouldWrite);
if (changedAnyPackage)
{
c.Info($"Writing changes to {projectJsonFile}");
WriteProject(projectRoot, projectJsonFile);
}
}
return(c.Success());
}
public void CrossgenDirectory(BuildTargetContext c, string pathToAssemblies)
{
// Check if we need to skip crossgen
if (string.Equals(Environment.GetEnvironmentVariable("DISABLE_CROSSGEN"), "1"))
{
c.Warn("Skipping crossgen for because DISABLE_CROSSGEN is set to 1");
return;
}
string sharedFxPath = c.BuildContext.Get <string>("SharedFrameworkPath");
// HACK
// The input directory can be a portable FAT app (example the CLI itself).
// In that case there can be RID specific managed dependencies which are not right next to the app binary (example System.Diagnostics.TraceSource).
// We need those dependencies during crossgen. For now we just pass all subdirectories of the input directory as input to crossgen.
// The right fix -
// If the assembly has deps.json then parse the json file to get all the dependencies, pass these dependencies as input to crossgen.
// else pass the current directory of assembly as input to crossgen.
var addtionalPaths = Directory.GetDirectories(pathToAssemblies, "*", SearchOption.AllDirectories).ToList();
var paths = new List <string>()
{
sharedFxPath, pathToAssemblies
};
paths.AddRange(addtionalPaths);
var platformAssembliesPaths = string.Join(Path.PathSeparator.ToString(), paths.Distinct());
var env = new Dictionary <string, string>()
{
// disable partial ngen
{ "COMPlus_PartialNGen", "0" }
};
foreach (var file in Directory.GetFiles(pathToAssemblies))
{
string fileName = Path.GetFileName(file);
if (s_excludedLibraries.Any(lib => String.Equals(lib, fileName, StringComparison.OrdinalIgnoreCase)) ||
!PEUtils.HasMetadata(file))
{
continue;
}
string tempPathName = Path.ChangeExtension(file, "readytorun");
IList <string> crossgenArgs = new List <string> {
"-readytorun", "-in", file, "-out", tempPathName,
"-JITPath", GetLibCLRJitPathForVersion(),
"-platform_assemblies_paths", platformAssembliesPaths
};
ExecSilent(_crossGenPath, crossgenArgs, env);
File.Copy(tempPathName, file, overwrite: true);
File.Delete(tempPathName);
}
}
public static BuildTargetResult ReplaceProjectJson(BuildTargetContext c)
{
List <DependencyInfo> dependencyInfos = c.GetDependencyInfos();
IEnumerable <string> projectJsonFiles = Directory.GetFiles(Dirs.RepoRoot, "project.json", SearchOption.AllDirectories);
JObject projectRoot;
foreach (string projectJsonFile in projectJsonFiles)
{
try
{
projectRoot = ReadProject(projectJsonFile);
}
catch (Exception e)
{
c.Warn($"Non-fatal exception occurred reading '{projectJsonFile}'. Skipping file. Exception: {e}. ");
continue;
}
if (projectRoot == null)
{
c.Warn($"A non valid JSON file was encountered '{projectJsonFile}'. Skipping file.");
continue;
}
bool changedAnyPackage = FindAllDependencyProperties(projectRoot)
.Select(dependencyProperty => ReplaceDependencyVersion(dependencyProperty, dependencyInfos))
.ToArray()
.Any(shouldWrite => shouldWrite);
if (changedAnyPackage)
{
c.Info($"Writing changes to {projectJsonFile}");
WriteProject(projectRoot, projectJsonFile);
}
}
return(c.Success());
}
private static BuildTargetResult CopyAppDeps(BuildTargetContext c, string outputDir)
{
var appDepOutputDir = Path.Combine(outputDir, "appdepsdk");
Rmdir(appDepOutputDir);
Mkdirp(appDepOutputDir);
// Find toolchain package
string packageId;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
packageId = "toolchain.win7-x64.Microsoft.DotNet.AppDep";
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
var osname = PlatformServices.Default.Runtime.OperatingSystem;
if (string.Equals(osname, "ubuntu", StringComparison.OrdinalIgnoreCase))
{
packageId = "toolchain.ubuntu.14.04-x64.Microsoft.DotNet.AppDep";
}
else if (string.Equals(osname, "centos", StringComparison.OrdinalIgnoreCase))
{
c.Warn("Native compilation is not yet working on CentOS");
return(c.Success());
}
else
{
return(c.Failed($"Unknown Linux Distro: {osname}"));
}
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
packageId = "toolchain.osx.10.10-x64.Microsoft.DotNet.AppDep";
}
else
{
return(c.Failed("Unsupported OS Platform"));
}
var appDepPath = Path.Combine(
Dirs.NuGetPackages,
packageId,
AppDepSdkVersion);
CopyRecursive(appDepPath, appDepOutputDir, overwrite: true);
return(c.Success());
}
public static BuildTargetResult CheckPackageCache(BuildTargetContext c)
{
var ciBuild = string.Equals(Environment.GetEnvironmentVariable("CI_BUILD"), "1", StringComparison.Ordinal);
if (ciBuild)
{
// On CI, HOME is redirected under the repo, which gets deleted after every build.
// So make NUGET_PACKAGES outside of the repo.
var nugetPackages = Path.GetFullPath(Path.Combine(c.BuildContext.BuildDirectory, "..", ".nuget", "packages"));
Environment.SetEnvironmentVariable("NUGET_PACKAGES", nugetPackages);
Dirs.NuGetPackages = nugetPackages;
}
// Set the package cache location in NUGET_PACKAGES just to be safe
if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("NUGET_PACKAGES")))
{
Environment.SetEnvironmentVariable("NUGET_PACKAGES", Dirs.NuGetPackages);
}
CleanNuGetTempCache();
// Determine cache expiration time
var cacheExpiration = 7 * 24; // cache expiration in hours
var cacheExpirationStr = Environment.GetEnvironmentVariable("NUGET_PACKAGES_CACHE_TIME_LIMIT");
if (!string.IsNullOrEmpty(cacheExpirationStr))
{
cacheExpiration = int.Parse(cacheExpirationStr);
}
if (ciBuild)
{
var cacheTimeFile = Path.Combine(Dirs.NuGetPackages, "packageCacheTime.txt");
DateTime?cacheTime = null;
try
{
// Read the cache file
if (File.Exists(cacheTimeFile))
{
var content = File.ReadAllText(cacheTimeFile);
if (!string.IsNullOrEmpty(content))
{
cacheTime = DateTime.ParseExact("O", content, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal);
}
}
}
catch (Exception ex)
{
c.Warn($"Error reading NuGet cache time file, leaving the cache alone");
c.Warn($"Error Detail: {ex.ToString()}");
}
if (cacheTime == null || (cacheTime.Value.AddHours(cacheExpiration) < DateTime.UtcNow))
{
// Cache has expired or the status is unknown, clear it and write the file
c.Info("Clearing NuGet cache");
Rmdir(Dirs.NuGetPackages);
Mkdirp(Dirs.NuGetPackages);
File.WriteAllText(cacheTimeFile, DateTime.UtcNow.ToString("O"));
}
}
return(c.Success());
}
public static BuildTargetResult CommitChanges(BuildTargetContext c)
{
CommandResult statusResult = Cmd("git", "status", "--porcelain")
.CaptureStdOut()
.Execute();
statusResult.EnsureSuccessful();
bool hasModifiedFiles = !string.IsNullOrWhiteSpace(statusResult.StdOut);
bool hasUpdatedDependencies = c.GetDependencyInfos().Where(d => d.IsUpdated).Any();
if (hasModifiedFiles != hasUpdatedDependencies)
{
return(c.Failed($"'git status' does not match DependencyInfo information. Git has modified files: {hasModifiedFiles}. DependencyInfo is updated: {hasUpdatedDependencies}."));
}
if (!hasUpdatedDependencies)
{
c.Warn("Dependencies are currently up to date");
return(c.Success());
}
string userName = s_config.UserName;
string email = s_config.Email;
string commitMessage = GetCommitMessage(c);
Cmd("git", "commit", "-a", "-m", commitMessage, "--author", $"{userName} <{email}>")
.EnvironmentVariable("GIT_COMMITTER_NAME", userName)
.EnvironmentVariable("GIT_COMMITTER_EMAIL", email)
.Execute()
.EnsureSuccessful();
string remoteUrl = $"github.com/{s_config.GitHubOriginOwner}/{s_config.GitHubProject}.git";
string remoteBranchName = $"UpdateDependencies{DateTime.UtcNow.ToString("yyyyMMddhhmmss")}";
string refSpec = $"HEAD:refs/heads/{remoteBranchName}";
string logMessage = $"git push https://{remoteUrl} {refSpec}";
BuildReporter.BeginSection("EXEC", logMessage);
CommandResult pushResult =
Cmd("git", "push", $"https://{userName}:{s_config.Password}@{remoteUrl}", refSpec)
.QuietBuildReporter() // we don't want secrets showing up in our logs
.CaptureStdErr() // git push will write to StdErr upon success, disable that
.CaptureStdOut()
.Execute();
var message = logMessage + $" exited with {pushResult.ExitCode}";
if (pushResult.ExitCode == 0)
{
BuildReporter.EndSection("EXEC", message.Green(), success: true);
}
else
{
BuildReporter.EndSection("EXEC", message.Red().Bold(), success: false);
}
pushResult.EnsureSuccessful(suppressOutput: true);
c.SetRemoteBranchName(remoteBranchName);
return(c.Success());
}
private static BuildTargetResult Crossgen(BuildTargetContext c, string outputDir)
{
// Check if we need to skip crossgen
if (string.Equals(Environment.GetEnvironmentVariable("DOTNET_BUILD_SKIP_CROSSGEN"), "1"))
{
c.Warn("Skipping crossgen because DOTNET_BUILD_SKIP_CROSSGEN is set");
return(c.Success());
}
// Find crossgen
string packageId;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
packageId = "runtime.win7-x64.Microsoft.NETCore.Runtime.CoreCLR";
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
var osname = PlatformServices.Default.Runtime.OperatingSystem;
if (string.Equals(osname, "ubuntu", StringComparison.OrdinalIgnoreCase))
{
packageId = "runtime.ubuntu.14.04-x64.Microsoft.NETCore.Runtime.CoreCLR";
}
else if (string.Equals(osname, "centos", StringComparison.OrdinalIgnoreCase))
{
// CentOS runtime is in the runtime.rhel.7-x64... package.
packageId = "runtime.rhel.7-x64.Microsoft.NETCore.Runtime.CoreCLR";
}
else
{
return(c.Failed($"Unknown Linux Distro: {osname}"));
}
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
packageId = "runtime.osx.10.10-x64.Microsoft.NETCore.Runtime.CoreCLR";
}
else
{
return(c.Failed("Unsupported OS Platform"));
}
var crossGenExePath = Path.Combine(
Dirs.NuGetPackages,
packageId,
CoreCLRVersion,
"tools",
$"crossgen{Constants.ExeSuffix}");
// We have to copy crossgen next to mscorlib
var crossgen = Path.Combine(outputDir, $"crossgen{Constants.ExeSuffix}");
File.Copy(crossGenExePath, crossgen, overwrite: true);
Chmod(crossgen, "a+x");
// And if we have mscorlib.ni.dll, we need to rename it to mscorlib.dll
if (File.Exists(Path.Combine(outputDir, "mscorlib.ni.dll")))
{
File.Copy(Path.Combine(outputDir, "mscorlib.ni.dll"), Path.Combine(outputDir, "mscorlib.dll"), overwrite: true);
}
foreach (var assemblyToCrossgen in AssembliesToCrossGen)
{
c.Info($"Crossgenning {assemblyToCrossgen}");
ExecInSilent(outputDir, crossgen, "-nologo", "-platform_assemblies_paths", outputDir, assemblyToCrossgen);
}
c.Info("Crossgen complete");
// Check if csc/vbc.ni.exe exists, and overwrite the dll with it just in case
if (File.Exists(Path.Combine(outputDir, "csc.ni.exe")) && !File.Exists(Path.Combine(outputDir, "csc.ni.dll")))
{
File.Move(Path.Combine(outputDir, "csc.ni.exe"), Path.Combine(outputDir, "csc.ni.dll"));
}
if (File.Exists(Path.Combine(outputDir, "vbc.ni.exe")) && !File.Exists(Path.Combine(outputDir, "vbc.ni.dll")))
{
File.Move(Path.Combine(outputDir, "vbc.ni.exe"), Path.Combine(outputDir, "vbc.ni.dll"));
}
return(c.Success());
}
