protected override IEnumerable <string> BuildCommandLineArguments(string assemblyFileName, string outputFileName)
{
// The file to compile
yield return(assemblyFileName);
// Output
yield return($"-o:{outputFileName}");
// Don't forget this one.
yield return("--readytorun");
// Todo: Allow control of some of these
yield return("-g");
yield return("--runtimeopt:RH_UseServerGC=1");
yield return("--targetarch=x64");
yield return("--stacktracedata");
foreach (var reference in ComputeManagedAssemblies.GetManagedAssembliesInFolder(_inputPath))
{
yield return($"-r:{reference}");
}
foreach (var referenceFolder in _referenceFolders)
{
foreach (var reference in ComputeManagedAssemblies.GetManagedAssembliesInFolder(referenceFolder))
{
yield return($"-r:{reference}");
}
}
}
private static IEnumerable <string> ResolveReferences(IEnumerable <string> folders)
{
foreach (string referenceFolder in folders)
{
foreach (string reference in ComputeManagedAssemblies.GetManagedAssembliesInFolder(referenceFolder))
{
yield return(reference);
}
}
}
public static BuildFolder FromDirectory(string inputDirectory, IEnumerable <CompilerRunner> compilerRunners, string outputRoot, BuildOptions options)
{
List <string> compilationInputFiles = new List <string>();
List <string> passThroughFiles = new List <string>();
List <string> mainExecutables = new List <string>();
List <string> executionScripts = new List <string>();
string scriptExtension = (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ".cmd" : ".sh");
// Copy unmanaged files (runtime, native dependencies, resources, etc)
foreach (string file in Directory.EnumerateFiles(inputDirectory, options.InputFileSearchString ?? "*"))
{
bool isManagedAssembly = ComputeManagedAssemblies.IsManaged(file);
if (isManagedAssembly)
{
compilationInputFiles.Add(file);
}
if ((!isManagedAssembly || options.Composite) &&
(Path.GetExtension(file) != ".pdb") && (Path.GetExtension(file) != ".ilk")) // exclude .pdb and .ilk files that are large and not needed in the target folder
{
passThroughFiles.Add(file);
}
string ext = Path.GetExtension(file);
if (ext.Equals(".exe", StringComparison.OrdinalIgnoreCase))
{
mainExecutables.Add(file);
}
else if (ext.Equals(scriptExtension, StringComparison.OrdinalIgnoreCase))
{
executionScripts.Add(file);
}
}
if (compilationInputFiles.Count == 0)
{
return(null);
}
foreach (CompilerRunner runner in compilerRunners)
{
string runnerOutputPath = runner.GetOutputPath(outputRoot);
if (!options.Exe)
{
runnerOutputPath.RecreateDirectory();
foreach (string file in passThroughFiles)
{
File.Copy(file, Path.Combine(runnerOutputPath, Path.GetFileName(file)));
}
}
}
return(new BuildFolder(compilationInputFiles, mainExecutables, executionScripts, compilerRunners, inputDirectory, outputRoot, options));
}
private List <string> ResolveReferences()
{
List <string> references = new List <string>();
foreach (var referenceFolder in _referenceFolders)
{
foreach (var reference in ComputeManagedAssemblies.GetManagedAssembliesInFolder(referenceFolder))
{
references.Add($"-r:{reference}");
}
}
return(references);
}
private List <string> ResolveReferences()
{
char referenceOption = (_options.Composite ? 'u' : 'r');
List <string> references = new List <string>();
foreach (var referenceFolder in _referenceFolders)
{
foreach (var reference in ComputeManagedAssemblies.GetManagedAssembliesInFolder(referenceFolder))
{
references.Add($"-{referenceOption}:{reference}");
}
}
return(references);
}
private IEnumerable <string> ResolveReferences(IEnumerable <string> folders, char referenceOption)
{
foreach (string referenceFolder in folders)
{
foreach (string reference in ComputeManagedAssemblies.GetManagedAssembliesInFolder(referenceFolder))
{
string simpleName = Path.GetFileNameWithoutExtension(reference);
if (!FrameworkExclusion.Exclude(simpleName, Index, out string reason))
{
yield return($"-{referenceOption}:{reference}");
}
}
}
}
private static IEnumerable <string> ResolveReferences(IEnumerable <string> folders)
{
foreach (string referenceFolder in folders)
{
foreach (string reference in ComputeManagedAssemblies.GetManagedAssembliesInFolder(referenceFolder))
{
if (reference.EndsWith(".ni.dll"))
{
continue;
}
yield return(reference);
}
}
}
public static BuildFolder FromDirectory(string inputDirectory, IEnumerable <CompilerRunner> compilerRunners, string outputRoot, BuildOptions options)
{
List <string> compilationInputFiles = new List <string>();
List <string> passThroughFiles = new List <string>();
List <string> mainExecutables = new List <string>();
List <string> executionScripts = new List <string>();
// Copy unmanaged files (runtime, native dependencies, resources, etc)
foreach (string file in Directory.EnumerateFiles(inputDirectory))
{
bool isManagedAssembly = ComputeManagedAssemblies.IsManaged(file);
if (isManagedAssembly)
{
compilationInputFiles.Add(file);
}
else
{
passThroughFiles.Add(file);
}
string ext = Path.GetExtension(file);
if (ext.Equals(".exe", StringComparison.OrdinalIgnoreCase))
{
mainExecutables.Add(file);
}
else if (ext.Equals(".cmd", StringComparison.OrdinalIgnoreCase))
{
executionScripts.Add(file);
}
}
if (compilationInputFiles.Count == 0)
{
return(null);
}
foreach (CompilerRunner runner in compilerRunners)
{
string runnerOutputPath = runner.GetOutputPath(outputRoot);
runnerOutputPath.RecreateDirectory();
foreach (string file in passThroughFiles)
{
File.Copy(file, Path.Combine(runnerOutputPath, Path.GetFileName(file)));
}
}
return(new BuildFolder(compilationInputFiles, mainExecutables, executionScripts, compilerRunners, outputRoot, options));
}
protected override IEnumerable <string> BuildCommandLineArguments(string assemblyFileName, string outputFileName)
{
// The file to compile
yield return(assemblyFileName);
// Output
yield return($"-o:{outputFileName}");
// Todo: Allow control of some of these
yield return("--targetarch=x64");
if (_options.GenerateMapFile)
{
yield return("--map");
}
if (_options.Release)
{
yield return("-O");
}
if (_options.LargeBubble)
{
yield return("--inputbubble");
}
if (_options.Crossgen2Parallelism != 0)
{
yield return($"--parallelism={_options.Crossgen2Parallelism}");
}
foreach (var reference in ComputeManagedAssemblies.GetManagedAssembliesInFolder(Path.GetDirectoryName(assemblyFileName)))
{
yield return($"-r:{reference}");
}
if (_resolvedReferences == null)
{
_resolvedReferences = ResolveReferences();
}
foreach (string asmRef in _resolvedReferences)
{
yield return(asmRef);
}
}
protected override IEnumerable <string> BuildCommandLineArguments(string assemblyFileName, string outputFileName)
{
// The file to compile
yield return(assemblyFileName);
// Output
yield return($"-o:{outputFileName}");
// Don't forget this one.
yield return("--readytorun");
// Todo: Allow control of some of these
yield return("-g");
yield return("--runtimeopt:RH_UseServerGC=1");
yield return("--targetarch=x64");
yield return("--stacktracedata");
if (_options.Release)
{
yield return("-O");
}
if (_options.LargeBubble)
{
yield return("--inputbubble");
}
foreach (var reference in ComputeManagedAssemblies.GetManagedAssembliesInFolder(Path.GetDirectoryName(assemblyFileName)))
{
yield return($"-r:{reference}");
}
if (_resolvedReferences == null)
{
_resolvedReferences = ResolveReferences();
}
foreach (string asmRef in _resolvedReferences)
{
yield return(asmRef);
}
}
public static int CompileDirectory(DirectoryInfo toolDirectory, DirectoryInfo inputDirectory, DirectoryInfo outputDirectory, bool crossgen, bool cpaot, DirectoryInfo[] referencePath)
{
if (toolDirectory == null)
{
Console.WriteLine("--tool-directory is a required argument.");
return(1);
}
if (inputDirectory == null)
{
Console.WriteLine("--input-directory is a required argument.");
return(1);
}
if (outputDirectory == null)
{
Console.WriteLine("--output-directory is a required argument.");
return(1);
}
CompilerRunner runner;
if (cpaot)
{
runner = new CpaotRunner(toolDirectory.ToString(), inputDirectory.ToString(), outputDirectory.ToString(), referencePath?.Select(x => x.ToString())?.ToList());
}
else
{
runner = new CrossgenRunner(toolDirectory.ToString(), inputDirectory.ToString(), outputDirectory.ToString(), referencePath?.Select(x => x.ToString())?.ToList());
}
bool success = true;
foreach (var assemblyToOptimize in ComputeManagedAssemblies.GetManagedAssembliesInFolder(inputDirectory.ToString()))
{
// Compile all assemblies in the input folder
if (!runner.CompileAssembly(assemblyToOptimize))
{
success = false;
}
}
return(success ? 0 : 1);
}
public static int CompileDirectory(
DirectoryInfo inputDirectory,
DirectoryInfo outputDirectory,
DirectoryInfo crossgenDirectory,
DirectoryInfo cpaotDirectory,
DirectoryInfo[] referencePath)
{
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
if (inputDirectory == null)
{
Console.WriteLine("--input-directory is a required argument.");
return(1);
}
if (outputDirectory == null)
{
outputDirectory = inputDirectory;
}
if (outputDirectory.IsParentOf(inputDirectory))
{
Console.WriteLine("Error: Input and output folders must be distinct, and the output directory (which gets deleted) better not be a parent of the input directory.");
return(1);
}
List <string> referencePaths = referencePath?.Select(x => x.ToString())?.ToList();
string coreRunPath = null;
foreach (string path in referencePaths)
{
string candidatePath = Path.Combine(path, "CoreRun.exe");
if (File.Exists(candidatePath))
{
coreRunPath = candidatePath;
break;
}
}
if (coreRunPath == null)
{
Console.Error.WriteLine("CoreRun.exe not found in reference folders, execution won't run");
}
List <CompilerRunner> runners = new List <CompilerRunner>();
if (cpaotDirectory != null)
{
runners.Add(new CpaotRunner(cpaotDirectory.ToString(), inputDirectory.ToString(), outputDirectory.ToString(), referencePaths));
}
if (crossgenDirectory != null)
{
runners.Add(new CrossgenRunner(crossgenDirectory.ToString(), inputDirectory.ToString(), outputDirectory.ToString(), referencePaths));
}
List <string> compilationInputFiles = new List <string>();
List <string> passThroughFiles = new List <string>();
string mainExecutable = null;
// Copy unmanaged files (runtime, native dependencies, resources, etc)
foreach (string file in Directory.EnumerateFiles(inputDirectory.FullName))
{
bool isManagedAssembly = ComputeManagedAssemblies.IsManaged(file);
if (isManagedAssembly)
{
compilationInputFiles.Add(file);
}
else
{
passThroughFiles.Add(file);
}
if (Path.GetExtension(file).Equals(".exe", StringComparison.OrdinalIgnoreCase))
{
mainExecutable = file;
}
}
foreach (CompilerRunner runner in runners)
{
string runnerOutputPath = runner.GetOutputPath();
runnerOutputPath.RecreateDirectory();
foreach (string file in passThroughFiles)
{
File.Copy(file, Path.Combine(runnerOutputPath, Path.GetFileName(file)));
}
}
List <ProcessInfo> compilationsToRun = new List <ProcessInfo>();
Application application = new Application(compilationInputFiles, mainExecutable, runners, coreRunPath);
foreach (ProcessInfo[] compilation in application.Compilations)
{
foreach (CompilerRunner runner in runners)
{
compilationsToRun.Add(compilation[(int)runner.Index]);
}
}
compilationsToRun.Sort((a, b) => b.CompilationCostHeuristic.CompareTo(a.CompilationCostHeuristic));
ParallelRunner.Run(compilationsToRun);
bool success = true;
List <KeyValuePair <string, string> > failedCompilationsPerBuilder = new List <KeyValuePair <string, string> >();
int successfulCompileCount = 0;
foreach (ProcessInfo[] compilation in application.Compilations)
{
string file = null;
string failedBuilders = null;
foreach (CompilerRunner runner in runners)
{
ProcessInfo runnerProcess = compilation[(int)runner.Index];
if (!runnerProcess.Succeeded)
{
File.Copy(runnerProcess.InputFileName, runnerProcess.OutputFileName);
if (file == null)
{
file = runnerProcess.InputFileName;
failedBuilders = runner.CompilerName;
}
else
{
failedBuilders += "; " + runner.CompilerName;
}
}
}
if (file != null)
{
failedCompilationsPerBuilder.Add(new KeyValuePair <string, string>(file, failedBuilders));
success = false;
}
else
{
successfulCompileCount++;
}
}
Console.WriteLine($"Compiled {successfulCompileCount} / {successfulCompileCount + failedCompilationsPerBuilder.Count} assemblies in {stopwatch.ElapsedMilliseconds} msecs.");
if (failedCompilationsPerBuilder.Count > 0)
{
Console.WriteLine($"Failed to compile {failedCompilationsPerBuilder.Count} assemblies:");
foreach (KeyValuePair <string, string> assemblyBuilders in failedCompilationsPerBuilder)
{
string assemblySpec = assemblyBuilders.Key;
if (runners.Count > 1)
{
assemblySpec += " (" + assemblyBuilders.Value + ")";
}
Console.WriteLine(assemblySpec);
}
}
if (coreRunPath != null)
{
List <ProcessInfo> executionsToRun = new List <ProcessInfo>();
foreach (CompilerRunner runner in runners)
{
bool compilationsSucceeded = application.Compilations.All(comp => comp[(int)runner.Index].Succeeded);
if (compilationsSucceeded)
{
ProcessInfo executionProcess = application.Execution[(int)runner.Index];
if (executionProcess != null)
{
executionsToRun.Add(executionProcess);
}
}
}
ParallelRunner.Run(executionsToRun);
}
return(success ? 0 : 1);
}
protected override IEnumerable <string> BuildCommandLineArguments(IEnumerable <string> assemblyFileNames, string outputFileName)
{
// The file to compile
foreach (string inputAssembly in assemblyFileNames)
{
yield return(inputAssembly);
}
// Output
yield return($"-o:{outputFileName}");
// Todo: Allow control of some of these
yield return("--targetarch=x64");
if (_options.Map)
{
yield return("--map");
}
if (_options.Release)
{
yield return("-O");
}
if (_options.LargeBubble || _options.Composite)
{
yield return("--inputbubble");
}
if (_options.Composite)
{
yield return("--composite");
}
if (_options.Crossgen2Parallelism != 0)
{
yield return($"--parallelism={_options.Crossgen2Parallelism}");
}
char referenceOption = (_options.Composite ? 'u' : 'r');
HashSet <string> uniqueFolders = new HashSet <string>();
foreach (string assemblyFileName in assemblyFileNames)
{
uniqueFolders.Add(Path.GetDirectoryName(assemblyFileName));
}
foreach (string folder in uniqueFolders)
{
foreach (var reference in ComputeManagedAssemblies.GetManagedAssembliesInFolder(folder))
{
yield return($"-{referenceOption}:{reference}");
}
}
if (_resolvedReferences == null)
{
_resolvedReferences = ResolveReferences();
}
foreach (string asmRef in _resolvedReferences)
{
yield return(asmRef);
}
}
public bool CompileFramework()
{
if (!_options.Framework)
{
return(true);
}
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
string coreRoot = _options.CoreRootDirectory.FullName;
string[] frameworkFolderFiles = Directory.GetFiles(coreRoot);
IEnumerable <CompilerRunner> frameworkRunners = _options.CompilerRunners(isFramework: true);
// Pre-populate the output folders with the input files so that we have backdrops
// for failing compilations.
foreach (CompilerRunner runner in frameworkRunners)
{
string outputPath = runner.GetOutputPath(coreRoot);
outputPath.RecreateDirectory();
}
List <ProcessInfo> compilationsToRun = new List <ProcessInfo>();
List <KeyValuePair <string, ProcessInfo[]> > compilationsPerRunner = new List <KeyValuePair <string, ProcessInfo[]> >();
foreach (string frameworkDll in ComputeManagedAssemblies.GetManagedAssembliesInFolder(_options.CoreRootDirectory.FullName))
{
ProcessInfo[] processes = new ProcessInfo[(int)CompilerIndex.Count];
compilationsPerRunner.Add(new KeyValuePair <string, ProcessInfo[]>(frameworkDll, processes));
foreach (CompilerRunner runner in frameworkRunners)
{
ProcessInfo compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, _options.CoreRootDirectory.FullName, frameworkDll));
compilationsToRun.Add(compilationProcess);
processes[(int)runner.Index] = compilationProcess;
}
}
ParallelRunner.Run(compilationsToRun, _options.DegreeOfParallelism);
HashSet <string> skipCopying = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
int[] failedCompilationsPerBuilder = new int[(int)CompilerIndex.Count];
int successfulCompileCount = 0;
int failedCompileCount = 0;
foreach (KeyValuePair <string, ProcessInfo[]> kvp in compilationsPerRunner)
{
bool anyCompilationsFailed = false;
foreach (CompilerRunner runner in frameworkRunners)
{
ProcessInfo compilationProcess = kvp.Value[(int)runner.Index];
if (compilationProcess.Succeeded)
{
skipCopying.Add(compilationProcess.Parameters.InputFileName);
}
else
{
anyCompilationsFailed = true;
failedCompilationsPerBuilder[(int)runner.Index]++;
_frameworkCompilationFailureBuckets.AddCompilation(compilationProcess);
}
}
if (anyCompilationsFailed)
{
failedCompileCount++;
}
else
{
successfulCompileCount++;
}
}
foreach (CompilerRunner runner in frameworkRunners)
{
string outputPath = runner.GetOutputPath(coreRoot);
foreach (string file in frameworkFolderFiles)
{
if (!skipCopying.Contains(file))
{
string targetFile = Path.Combine(outputPath, Path.GetFileName(file));
File.Copy(file, targetFile, overwrite: true);
}
}
}
_frameworkCompilationMilliseconds = stopwatch.ElapsedMilliseconds;
return(failedCompileCount == 0);
}
public static BuildFolder FromDirectory(string inputDirectory, IEnumerable <CompilerRunner> compilerRunners, string outputRoot, BuildOptions options)
{
List <string> compilationInputFiles = new List <string>();
HashSet <string> passThroughFiles = new HashSet <string>();
List <string> mainExecutables = new List <string>();
List <string> executionScripts = new List <string>();
string scriptExtension = (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ".cmd" : ".sh");
// Copy unmanaged files (runtime, native dependencies, resources, etc)
foreach (string file in Directory.EnumerateFiles(inputDirectory, options.InputFileSearchString ?? "*"))
{
bool isManagedAssembly = ComputeManagedAssemblies.IsManaged(file);
if (isManagedAssembly)
{
compilationInputFiles.Add(file);
}
if ((!isManagedAssembly || options.Composite) &&
(Path.GetExtension(file) != ".pdb") && (Path.GetExtension(file) != ".ilk")) // exclude .pdb and .ilk files that are large and not needed in the target folder
{
passThroughFiles.Add(file);
}
string ext = Path.GetExtension(file);
if (ext.Equals(".exe", StringComparison.OrdinalIgnoreCase))
{
mainExecutables.Add(file);
}
else if (ext.Equals(scriptExtension, StringComparison.OrdinalIgnoreCase))
{
executionScripts.Add(file);
}
}
if (compilationInputFiles.Count == 0)
{
return(null);
}
if (options.Composite && !(options.Framework || options.UseFramework))
{
// In composite mode we copy the native runtime to the app folder and pretend that is CORE_ROOT,
// otherwise CoreRun picks up the original MSIL versions of framework assemblies from CORE_ROOT
// instead of the rewritten ones next to the app.
foreach (string exe in s_runtimeExecutables)
{
passThroughFiles.Add(Path.Combine(options.CoreRootDirectory.FullName, exe.AppendOSExeSuffix()));
}
string libraryPrefix = (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "" : "lib");
foreach (string lib in s_runtimeLibraries)
{
passThroughFiles.Add(Path.Combine(options.CoreRootDirectory.FullName, (libraryPrefix + lib).AppendOSDllSuffix()));
}
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
foreach (string lib in s_runtimeWindowsOnlyLibraries)
{
passThroughFiles.Add(Path.Combine(options.CoreRootDirectory.FullName, lib.AppendOSDllSuffix()));
}
}
else
{
// Several native lib*.so / dylib are needed by the runtime
foreach (string nativeLib in Directory.EnumerateFiles(options.CoreRootDirectory.FullName, "lib*".AppendOSDllSuffix()))
{
passThroughFiles.Add(nativeLib);
}
}
}
foreach (CompilerRunner runner in compilerRunners)
{
string runnerOutputPath = runner.GetOutputPath(outputRoot);
if (!options.Exe)
{
runnerOutputPath.RecreateDirectory();
foreach (string file in passThroughFiles)
{
File.Copy(file, Path.Combine(runnerOutputPath, Path.GetFileName(file)));
}
}
}
return(new BuildFolder(compilationInputFiles, mainExecutables, executionScripts, compilerRunners, inputDirectory, outputRoot, options));
}
public static int CompileDirectory(DirectoryInfo toolDirectory, DirectoryInfo inputDirectory, DirectoryInfo outputDirectory, bool crossgen, bool cpaot, DirectoryInfo[] referencePath)
{
if (toolDirectory == null)
{
Console.WriteLine("--tool-directory is a required argument.");
return(1);
}
if (inputDirectory == null)
{
Console.WriteLine("--input-directory is a required argument.");
return(1);
}
if (outputDirectory == null)
{
Console.WriteLine("--output-directory is a required argument.");
return(1);
}
if (OutputPathIsParentOfInputPath(inputDirectory, outputDirectory))
{
Console.WriteLine("Error: Input and output folders must be distinct, and the output directory (which gets deleted) better not be a parent of the input directory.");
return(1);
}
CompilerRunner runner;
if (cpaot)
{
runner = new CpaotRunner(toolDirectory.ToString(), inputDirectory.ToString(), outputDirectory.ToString(), referencePath?.Select(x => x.ToString())?.ToList());
}
else
{
runner = new CrossgenRunner(toolDirectory.ToString(), inputDirectory.ToString(), outputDirectory.ToString(), referencePath?.Select(x => x.ToString())?.ToList());
}
if (outputDirectory.Exists)
{
try
{
outputDirectory.Delete(recursive: true);
}
catch (Exception ex) when(
ex is UnauthorizedAccessException ||
ex is DirectoryNotFoundException ||
ex is IOException
)
{
Console.WriteLine($"Error: Could not delete output folder {outputDirectory.FullName}. {ex.Message}");
return(1);
}
}
outputDirectory.Create();
bool success = true;
// Copy unmanaged files (runtime, native dependencies, resources, etc)
foreach (string file in Directory.EnumerateFiles(inputDirectory.FullName))
{
if (ComputeManagedAssemblies.IsManaged(file))
{
// Compile managed code
if (!runner.CompileAssembly(file))
{
success = false;
// On compile failure, pass through the input IL assembly so the output is still usable
File.Copy(file, Path.Combine(outputDirectory.FullName, Path.GetFileName(file)));
}
}
else
{
// Copy through all other files
File.Copy(file, Path.Combine(outputDirectory.FullName, Path.GetFileName(file)));
}
}
return(success ? 0 : 1);
}
public static int CompileDirectory(DirectoryInfo toolDirectory, DirectoryInfo inputDirectory, DirectoryInfo outputDirectory, bool crossgen, bool cpaot, DirectoryInfo[] referencePath)
{
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
if (toolDirectory == null)
{
Console.WriteLine("--tool-directory is a required argument.");
return(1);
}
if (inputDirectory == null)
{
Console.WriteLine("--input-directory is a required argument.");
return(1);
}
if (outputDirectory == null)
{
outputDirectory = inputDirectory;
}
if (OutputPathIsParentOfInputPath(inputDirectory, outputDirectory))
{
Console.WriteLine("Error: Input and output folders must be distinct, and the output directory (which gets deleted) better not be a parent of the input directory.");
return(1);
}
CompilerRunner runner;
if (cpaot)
{
runner = new CpaotRunner(toolDirectory.ToString(), inputDirectory.ToString(), outputDirectory.ToString(), referencePath?.Select(x => x.ToString())?.ToList());
}
else
{
runner = new CrossgenRunner(toolDirectory.ToString(), inputDirectory.ToString(), outputDirectory.ToString(), referencePath?.Select(x => x.ToString())?.ToList());
}
string runnerOutputPath = runner.GetOutputPath();
if (Directory.Exists(runnerOutputPath))
{
try
{
Directory.Delete(runnerOutputPath, recursive: true);
}
catch (Exception ex) when(
ex is UnauthorizedAccessException ||
ex is DirectoryNotFoundException ||
ex is IOException
)
{
Console.WriteLine($"Error: Could not delete output folder {runnerOutputPath}. {ex.Message}");
return(1);
}
}
Directory.CreateDirectory(runnerOutputPath);
List <ProcessInfo> compilationsToRun = new List <ProcessInfo>();
// Copy unmanaged files (runtime, native dependencies, resources, etc)
foreach (string file in Directory.EnumerateFiles(inputDirectory.FullName))
{
if (ComputeManagedAssemblies.IsManaged(file))
{
ProcessInfo compilationToRun = runner.CompilationProcess(file);
compilationToRun.InputFileName = file;
compilationsToRun.Add(compilationToRun);
}
else
{
// Copy through all other files
File.Copy(file, Path.Combine(runnerOutputPath, Path.GetFileName(file)));
}
}
ParallelRunner.Run(compilationsToRun);
bool success = true;
List <string> failedCompilationAssemblies = new List <string>();
int successfulCompileCount = 0;
foreach (ProcessInfo processInfo in compilationsToRun)
{
if (processInfo.Succeeded)
{
successfulCompileCount++;
}
else
{
File.Copy(processInfo.InputFileName, Path.Combine(runnerOutputPath, Path.GetFileName(processInfo.InputFileName)));
failedCompilationAssemblies.Add(processInfo.InputFileName);
}
}
Console.WriteLine($"Compiled {successfulCompileCount}/{successfulCompileCount + failedCompilationAssemblies.Count} assemblies in {stopwatch.ElapsedMilliseconds} msecs.");
if (failedCompilationAssemblies.Count > 0)
{
Console.WriteLine($"Failed to compile {failedCompilationAssemblies.Count} assemblies:");
foreach (var assembly in failedCompilationAssemblies)
{
Console.WriteLine(assembly);
}
}
return(success ? 0 : 1);
}
public CompileSerpCommand(BuildOptions options)
{
// This command does not work in the context of an app, just a loose set of rsp files so don't execute anything we compile
options.NoJit = true;
options.NoEtw = true;
options.Release = true;
options.LargeBubble = true;
_options = options;
if (_options.InputDirectory == null)
{
throw new ArgumentException("Specify --response-file or --input-directory containing multiple response files.");
}
if (_options.CoreRootDirectory == null)
{
throw new ArgumentException("--core-root-directory (--cr) is a required argument.");
}
if (_options.AspNetPath == null || !File.Exists(Path.Combine(_options.AspNetPath.FullName, "Microsoft.AspNetCore.dll")))
{
throw new ArgumentException($"Error: Asp.NET Core path must contain Microsoft.AspNetCore.dll");
}
SerpDir = _options.InputDirectory.FullName;
BinDir = Path.Combine(SerpDir, "bin");
if (!File.Exists(Path.Combine(SerpDir, "runserp.cmd")))
{
throw new ArgumentException($"Error: InputDirectory must point at a SERP build. Could not find {Path.Combine(SerpDir, "runserp.cmd")}");
}
string allowListFilePath = Path.Combine(SerpDir, "App_Data", "AllowedDllList.txt");
if (!File.Exists(allowListFilePath))
{
throw new ArgumentException($"File {allowListFilePath} was not found");
}
// Add all assemblies from the various SERP packages (filtered by ShouldInclude)
_packageCompileAssemblies = Directory.GetFiles(Path.Combine(SerpDir, "App_Data\\Answers\\Services\\Packages"), "*.dll", SearchOption.AllDirectories)
.Where((string x) => ShouldInclude(x))
.ToList();
_packageReferenceAssemblies = new List <string>();
{
HashSet <string> packageReferenceAssemblyDirectories = new HashSet <string>();
foreach (var binFile in _packageCompileAssemblies)
{
var directory = Path.GetDirectoryName(binFile);
if (!packageReferenceAssemblyDirectories.Contains(directory))
{
packageReferenceAssemblyDirectories.Add(directory);
}
}
foreach (string binFile in ResolveReferences(packageReferenceAssemblyDirectories))
{
_packageReferenceAssemblies.Add(binFile);
}
}
_coreCompileAssemblies = new List <string>();
_coreReferenceAssemblies = new List <string>();
{
// Add an allow-list of assemblies from bin. This unified list includes binaries from /bin and /App_data so filter just the /bin assemblies
foreach (string item in new HashSet <string>(File.ReadAllLines(allowListFilePath)))
{
string binAssembly = Path.Combine(BinDir, item);
if (File.Exists(binAssembly) &&
!FrameworkExclusion.Exclude(Path.GetFileNameWithoutExtension(binAssembly), CompilerIndex.CPAOT, out string reason))
{
_coreCompileAssemblies.Add(binAssembly);
}
}
HashSet <string> coreReferenceAssemblyDirectories = new HashSet <string>();
foreach (var binFile in _coreCompileAssemblies)
{
var directory = Path.GetDirectoryName(binFile);
if (!coreReferenceAssemblyDirectories.Contains(directory))
{
coreReferenceAssemblyDirectories.Add(directory);
}
}
foreach (string binFile in ResolveReferences(coreReferenceAssemblyDirectories))
{
_coreReferenceAssemblies.Add(binFile);
}
}
_frameworkCompileAssemblies = new List <string>();
_frameworkReferenceAssemblies = new List <string>();
{
foreach (string frameworkDll in ComputeManagedAssemblies.GetManagedAssembliesInFolder(options.CoreRootDirectory.FullName, "System.*.dll"))
{
string simpleName = Path.GetFileNameWithoutExtension(frameworkDll);
if (!FrameworkExclusion.Exclude(simpleName, CompilerIndex.CPAOT, out string reason))
{
_frameworkCompileAssemblies.Add(frameworkDll);
}
}
foreach (string frameworkDll in ComputeManagedAssemblies.GetManagedAssembliesInFolder(options.CoreRootDirectory.FullName, "Microsoft.*.dll"))
{
string simpleName = Path.GetFileNameWithoutExtension(frameworkDll);
if (!FrameworkExclusion.Exclude(simpleName, CompilerIndex.CPAOT, out string reason))
{
_frameworkCompileAssemblies.Add(frameworkDll);
}
}
_frameworkCompileAssemblies.Add(Path.Combine(options.CoreRootDirectory.FullName, "mscorlib.dll"));
_frameworkCompileAssemblies.Add(Path.Combine(options.CoreRootDirectory.FullName, "netstandard.dll"));
_frameworkReferenceAssemblies.AddRange(ComputeManagedAssemblies.GetManagedAssembliesInFolder(options.CoreRootDirectory.FullName, "System.*.dll"));
_frameworkReferenceAssemblies.AddRange(ComputeManagedAssemblies.GetManagedAssembliesInFolder(options.CoreRootDirectory.FullName, "Microsoft.*.dll"));
_frameworkReferenceAssemblies.Add(Path.Combine(options.CoreRootDirectory.FullName, "mscorlib.dll"));
_frameworkReferenceAssemblies.Add(Path.Combine(options.CoreRootDirectory.FullName, "netstandard.dll"));
}
_aspCompileAssemblies = new List <string>();
_aspReferenceAssemblies = new List <string>();
{
_aspCompileAssemblies.AddRange(ComputeManagedAssemblies.GetManagedAssembliesInFolder(options.AspNetPath.FullName, "Microsoft.AspNetCore.*.dll"));
_aspCompileAssemblies.AddRange(ComputeManagedAssemblies.GetManagedAssembliesInFolder(options.AspNetPath.FullName, "Microsoft.Extensions.*.dll"));
_aspCompileAssemblies.Add(Path.Combine(options.AspNetPath.FullName, "Microsoft.JSInterop.dll"));
_aspCompileAssemblies.Add(Path.Combine(options.AspNetPath.FullName, "Microsoft.Net.Http.Headers.dll"));
_aspCompileAssemblies.Add(Path.Combine(options.AspNetPath.FullName, "Microsoft.Win32.SystemEvents.dll"));
_aspCompileAssemblies.Add(Path.Combine(options.AspNetPath.FullName, "System.Diagnostics.EventLog.dll"));
_aspCompileAssemblies.Add(Path.Combine(options.AspNetPath.FullName, "System.Drawing.Common.dll"));
_aspCompileAssemblies.Add(Path.Combine(options.AspNetPath.FullName, "System.IO.Pipelines.dll"));
_aspCompileAssemblies.Add(Path.Combine(options.AspNetPath.FullName, "System.Security.Cryptography.Pkcs.dll"));
_aspCompileAssemblies.Add(Path.Combine(options.AspNetPath.FullName, "System.Security.Cryptography.Xml.dll"));
_aspCompileAssemblies.Add(Path.Combine(options.AspNetPath.FullName, "System.Security.Permissions.dll"));
_aspCompileAssemblies.Add(Path.Combine(options.AspNetPath.FullName, "System.Windows.Extensions.dll"));
_aspReferenceAssemblies = new List <string>(_aspCompileAssemblies);
}
}
public static int CompileSerpAssemblies(BuildOptions options)
{
if (options.InputDirectory == null)
{
Console.Error.WriteLine("Specify --response-file or --input-directory containing multiple response files.");
return(1);
}
if (options.CoreRootDirectory == null)
{
Console.Error.WriteLine("--core-root-directory (--cr) is a required argument.");
return(1);
}
// This command does not work in the context of an app, just a loose set of rsp files so don't execute anything we compile
options.NoJit = true;
options.NoEtw = true;
string serpDir = options.InputDirectory.FullName;
if (!File.Exists(Path.Combine(serpDir, "runserp.cmd")))
{
Console.Error.WriteLine($"Error: InputDirectory must point at a SERP build. Could not find {Path.Combine(serpDir, "runserp.cmd")}");
return(1);
}
string whiteListFilePath = Path.Combine(serpDir, "WhitelistDlls.txt");
if (!File.Exists(whiteListFilePath))
{
Console.Error.WriteLine($"File {whiteListFilePath} was not found");
return(1);
}
if (!File.Exists(Path.Combine(options.AspNetPath.FullName, "Microsoft.AspNetCore.dll")))
{
Console.Error.WriteLine($"Error: Asp.NET Core path must contain Microsoft.AspNetCore.dll");
return(1);
}
string binDir = Path.Combine(serpDir, "bin");
// Remove existing native images
foreach (var file in Directory.GetFiles(Path.Combine(serpDir, "App_Data\\Answers\\Services\\Packages"), "*.dll", SearchOption.AllDirectories))
{
if (file.EndsWith(".ni.dll") || file.EndsWith(".ni.exe"))
{
File.Delete(file);
}
}
foreach (var file in Directory.GetFiles(binDir, "*.dll", SearchOption.AllDirectories))
{
if (file.EndsWith(".ni.dll") || file.EndsWith(".ni.exe"))
{
File.Delete(file);
}
}
// Add all assemblies from the various SERP packages (filtered by ShouldInclude)
List <string> binFiles = Directory.GetFiles(Path.Combine(serpDir, "App_Data\\Answers\\Services\\Packages"), "*.dll", SearchOption.AllDirectories)
.Where((string x) => ShouldInclude(x))
.ToList();
// Add a whitelist of assemblies from bin
foreach (string item in new HashSet <string>(File.ReadAllLines(whiteListFilePath)))
{
binFiles.Add(Path.Combine(binDir, item));
}
HashSet <string> referenceAssemblyDirectories = new HashSet <string>();
foreach (var binFile in binFiles)
{
var directory = Path.GetDirectoryName(binFile);
if (!referenceAssemblyDirectories.Contains(directory))
{
referenceAssemblyDirectories.Add(directory);
}
}
// TestILC needs a list of all directories containing assemblies that are referenced from crossgen
List <string> referenceAssemblies = new List <string>();
HashSet <string> simpleNames = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
// Reference all managed assemblies in /bin and /App_Data/answers/services/packages
foreach (string binFile in ResolveReferences(referenceAssemblyDirectories))
{
simpleNames.Add(Path.GetFileNameWithoutExtension(binFile));
referenceAssemblies.Add(binFile);
}
referenceAssemblies.AddRange(ComputeManagedAssemblies.GetManagedAssembliesInFolderNoSimpleNameDuplicates(simpleNames, options.AspNetPath.FullName, "*.dll"));
// Add CoreRoot last because it contains various non-framework assemblies that are duplicated in SERP and we want SERP's to be used
referenceAssemblies.AddRange(ComputeManagedAssemblies.GetManagedAssembliesInFolderNoSimpleNameDuplicates(simpleNames, options.CoreRootDirectory.FullName, "System.*.dll"));
referenceAssemblies.AddRange(ComputeManagedAssemblies.GetManagedAssembliesInFolderNoSimpleNameDuplicates(simpleNames, options.CoreRootDirectory.FullName, "Microsoft.*.dll"));
referenceAssemblies.Add(Path.Combine(options.CoreRootDirectory.FullName, "mscorlib.dll"));
referenceAssemblies.Add(Path.Combine(options.CoreRootDirectory.FullName, "netstandard.dll"));
//
// binFiles is now all the assemblies that we want to compile (either individually or as composite)
// referenceAssemblies is all managed assemblies that are referenceable
//
// Remove all bin files except serp.dll so they're just referenced (eventually we'll be able to compile all these in a single composite)
foreach (string item in new HashSet <string>(File.ReadAllLines(whiteListFilePath)))
{
if (item == "Serp.dll")
{
continue;
}
binFiles.Remove(Path.Combine(binDir, item));
}
List <ProcessInfo> fileCompilations = new List <ProcessInfo>();
if (options.Composite)
{
string serpDll = Path.Combine(binDir, "Serp.dll");
var runner = new CpaotRunner(options, referenceAssemblies);
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, Path.ChangeExtension(serpDll, ".ni.dll"), binFiles));
fileCompilations.Add(compilationProcess);
}
else
{
var runner = new CpaotRunner(options, referenceAssemblies);
foreach (string assemblyName in binFiles)
{
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, Path.ChangeExtension(assemblyName, ".ni.dll"), new string[] { assemblyName }));
fileCompilations.Add(compilationProcess);
}
}
ParallelRunner.Run(fileCompilations, options.DegreeOfParallelism);
bool success = true;
int compilationFailures = 0;
foreach (var compilationProcess in fileCompilations)
{
if (!compilationProcess.Succeeded)
{
success = false;
compilationFailures++;
Console.WriteLine($"Failed compiling {compilationProcess.Parameters.OutputFileName}");
}
}
Console.WriteLine("Serp Compilation Results");
Console.WriteLine($"Total compilations: {fileCompilations.Count}");
Console.WriteLine($"Compilation failures: {compilationFailures}");
return(success ? 0 : 1);
}
