public JitRunner(BuildOptions options)
: base(options, new string[] { options.CoreRootDirectory.FullName }.Concat(options.ReferencePaths()))
{
}
public static TestExclusionMap Create(BuildOptions options)
{
TestExclusionMap outputMap = new TestExclusionMap();
if (options.IssuesPath != null)
{
Dictionary <string, List <TestExclusion> > exclusionsByCondition = new Dictionary <string, List <TestExclusion> >();
foreach (FileInfo issuesProject in options.IssuesPath)
{
string issuesProjectPath = issuesProject.FullName;
XDocument issuesXml = XDocument.Load(issuesProjectPath);
foreach (XElement itemGroupElement in issuesXml.Root.Elements(s_xmlNamespace + "ItemGroup"))
{
string condition = itemGroupElement.Attribute("Condition")?.Value ?? "";
List <TestExclusion> exclusions;
if (!exclusionsByCondition.TryGetValue(condition, out exclusions))
{
exclusions = new List <TestExclusion>();
exclusionsByCondition.Add(condition, exclusions);
}
foreach (XElement excludeListElement in itemGroupElement.Elements(s_xmlNamespace + "ExcludeList"))
{
string testPath = excludeListElement.Attribute("Include")?.Value ?? "";
string issueID = excludeListElement.Element(s_xmlNamespace + "Issue")?.Value ?? "N/A";
exclusions.Add(CreateTestExclusion(testPath, issueID));
}
}
}
Project project = new Project();
project.SetGlobalProperty("XunitTestBinBase", "*");
project.SetGlobalProperty("TargetArchitecture", "x64");
project.SetGlobalProperty("RuntimeFlavor", "coreclr");
// TODO: cross-OS CPAOT
project.SetGlobalProperty("TargetsWindows", (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "true" : "false"));
project.SetGlobalProperty("AltJitArch", "x64");
project.SetGlobalProperty("RunTestViaIlLink", "false");
ProjectRootElement root = project.Xml;
root.AddTarget("GetListOfTestCmds");
ProjectPropertyGroupElement propertyGroup = root.AddPropertyGroup();
// Generate properties into the project to make it evaluate all conditions found in the targets file
List <List <TestExclusion> > testExclusionLists = new List <List <TestExclusion> >();
testExclusionLists.Capacity = exclusionsByCondition.Count;
foreach (KeyValuePair <string, List <TestExclusion> > kvp in exclusionsByCondition)
{
string propertyName = "Condition_" + testExclusionLists.Count.ToString();
bool emptyKey = string.IsNullOrEmpty(kvp.Key);
propertyGroup.AddProperty(propertyName, emptyKey ? "true" : "false");
if (!emptyKey)
{
propertyGroup.AddProperty(propertyName, "true").Condition = kvp.Key;
}
testExclusionLists.Add(kvp.Value);
}
project.Build();
for (int exclusionListIndex = 0; exclusionListIndex < testExclusionLists.Count; exclusionListIndex++)
{
string conditionValue = project.GetProperty("Condition_" + exclusionListIndex.ToString()).EvaluatedValue;
if (conditionValue.Equals("true", StringComparison.OrdinalIgnoreCase))
{
foreach (TestExclusion exclusion in testExclusionLists[exclusionListIndex])
{
outputMap.Add(exclusion);
}
}
}
}
return(outputMap);
}
public static int CompileSubtree(BuildOptions options)
{
if (options.InputDirectory == null)
{
Console.WriteLine("--input-directory is a required argument.");
return(1);
}
if (options.CoreRootDirectory == null)
{
Console.Error.WriteLine("--core-root-directory (--cr) is a required argument.");
return(1);
}
if (options.OutputDirectory == null)
{
options.OutputDirectory = options.InputDirectory;
}
if (options.OutputDirectory.IsParentOf(options.InputDirectory))
{
Console.WriteLine("Error: Input and output folders must be distinct, and the output directory (which gets deleted) better not be a parent of the input directory.");
return(1);
}
IEnumerable <CompilerRunner> runners = options.CompilerRunners(isFramework: false);
if (!options.Exe)
{
PathExtensions.DeleteOutputFolders(options.OutputDirectory.FullName, options.CoreRootDirectory.FullName, runners, recursive: true);
}
string[] directories = LocateSubtree(options.InputDirectory.FullName, options.CoreRootDirectory.FullName).ToArray();
ConcurrentBag <BuildFolder> folders = new ConcurrentBag <BuildFolder>();
int relativePathOffset = options.InputDirectory.FullName.Length;
if (relativePathOffset > 0 && options.InputDirectory.FullName[relativePathOffset - 1] != Path.DirectorySeparatorChar)
{
relativePathOffset++;
}
int folderCount = 0;
int compilationCount = 0;
int executionCount = 0;
Parallel.ForEach(directories, (string directory) =>
{
string outputDirectoryPerFolder = options.OutputDirectory.FullName;
if (directory.Length > relativePathOffset)
{
outputDirectoryPerFolder = Path.Combine(outputDirectoryPerFolder, directory.Substring(relativePathOffset));
}
try
{
BuildFolder folder = BuildFolder.FromDirectory(directory.ToString(), runners, outputDirectoryPerFolder, options);
if (folder != null)
{
folders.Add(folder);
Interlocked.Add(ref compilationCount, folder.Compilations.Count);
Interlocked.Add(ref executionCount, folder.Executions.Count);
}
}
catch (Exception ex)
{
Console.Error.WriteLine("Error scanning folder {0}: {1}", directory, ex.Message);
}
int currentCount = Interlocked.Increment(ref folderCount);
if (currentCount % 100 == 0)
{
StringBuilder lineReport = new StringBuilder();
lineReport.Append($@"Found {folders.Count} folders to build ");
lineReport.Append($@"({compilationCount} compilations, ");
if (!options.NoExe)
{
lineReport.Append($@"{executionCount} executions, ");
}
lineReport.Append($@"{currentCount} / {directories.Length} folders scanned)");
Console.WriteLine(lineReport.ToString());
}
});
Console.Write($@"Found {folders.Count} folders to build ({compilationCount} compilations, ");
if (!options.NoExe)
{
Console.Write($@"{executionCount} executions, ");
}
Console.WriteLine($@"{directories.Length} folders scanned)");
BuildFolderSet folderSet = new BuildFolderSet(folders, runners, options);
bool success = folderSet.Build();
folderSet.WriteLogs();
if (!options.NoCleanup)
{
PathExtensions.DeleteOutputFolders(options.OutputDirectory.FullName, options.CoreRootDirectory.FullName, runners, recursive: true);
}
return(success ? 0 : 1);
}
public static int CompileNuget(BuildOptions options)
{
if (options.CoreRootDirectory == null)
{
Console.Error.WriteLine("--core-root-directory (--cr) is a required argument.");
return(1);
}
// We don't want to launch these apps when building the folder set below
options.NoExe = true;
options.NoJit = true;
options.InputDirectory = options.OutputDirectory;
IList <string> packageList = ReadPackageNames(options.PackageList.FullName);
if (options.OutputDirectory == null)
{
Console.Error.WriteLine("--output-directory is a required argument.");
return(1);
}
IEnumerable <string> referencePaths = options.ReferencePaths();
IEnumerable <CompilerRunner> runners = options.CompilerRunners(false);
if (!options.Exe)
{
PathExtensions.DeleteOutputFolders(options.OutputDirectory.FullName, options.CoreRootDirectory.FullName, runners, recursive: false);
}
string nugetOutputFolder = Path.Combine(options.OutputDirectory.FullName, "nuget.out");
Directory.CreateDirectory(nugetOutputFolder);
var publishedAppFoldersToCompile = new List <BuildFolder>();
using (StreamWriter nugetLog = File.CreateText(Path.Combine(nugetOutputFolder, "nugetLog.txt")))
{
foreach (var package in packageList)
{
nugetLog.WriteLine($"Creating empty app for {package}");
// Create an app folder
string appFolder = Path.Combine(nugetOutputFolder, $"{package}.TestApp");
Directory.CreateDirectory(appFolder);
int exitCode = DotnetCli.New(appFolder, "console", nugetLog);
if (exitCode != 0)
{
nugetLog.WriteLine($"dotnet new console for {package} failed with exit code {exitCode}");
continue;
}
exitCode = DotnetCli.AddPackage(appFolder, package, nugetLog);
if (exitCode != 0)
{
nugetLog.WriteLine($"dotnet add package {package} failed with exit code {exitCode}");
continue;
}
exitCode = DotnetCli.Publish(appFolder, nugetLog);
if (exitCode != 0)
{
nugetLog.WriteLine($"dotnet publish failed with exit code {exitCode}");
continue;
}
// This is not a reliable way of building the publish folder
string publishFolder = Path.Combine(appFolder, @"artifacts\Debug\net5.0\publish");
if (!Directory.Exists(publishFolder))
{
nugetLog.WriteLine($"Could not find folder {publishFolder} containing the published app.");
continue;
}
publishedAppFoldersToCompile.Add(BuildFolder.FromDirectory(publishFolder, runners, appFolder, options));
}
BuildFolderSet folderSet = new BuildFolderSet(publishedAppFoldersToCompile, runners, options);
bool success = folderSet.Build();
folderSet.WriteLogs();
if (!options.NoCleanup && !options.Exe)
{
PathExtensions.DeleteOutputFolders(options.OutputDirectory.FullName, options.CoreRootDirectory.FullName, runners, recursive: false);
}
return(success ? 0 : 1);
}
}
public CompileSerpCommand(BuildOptions options)
{
// This command does not work in the context of an app, just a loose set of rsp files so don't execute anything we compile
options.NoJit = true;
options.NoEtw = true;
options.Release = true;
_options = options;
if (_options.InputDirectory == null)
{
throw new ArgumentException("Specify --response-file or --input-directory containing multiple response files.");
}
if (_options.CoreRootDirectory == null)
{
throw new ArgumentException("--core-root-directory (--cr) is a required argument.");
}
if (_options.AspNetPath == null || !File.Exists(Path.Combine(_options.AspNetPath.FullName, "Microsoft.AspNetCore.dll")))
{
throw new ArgumentException($"Error: Asp.NET Core path must contain Microsoft.AspNetCore.dll");
}
SerpDir = _options.InputDirectory.FullName;
BinDir = Path.Combine(SerpDir, "bin");
if (!File.Exists(Path.Combine(SerpDir, "runserp.cmd")))
{
throw new ArgumentException($"Error: InputDirectory must point at a SERP build. Could not find {Path.Combine(SerpDir, "runserp.cmd")}");
}
// Check that the "Microsoft.Search.Frontend.AnswersGlue.dll" assembly was generated in this drop
string answersGlueAssembly = Path.Combine(BinDir, AnswersGlueAssemblyFilename);
if (!File.Exists(answersGlueAssembly))
{
throw new FileNotFoundException($"Could not find {answersGlueAssembly}. Run {SerpDir}\\GenAnswersEntryPoints.bat first!");
}
// Add all assemblies from the various SERP packages (filtered by ShouldInclude)
_packageCompileAssemblies = Directory.GetFiles(Path.Combine(SerpDir, "App_Data\\Answers\\Services\\Packages"), "*.dll", SearchOption.AllDirectories)
.Where((string x) => ShouldInclude(x))
.ToList();
_packageReferenceAssemblies = new List <string>();
{
HashSet <string> packageReferenceAssemblyDirectories = new HashSet <string>();
foreach (var binFile in _packageCompileAssemblies)
{
var directory = Path.GetDirectoryName(binFile);
if (!packageReferenceAssemblyDirectories.Contains(directory))
{
packageReferenceAssemblyDirectories.Add(directory);
}
}
foreach (string binFile in ResolveReferences(packageReferenceAssemblyDirectories))
{
_packageReferenceAssemblies.Add(binFile);
}
}
_coreCompileAssemblies = new List <string>();
_coreReferenceAssemblies = new List <string>();
{
// Add an allow-list of assemblies from bin. This unified list includes binaries from /bin and /App_data so filter just the /bin assemblies
foreach (string item in GetCrossgenAllowedBinDlls())
{
string binAssembly = Path.Combine(BinDir, item);
if (File.Exists(binAssembly) &&
!FrameworkExclusion.Exclude(Path.GetFileNameWithoutExtension(binAssembly), CompilerIndex.CPAOT, out string reason))
{
_coreCompileAssemblies.Add(binAssembly);
}
}
HashSet <string> coreReferenceAssemblyDirectories = new HashSet <string>();
foreach (var binFile in _coreCompileAssemblies)
{
var directory = Path.GetDirectoryName(binFile);
if (!coreReferenceAssemblyDirectories.Contains(directory))
{
coreReferenceAssemblyDirectories.Add(directory);
}
}
foreach (string binFile in ResolveReferences(coreReferenceAssemblyDirectories))
{
_coreReferenceAssemblies.Add(binFile);
}
}
_frameworkCompileAssemblies = new List <string>();
_frameworkReferenceAssemblies = new List <string>();
{
foreach (string frameworkDll in ComputeManagedAssemblies.GetManagedAssembliesInFolder(options.CoreRootDirectory.FullName, "System.*.dll"))
{
string simpleName = Path.GetFileNameWithoutExtension(frameworkDll);
if (!FrameworkExclusion.Exclude(simpleName, CompilerIndex.CPAOT, out string reason))
{
_frameworkCompileAssemblies.Add(frameworkDll);
}
}
foreach (string frameworkDll in ComputeManagedAssemblies.GetManagedAssembliesInFolder(options.CoreRootDirectory.FullName, "Microsoft.*.dll"))
{
string simpleName = Path.GetFileNameWithoutExtension(frameworkDll);
if (!FrameworkExclusion.Exclude(simpleName, CompilerIndex.CPAOT, out string reason))
{
_frameworkCompileAssemblies.Add(frameworkDll);
}
}
_frameworkCompileAssemblies.Add(Path.Combine(options.CoreRootDirectory.FullName, "mscorlib.dll"));
_frameworkCompileAssemblies.Add(Path.Combine(options.CoreRootDirectory.FullName, "netstandard.dll"));
_frameworkReferenceAssemblies.AddRange(ComputeManagedAssemblies.GetManagedAssembliesInFolder(options.CoreRootDirectory.FullName, "System.*.dll"));
_frameworkReferenceAssemblies.AddRange(ComputeManagedAssemblies.GetManagedAssembliesInFolder(options.CoreRootDirectory.FullName, "Microsoft.*.dll"));
_frameworkReferenceAssemblies.Add(Path.Combine(options.CoreRootDirectory.FullName, "mscorlib.dll"));
_frameworkReferenceAssemblies.Add(Path.Combine(options.CoreRootDirectory.FullName, "netstandard.dll"));
}
_aspCompileAssemblies = new List <string>();
_aspReferenceAssemblies = new List <string>();
{
_aspCompileAssemblies.AddRange(ComputeManagedAssemblies.GetManagedAssembliesInFolder(options.AspNetPath.FullName, "Microsoft.AspNetCore.*.dll"));
_aspCompileAssemblies.AddRange(ComputeManagedAssemblies.GetManagedAssembliesInFolder(options.AspNetPath.FullName, "Microsoft.Extensions.*.dll"));
_aspCompileAssemblies.Add(Path.Combine(options.AspNetPath.FullName, "Microsoft.JSInterop.dll"));
_aspCompileAssemblies.Add(Path.Combine(options.AspNetPath.FullName, "Microsoft.Net.Http.Headers.dll"));
_aspCompileAssemblies.Add(Path.Combine(options.AspNetPath.FullName, "Microsoft.Win32.SystemEvents.dll"));
_aspCompileAssemblies.Add(Path.Combine(options.AspNetPath.FullName, "System.Diagnostics.EventLog.dll"));
_aspCompileAssemblies.Add(Path.Combine(options.AspNetPath.FullName, "System.Drawing.Common.dll"));
_aspCompileAssemblies.Add(Path.Combine(options.AspNetPath.FullName, "System.IO.Pipelines.dll"));
_aspCompileAssemblies.Add(Path.Combine(options.AspNetPath.FullName, "System.Security.Cryptography.Pkcs.dll"));
_aspCompileAssemblies.Add(Path.Combine(options.AspNetPath.FullName, "System.Security.Cryptography.Xml.dll"));
_aspCompileAssemblies.Add(Path.Combine(options.AspNetPath.FullName, "System.Security.Permissions.dll"));
_aspCompileAssemblies.Add(Path.Combine(options.AspNetPath.FullName, "System.Windows.Extensions.dll"));
_aspReferenceAssemblies = new List <string>(_aspCompileAssemblies);
}
}
public CrossgenRunner(BuildOptions options, IEnumerable <string> referencePaths)
: base(options, referencePaths)
{
}