private void WriteJittedMethodSummary(StreamWriter logWriter)
{
Dictionary <string, HashSet <string> >[] allMethodsPerModulePerCompiler = new Dictionary <string, HashSet <string> > [(int)CompilerIndex.Count];
foreach (CompilerRunner runner in _compilerRunners)
{
allMethodsPerModulePerCompiler[(int)runner.Index] = new Dictionary <string, HashSet <string> >();
}
foreach (BuildFolder folder in FoldersToBuild)
{
for (int exeIndex = 0; exeIndex < folder.Executions.Count; exeIndex++)
{
Dictionary <string, HashSet <string> >[] appMethodsPerModulePerCompiler = new Dictionary <string, HashSet <string> > [(int)CompilerIndex.Count];
foreach (CompilerRunner runner in _compilerRunners)
{
appMethodsPerModulePerCompiler[(int)runner.Index] = new Dictionary <string, HashSet <string> >();
folder.AddModuleToJittedMethodsMapping(allMethodsPerModulePerCompiler[(int)runner.Index], exeIndex, runner.Index);
folder.AddModuleToJittedMethodsMapping(appMethodsPerModulePerCompiler[(int)runner.Index], exeIndex, runner.Index);
}
folder.WriteJitStatistics(appMethodsPerModulePerCompiler, _compilerRunners);
}
}
BuildFolder.WriteJitStatistics(logWriter, allMethodsPerModulePerCompiler, _compilerRunners);
}
public static int CompileDirectory(BuildOptions options)
{
if (options.InputDirectory == null)
{
Console.Error.WriteLine("--input-directory is a required argument.");
return(1);
}
if (options.CoreRootDirectory == null)
{
Console.Error.WriteLine("--core-root-directory (--cr) is a required argument.");
return(1);
}
if (options.OutputDirectory == null)
{
options.OutputDirectory = options.InputDirectory;
}
if (options.OutputDirectory.IsParentOf(options.InputDirectory))
{
Console.Error.WriteLine("Error: Input and output folders must be distinct, and the output directory (which gets deleted) better not be a parent of the input directory.");
return(1);
}
IEnumerable <CompilerRunner> runners = options.CompilerRunners(isFramework: false);
if (!options.Exe)
{
PathExtensions.DeleteOutputFolders(options.OutputDirectory.FullName, options.CoreRootDirectory.FullName, recursive: false);
}
BuildFolder folder = BuildFolder.FromDirectory(options.InputDirectory.FullName, runners, options.OutputDirectory.FullName, options);
if (folder == null)
{
Console.Error.WriteLine($"No managed app found in {options.InputDirectory.FullName}");
}
BuildFolderSet folderSet = new BuildFolderSet(new BuildFolder[] { folder }, runners, options);
bool success = folderSet.Build();
folderSet.WriteLogs();
if (!options.NoCleanup && !options.Exe)
{
PathExtensions.DeleteOutputFolders(options.OutputDirectory.FullName, options.CoreRootDirectory.FullName, recursive: false);
}
return(success ? 0 : 1);
}
public static int CompileDirectory(BuildOptions options)
{
if (options.InputDirectory == null)
{
Console.Error.WriteLine("--input-directory is a required argument.");
return(1);
}
if (options.OutputDirectory == null)
{
options.OutputDirectory = options.InputDirectory;
}
if (options.OutputDirectory.IsParentOf(options.InputDirectory))
{
Console.Error.WriteLine("Error: Input and output folders must be distinct, and the output directory (which gets deleted) better not be a parent of the input directory.");
return(1);
}
IEnumerable <string> referencePaths = options.ReferencePaths();
IEnumerable <CompilerRunner> runners = options.CompilerRunners();
PathExtensions.DeleteOutputFolders(options.OutputDirectory.FullName, recursive: false);
BuildFolder folder = BuildFolder.FromDirectory(options.InputDirectory.FullName, runners, options.OutputDirectory.FullName, options);
if (folder == null)
{
Console.Error.WriteLine($"No managed app found in {options.InputDirectory.FullName}");
}
string timeStamp = DateTime.Now.ToString("MMdd-hhmm");
string folderSetLogPath = Path.Combine(options.InputDirectory.ToString(), "directory-" + timeStamp + ".log");
using (BuildFolderSet folderSet = new BuildFolderSet(new BuildFolder[] { folder }, runners, options, folderSetLogPath))
{
bool success = folderSet.Build(runners, folderSetLogPath);
if (!options.NoCleanup)
{
PathExtensions.DeleteOutputFolders(options.OutputDirectory.FullName, recursive: false);
}
return(success ? 0 : 1);
}
}
private void AddBuildFolderExecutions(List <ProcessInfo> executionsToRun, BuildFolder folder, Stopwatch stopwatch)
{
foreach (ProcessInfo[] execution in folder.Executions)
{
foreach (CompilerRunner runner in _compilerRunners)
{
ProcessInfo executionProcess = execution[(int)runner.Index];
if (executionProcess != null)
{
bool compilationsSucceeded = folder.Compilations.All(comp => comp[(int)runner.Index]?.Succeeded ?? true);
if (compilationsSucceeded)
{
executionsToRun.Add(executionProcess);
}
}
}
}
}
public static int CompileNuget(BuildOptions options)
{
if (options.CoreRootDirectory == null)
{
Console.Error.WriteLine("--core-root-directory (--cr) is a required argument.");
return(1);
}
// We don't want to launch these apps when building the folder set below
options.NoExe = true;
options.NoJit = true;
options.InputDirectory = options.OutputDirectory;
IList <string> packageList = ReadPackageNames(options.PackageList.FullName);
if (options.OutputDirectory == null)
{
Console.Error.WriteLine("--output-directory is a required argument.");
return(1);
}
IEnumerable <string> referencePaths = options.ReferencePaths();
IEnumerable <CompilerRunner> runners = options.CompilerRunners(false);
if (!options.Exe)
{
PathExtensions.DeleteOutputFolders(options.OutputDirectory.FullName, options.CoreRootDirectory.FullName, runners, recursive: false);
}
string nugetOutputFolder = Path.Combine(options.OutputDirectory.FullName, "nuget.out");
Directory.CreateDirectory(nugetOutputFolder);
var publishedAppFoldersToCompile = new List <BuildFolder>();
using (StreamWriter nugetLog = File.CreateText(Path.Combine(nugetOutputFolder, "nugetLog.txt")))
{
foreach (var package in packageList)
{
nugetLog.WriteLine($"Creating empty app for {package}");
// Create an app folder
string appFolder = Path.Combine(nugetOutputFolder, $"{package}.TestApp");
Directory.CreateDirectory(appFolder);
int exitCode = DotnetCli.New(appFolder, "console", nugetLog);
if (exitCode != 0)
{
nugetLog.WriteLine($"dotnet new console for {package} failed with exit code {exitCode}");
continue;
}
exitCode = DotnetCli.AddPackage(appFolder, package, nugetLog);
if (exitCode != 0)
{
nugetLog.WriteLine($"dotnet add package {package} failed with exit code {exitCode}");
continue;
}
exitCode = DotnetCli.Publish(appFolder, nugetLog);
if (exitCode != 0)
{
nugetLog.WriteLine($"dotnet publish failed with exit code {exitCode}");
continue;
}
// This is not a reliable way of building the publish folder
string publishFolder = Path.Combine(appFolder, @"bin\Debug\netcoreapp3.0\publish");
if (!Directory.Exists(publishFolder))
{
nugetLog.WriteLine($"Could not find folder {publishFolder} containing the published app.");
continue;
}
publishedAppFoldersToCompile.Add(BuildFolder.FromDirectory(publishFolder, runners, appFolder, options));
}
BuildFolderSet folderSet = new BuildFolderSet(publishedAppFoldersToCompile, runners, options);
bool success = folderSet.Build();
folderSet.WriteLogs();
if (!options.NoCleanup && !options.Exe)
{
PathExtensions.DeleteOutputFolders(options.OutputDirectory.FullName, options.CoreRootDirectory.FullName, runners, recursive: false);
}
return(success ? 0 : 1);
}
}
public static int CompileSubtree(BuildOptions options)
{
if (options.InputDirectory == null)
{
Console.WriteLine("--input-directory is a required argument.");
return(1);
}
if (options.CoreRootDirectory == null)
{
Console.Error.WriteLine("--core-root-directory (--cr) is a required argument.");
return(1);
}
if (options.OutputDirectory == null)
{
options.OutputDirectory = options.InputDirectory;
}
if (options.OutputDirectory.IsParentOf(options.InputDirectory))
{
Console.WriteLine("Error: Input and output folders must be distinct, and the output directory (which gets deleted) better not be a parent of the input directory.");
return(1);
}
IEnumerable <CompilerRunner> runners = options.CompilerRunners(isFramework: false);
if (!options.Exe)
{
PathExtensions.DeleteOutputFolders(options.OutputDirectory.FullName, options.CoreRootDirectory.FullName, runners, recursive: true);
}
string[] directories = LocateSubtree(options.InputDirectory.FullName, options.CoreRootDirectory.FullName).ToArray();
ConcurrentBag <BuildFolder> folders = new ConcurrentBag <BuildFolder>();
int relativePathOffset = options.InputDirectory.FullName.Length;
if (relativePathOffset > 0 && options.InputDirectory.FullName[relativePathOffset - 1] != Path.DirectorySeparatorChar)
{
relativePathOffset++;
}
int folderCount = 0;
int compilationCount = 0;
int executionCount = 0;
Parallel.ForEach(directories, (string directory) =>
{
string outputDirectoryPerFolder = options.OutputDirectory.FullName;
if (directory.Length > relativePathOffset)
{
outputDirectoryPerFolder = Path.Combine(outputDirectoryPerFolder, directory.Substring(relativePathOffset));
}
try
{
BuildFolder folder = BuildFolder.FromDirectory(directory.ToString(), runners, outputDirectoryPerFolder, options);
if (folder != null)
{
folders.Add(folder);
Interlocked.Add(ref compilationCount, folder.Compilations.Count);
Interlocked.Add(ref executionCount, folder.Executions.Count);
}
}
catch (Exception ex)
{
Console.Error.WriteLine("Error scanning folder {0}: {1}", directory, ex.Message);
}
int currentCount = Interlocked.Increment(ref folderCount);
if (currentCount % 100 == 0)
{
StringBuilder lineReport = new StringBuilder();
lineReport.Append($@"Found {folders.Count} folders to build ");
lineReport.Append($@"({compilationCount} compilations, ");
if (!options.NoExe)
{
lineReport.Append($@"{executionCount} executions, ");
}
lineReport.Append($@"{currentCount} / {directories.Length} folders scanned)");
Console.WriteLine(lineReport.ToString());
}
});
Console.Write($@"Found {folders.Count} folders to build ({compilationCount} compilations, ");
if (!options.NoExe)
{
Console.Write($@"{executionCount} executions, ");
}
Console.WriteLine($@"{directories.Length} folders scanned)");
BuildFolderSet folderSet = new BuildFolderSet(folders, runners, options);
bool success = folderSet.Build();
folderSet.WriteLogs();
if (!options.NoCleanup)
{
PathExtensions.DeleteOutputFolders(options.OutputDirectory.FullName, options.CoreRootDirectory.FullName, runners, recursive: true);
}
return(success ? 0 : 1);
}
public static int CompileSubtree(BuildOptions options)
{
if (options.InputDirectory == null)
{
Console.WriteLine("--input-directory is a required argument.");
return(1);
}
if (options.OutputDirectory == null)
{
options.OutputDirectory = options.InputDirectory;
}
if (options.OutputDirectory.IsParentOf(options.InputDirectory))
{
Console.WriteLine("Error: Input and output folders must be distinct, and the output directory (which gets deleted) better not be a parent of the input directory.");
return(1);
}
IEnumerable <CompilerRunner> runners = options.CompilerRunners(isFramework: false);
PathExtensions.DeleteOutputFolders(options.OutputDirectory.ToString(), recursive: true);
string[] directories = new string[] { options.InputDirectory.FullName }
.Concat(
options.InputDirectory
.EnumerateDirectories("*", SearchOption.AllDirectories)
.Select(dirInfo => dirInfo.FullName)
.Where(path => !Path.GetExtension(path).Equals(".out", StringComparison.OrdinalIgnoreCase)))
.ToArray();
List <BuildFolder> folders = new List <BuildFolder>();
int relativePathOffset = directories[0].Length + 1;
int folderCount = 0;
int compilationCount = 0;
int executionCount = 0;
foreach (string directory in directories)
{
string outputDirectoryPerFolder = options.OutputDirectory.FullName;
if (directory.Length > relativePathOffset)
{
outputDirectoryPerFolder = Path.Combine(outputDirectoryPerFolder, directory.Substring(relativePathOffset));
}
try
{
BuildFolder folder = BuildFolder.FromDirectory(directory.ToString(), runners, outputDirectoryPerFolder, options);
if (folder != null)
{
folders.Add(folder);
compilationCount += folder.Compilations.Count;
executionCount += folder.Executions.Count;
}
}
catch (Exception ex)
{
Console.Error.WriteLine("Error scanning folder {0}: {1}", directory, ex.Message);
}
if (++folderCount % 100 == 0)
{
Console.Write($@"Found {folders.Count} folders to build ");
Console.Write($@"({compilationCount} compilations, ");
if (!options.NoExe)
{
Console.Write($@"{executionCount} executions, ");
}
Console.WriteLine($@"{folderCount} / {directories.Length} folders scanned)");
}
}
Console.Write($@"Found {folders.Count} folders to build ({compilationCount} compilations, ");
if (!options.NoExe)
{
Console.Write($@"{executionCount} executions, ");
}
Console.WriteLine($@"{directories.Length} folders scanned)");
BuildFolderSet folderSet = new BuildFolderSet(folders, runners, options);
bool success = folderSet.Build(runners);
folderSet.WriteLogs();
if (!options.NoCleanup)
{
PathExtensions.DeleteOutputFolders(options.OutputDirectory.ToString(), recursive: true);
}
return(success ? 0 : 1);
}
public static int CompileSubtree(BuildOptions options)
{
if (options.InputDirectory == null)
{
Console.WriteLine("--input-directory is a required argument.");
return(1);
}
if (options.OutputDirectory == null)
{
options.OutputDirectory = options.InputDirectory;
}
if (options.OutputDirectory.IsParentOf(options.InputDirectory))
{
Console.WriteLine("Error: Input and output folders must be distinct, and the output directory (which gets deleted) better not be a parent of the input directory.");
return(1);
}
IEnumerable <string> referencePaths = options.ReferencePaths();
IEnumerable <CompilerRunner> runners = options.CompilerRunners();
PathExtensions.DeleteOutputFolders(options.OutputDirectory.ToString(), recursive: true);
string[] directories = new string[] { options.InputDirectory.FullName }
.Concat(
options.InputDirectory
.EnumerateDirectories("*", SearchOption.AllDirectories)
.Select(dirInfo => dirInfo.FullName)
.Where(path => !Path.GetExtension(path).Equals(".out", StringComparison.OrdinalIgnoreCase)))
.ToArray();
List <BuildFolder> folders = new List <BuildFolder>();
int relativePathOffset = directories[0].Length + 1;
int folderCount = 0;
int compilationCount = 0;
int executionCount = 0;
foreach (string directory in directories)
{
string outputDirectoryPerFolder = options.OutputDirectory.FullName;
if (directory.Length > relativePathOffset)
{
outputDirectoryPerFolder = Path.Combine(outputDirectoryPerFolder, directory.Substring(relativePathOffset));
}
try
{
BuildFolder folder = BuildFolder.FromDirectory(directory.ToString(), runners, outputDirectoryPerFolder, options);
if (folder != null)
{
folders.Add(folder);
compilationCount += folder.Compilations.Count;
executionCount += folder.Executions.Count;
}
}
catch (Exception ex)
{
Console.Error.WriteLine("Error scanning folder {0}: {1}", directory, ex.Message);
}
if (++folderCount % 100 == 0)
{
Console.Write($@"Found {folders.Count} folders to build ");
Console.Write($@"({compilationCount} compilations, ");
if (!options.NoExe)
{
Console.Write($@"{executionCount} executions, ");
}
Console.WriteLine($@"{folderCount} / {directories.Length} folders scanned)");
}
}
Console.Write($@"Found {folders.Count} folders to build ({compilationCount} compilations, ");
if (!options.NoExe)
{
Console.Write($@"{executionCount} executions, ");
}
Console.WriteLine($@"{directories.Length} folders scanned)");
string timeStamp = DateTime.Now.ToString("MMdd-hhmm");
string folderSetLogPath = Path.Combine(options.OutputDirectory.ToString(), "subtree-" + timeStamp + ".log");
using (BuildFolderSet folderSet = new BuildFolderSet(folders, runners, options, folderSetLogPath))
{
bool success = folderSet.Build(runners, folderSetLogPath);
Dictionary <string, List <ProcessInfo> > compilationFailureBuckets = new Dictionary <string, List <ProcessInfo> >();
Dictionary <string, List <ProcessInfo> > executionFailureBuckets = new Dictionary <string, List <ProcessInfo> >();
string combinedSetLogPath = Path.Combine(options.OutputDirectory.ToString(), "combined-" + timeStamp + ".log");
using (StreamWriter combinedLog = new StreamWriter(combinedSetLogPath))
{
StreamWriter[] perRunnerLog = new StreamWriter[(int)CompilerIndex.Count];
foreach (CompilerRunner runner in runners)
{
string runnerLogPath = Path.Combine(options.OutputDirectory.ToString(), runner.CompilerName + "-" + timeStamp + ".log");
perRunnerLog[(int)runner.Index] = new StreamWriter(runnerLogPath);
}
foreach (BuildFolder folder in folderSet.BuildFolders)
{
bool[] compilationErrorPerRunner = new bool[(int)CompilerIndex.Count];
foreach (ProcessInfo[] compilation in folder.Compilations)
{
foreach (CompilerRunner runner in runners)
{
ProcessInfo compilationProcess = compilation[(int)runner.Index];
if (compilationProcess != null)
{
string log = $"\nCOMPILE {runner.CompilerName}:{compilationProcess.InputFileName}\n" + File.ReadAllText(compilationProcess.LogPath);
perRunnerLog[(int)runner.Index].Write(log);
combinedLog.Write(log);
if (!compilationProcess.Succeeded)
{
string bucket = AnalyzeCompilationFailure(compilationProcess);
List <ProcessInfo> processes;
if (!compilationFailureBuckets.TryGetValue(bucket, out processes))
{
processes = new List <ProcessInfo>();
compilationFailureBuckets.Add(bucket, processes);
}
processes.Add(compilationProcess);
compilationErrorPerRunner[(int)runner.Index] = true;
}
}
}
}
foreach (ProcessInfo[] execution in folder.Executions)
{
foreach (CompilerRunner runner in runners)
{
if (!compilationErrorPerRunner[(int)runner.Index])
{
ProcessInfo executionProcess = execution[(int)runner.Index];
if (executionProcess != null)
{
string log = $"\nEXECUTE {runner.CompilerName}:{executionProcess.InputFileName}\n";
try
{
log += File.ReadAllText(executionProcess.LogPath);
}
catch (Exception ex)
{
log += " -> " + ex.Message;
}
perRunnerLog[(int)runner.Index].Write(log);
combinedLog.Write(log);
if (!executionProcess.Succeeded)
{
string bucket = AnalyzeExecutionFailure(executionProcess);
List <ProcessInfo> processes;
if (!executionFailureBuckets.TryGetValue(bucket, out processes))
{
processes = new List <ProcessInfo>();
executionFailureBuckets.Add(bucket, processes);
}
processes.Add(executionProcess);
}
}
}
}
}
}
foreach (CompilerRunner runner in runners)
{
perRunnerLog[(int)runner.Index].Dispose();
}
}
string compilationBucketsFile = Path.Combine(options.OutputDirectory.ToString(), "compilation-buckets-" + timeStamp + ".log");
OutputBuckets(compilationFailureBuckets, compilationBucketsFile);
string executionBucketsFile = Path.Combine(options.OutputDirectory.ToString(), "execution-buckets-" + timeStamp + ".log");
OutputBuckets(executionFailureBuckets, executionBucketsFile);
if (!options.NoCleanup)
{
PathExtensions.DeleteOutputFolders(options.OutputDirectory.ToString(), recursive: true);
}
return(success ? 0 : 1);
}
}
