public bool Execute()
{
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
List <ProcessInfo> executionsToRun = new List <ProcessInfo>();
foreach (BuildFolder folder in FoldersToBuild)
{
AddBuildFolderExecutions(executionsToRun, folder, stopwatch);
}
ParallelRunner.Run(executionsToRun, degreeOfParallelism: _options.Sequential ? 1 : 0);
List <KeyValuePair <string, string> > failedExecutionsPerBuilder = new List <KeyValuePair <string, string> >();
int successfulExecuteCount = 0;
bool success = true;
foreach (BuildFolder folder in FoldersToBuild)
{
foreach (ProcessInfo[] execution in folder.Executions)
{
string file = null;
string failedBuilders = null;
foreach (CompilerRunner runner in _compilerRunners)
{
ProcessInfo runnerProcess = execution[(int)runner.Index];
if (runnerProcess != null && !runnerProcess.Succeeded)
{
_executionFailureBuckets.AddExecution(runnerProcess);
if (file == null)
{
file = runnerProcess.Parameters.InputFileName;
failedBuilders = runner.CompilerName;
}
else
{
failedBuilders += "; " + runner.CompilerName;
}
}
}
if (file != null)
{
failedExecutionsPerBuilder.Add(new KeyValuePair <string, string>(file, failedBuilders));
success = false;
}
else
{
successfulExecuteCount++;
}
}
}
_executionMilliseconds = stopwatch.ElapsedMilliseconds;
return(success);
}
public void Run()
{
var duration = Duration.TotalSeconds;
var threadCount = ParallelRunner.ThreadCount;
using (Writer.Section($"Test settings:")) {
Writer.AppendMetric("Duration", duration * 1000, "ms");
Writer.AppendMetric("Thread count", threadCount, "");
Writer.AppendMetric("Unit size", UnitAllocator.UnitSize, "B");
}
Writer.AppendLine();
var totalCount = 0L;
for (var pass = 0; pass < PassCount; pass++)
{
var runner = ParallelRunner.New(i => new UnitAllocator(Duration));
var allocators = runner.Run();
totalCount = Math.Max(totalCount, allocators.Sum(a => a.AllocationCount));
}
var totalSize = totalCount * UnitAllocator.UnitSize;
var totalSizeWithOverhead = totalCount * (UnitAllocator.UnitSize + GarbageAllocator.ObjectSize);
using (Writer.Section($"Allocation speed:")) {
Writer.AppendMetric("Operations per second", totalCount / Sizes.Mega / duration, "M/s");
// Writer.AppendMetric("Bytes per second", totalSize / duration / Sizes.GB, "GB/s");
// Writer.AppendMetric("Bytes per second (incl. overhead)", totalSizeWithOverhead / duration / Sizes.GB, "GB/s");
}
Writer.AppendLine();
}
/// <summary>
/// Saves a scriptable object behavior tree and sets the active asset back to the behavior manager
/// </summary>
/// <param name="behaviorManager"></param>
/// <param name="filePath"></param>
/// <param name="asset"></param>
public static void SaveBehaviorAsset(this BehaviorManager behaviorManager, string filePath,
BehaviorTreeManagerAsset asset, ParallelRunner root = null)
{
if (asset == null)
{
asset = ScriptableObject.CreateInstance <BehaviorTreeManagerAsset>();
}
var runnerElementList = new List <BehaviorTreeElement>();
Debug.Log("Attempting save at path: " + filePath);
int indexS = filePath.LastIndexOf("/") + 1;
int indexD = filePath.LastIndexOf(".") - indexS;
asset.name = filePath.Substring(indexS, indexD);
var json = asset.RunnerElementsJSON;
if (behaviorManager != null)
{
behaviorManager.Reinitialize();
asset.SecondsBetweenTicks = behaviorManager.SecondsBetweenTicks;
asset.TimesToTick = behaviorManager.TimesToTick;
TreeElementUtility.TreeToList(behaviorManager.Runner, runnerElementList);
}
if (root != null)
{
TreeElementUtility.TreeToList(root, runnerElementList);
}
if (json == "" || runnerElementList.Count == 0)
{
var runner = new ParallelRunner("Extension Root", -1, -1);
runnerElementList.Add(runner);
json = JsonConvert.SerializeObject(runnerElementList, Formatting.Indented);
}
json = JsonConvert.SerializeObject(runnerElementList, Formatting.Indented);
asset.RunnerElementsJSON = json;
Debug.Log("JSON Saved: " + asset.RunnerElementsJSON);
var curPath = AssetDatabase.GetAssetPath(asset);
if (curPath == null || curPath == "")
{
Debug.Log("Creating asset: " + filePath);
AssetDatabase.CreateAsset(asset, filePath);
}
//AssetDatabase.Refresh();
EditorUtility.SetDirty(asset);
AssetDatabase.SaveAssets();
}
void CreateNewTree()
{
CustomAssetUtility.CreateAsset <BehaviorTreeManagerAsset>();
_BehaviorTreeManagerAsset = (BehaviorTreeManagerAsset)Selection.activeObject;
var root = new ParallelRunner("root", -1, -1);
BehaviorExtensions.SaveBehaviorAsset(null, AssetDatabase.GetAssetPath(_BehaviorTreeManagerAsset),
_BehaviorTreeManagerAsset, (ParallelRunner)root);
}
public void RunAndWait()
{
Task <int>[] tasks = new Task <int> [3];
tasks[0] = Task.Run(() => { Thread.Sleep(2000); return(1); });
tasks[1] = Task.Run(() => { Thread.Sleep(1000); return(2); });
tasks[2] = Task.Run(() => { Thread.Sleep(3000); return(3); });
ParallelRunner runner = new ParallelRunner();
runner.RunAndWait(tasks);
Assert.True(tasks.Select(t => t.Status).ToList().TrueForAll(s => s.Equals(TaskStatus.RanToCompletion)));
}
IList <BehaviorTreeElement> GetData()
{
if (_BehaviorTreeManagerAsset == null)
{
CreateNewTree();
}
var treeRoot = _BehaviorTreeManagerAsset.LoadFromJSON();
if (treeRoot == null)
{
treeRoot = new ParallelRunner("New Root", -1, -1);
}
var treeList = new List <BehaviorTreeElement>();
TreeElementUtility.TreeToList(treeRoot, treeList);
return(treeList);
}
public void RunsAll_AndOrdered()
{
const int count = 20;
int totalCalls = 0;
var runner = new ParallelRunner<int,int>((index, item) =>
{
Thread.Sleep(item);
Interlocked.Increment(ref totalCalls);
return new Tuple<bool, int>(true, index);
});
var items = Enumerable.Range(0, count)
.Select(x => _random.Next(100, 500)).ToArray();
var results = runner.Run(items,0, count).ToArray();
for (int i = 0; i < results.Length; i++)
{
Assert.Equal(i, results[i]);
}
Assert.Equal(count, totalCalls);
}
public void RunsAll_AndOrdered()
{
const int count = 20;
int totalCalls = 0;
var runner = new ParallelRunner <int, int>((index, item) =>
{
Thread.Sleep(item);
Interlocked.Increment(ref totalCalls);
return(new Tuple <bool, int>(true, index));
});
var items = Enumerable.Range(0, count)
.Select(x => _random.Next(100, 500)).ToArray();
var results = runner.Run(items, 0, count).ToArray();
for (int i = 0; i < results.Length; i++)
{
Assert.Equal(i, results[i]);
}
Assert.Equal(count, totalCalls);
}
public bool Compile()
{
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
List <ProcessInfo> compilationsToRun = new List <ProcessInfo>();
foreach (BuildFolder folder in _buildFolders)
{
foreach (ProcessInfo[] compilation in folder.Compilations)
{
foreach (CompilerRunner runner in _compilerRunners)
{
ProcessInfo compilationProcess = compilation[(int)runner.Index];
if (compilationProcess != null)
{
compilationsToRun.Add(compilationProcess);
}
}
}
}
_logWriter.WriteLine();
_logWriter.WriteLine($"Building {_buildFolders.Count()} folders ({compilationsToRun.Count} compilations total)");
compilationsToRun.Sort((a, b) => b.CompilationCostHeuristic.CompareTo(a.CompilationCostHeuristic));
ParallelRunner.Run(compilationsToRun, _logWriter);
bool success = true;
List <KeyValuePair <string, string> > failedCompilationsPerBuilder = new List <KeyValuePair <string, string> >();
int successfulCompileCount = 0;
foreach (BuildFolder folder in _buildFolders)
{
foreach (ProcessInfo[] compilation in folder.Compilations)
{
string file = null;
string failedBuilders = null;
foreach (CompilerRunner runner in _compilerRunners)
{
ProcessInfo runnerProcess = compilation[(int)runner.Index];
if (runnerProcess != null && !runnerProcess.Succeeded)
{
try
{
File.Copy(runnerProcess.InputFileName, runnerProcess.OutputFileName);
}
catch (Exception ex)
{
Console.Error.WriteLine("Error copying {0} to {1}: {2}", runnerProcess.InputFileName, runnerProcess.OutputFileName, ex.Message);
}
if (file == null)
{
file = runnerProcess.InputFileName;
failedBuilders = runner.CompilerName;
}
else
{
failedBuilders += "; " + runner.CompilerName;
}
}
}
if (file != null)
{
failedCompilationsPerBuilder.Add(new KeyValuePair <string, string>(file, failedBuilders));
success = false;
}
else
{
successfulCompileCount++;
}
}
}
_logWriter.WriteLine($"Compiled {successfulCompileCount} / {successfulCompileCount + failedCompilationsPerBuilder.Count} assemblies in {stopwatch.ElapsedMilliseconds} msecs.");
if (failedCompilationsPerBuilder.Count > 0)
{
int compilerRunnerCount = _compilerRunners.Count();
_logWriter.WriteLine($"Failed to compile {failedCompilationsPerBuilder.Count} assemblies:");
foreach (KeyValuePair <string, string> assemblyBuilders in failedCompilationsPerBuilder)
{
string assemblySpec = assemblyBuilders.Key;
if (compilerRunnerCount > 1)
{
assemblySpec += " (" + assemblyBuilders.Value + ")";
}
_logWriter.WriteLine(assemblySpec);
}
}
_compilationMilliseconds = stopwatch.ElapsedMilliseconds;
return(success);
}
public bool CompileFramework()
{
if (!_options.Framework)
{
return(true);
}
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
string coreRoot = _options.CoreRootDirectory.FullName;
string[] frameworkFolderFiles = Directory.GetFiles(coreRoot);
IEnumerable <CompilerRunner> frameworkRunners = _options.CompilerRunners(isFramework: true);
// Pre-populate the output folders with the input files so that we have backdrops
// for failing compilations.
foreach (CompilerRunner runner in frameworkRunners)
{
string outputPath = runner.GetOutputPath(coreRoot);
outputPath.RecreateDirectory();
}
List <ProcessInfo> compilationsToRun = new List <ProcessInfo>();
List <KeyValuePair <string, ProcessInfo[]> > compilationsPerRunner = new List <KeyValuePair <string, ProcessInfo[]> >();
foreach (string frameworkDll in ComputeManagedAssemblies.GetManagedAssembliesInFolder(_options.CoreRootDirectory.FullName))
{
ProcessInfo[] processes = new ProcessInfo[(int)CompilerIndex.Count];
compilationsPerRunner.Add(new KeyValuePair <string, ProcessInfo[]>(frameworkDll, processes));
foreach (CompilerRunner runner in frameworkRunners)
{
ProcessInfo compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, _options.CoreRootDirectory.FullName, frameworkDll));
compilationsToRun.Add(compilationProcess);
processes[(int)runner.Index] = compilationProcess;
}
}
ParallelRunner.Run(compilationsToRun, _options.DegreeOfParallelism);
HashSet <string> skipCopying = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
int[] failedCompilationsPerBuilder = new int[(int)CompilerIndex.Count];
int successfulCompileCount = 0;
int failedCompileCount = 0;
foreach (KeyValuePair <string, ProcessInfo[]> kvp in compilationsPerRunner)
{
bool anyCompilationsFailed = false;
foreach (CompilerRunner runner in frameworkRunners)
{
ProcessInfo compilationProcess = kvp.Value[(int)runner.Index];
if (compilationProcess.Succeeded)
{
skipCopying.Add(compilationProcess.Parameters.InputFileName);
}
else
{
anyCompilationsFailed = true;
failedCompilationsPerBuilder[(int)runner.Index]++;
_frameworkCompilationFailureBuckets.AddCompilation(compilationProcess);
}
}
if (anyCompilationsFailed)
{
failedCompileCount++;
}
else
{
successfulCompileCount++;
}
}
foreach (CompilerRunner runner in frameworkRunners)
{
string outputPath = runner.GetOutputPath(coreRoot);
foreach (string file in frameworkFolderFiles)
{
if (!skipCopying.Contains(file))
{
string targetFile = Path.Combine(outputPath, Path.GetFileName(file));
File.Copy(file, targetFile, overwrite: true);
}
}
}
_frameworkCompilationMilliseconds = stopwatch.ElapsedMilliseconds;
return(failedCompileCount == 0);
}
public bool Compile()
{
CompileFramework();
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
ResolveTestExclusions();
List <ProcessInfo> compilationsToRun = new List <ProcessInfo>();
foreach (BuildFolder folder in FoldersToBuild)
{
foreach (ProcessInfo[] compilation in folder.Compilations)
{
foreach (CompilerRunner runner in _compilerRunners)
{
ProcessInfo compilationProcess = compilation[(int)runner.Index];
if (compilationProcess != null)
{
compilationsToRun.Add(compilationProcess);
}
}
}
}
ParallelRunner.Run(compilationsToRun, _options.DegreeOfParallelism);
bool success = true;
List <KeyValuePair <string, string> > failedCompilationsPerBuilder = new List <KeyValuePair <string, string> >();
int successfulCompileCount = 0;
foreach (BuildFolder folder in FoldersToBuild)
{
foreach (ProcessInfo[] compilation in folder.Compilations)
{
string file = null;
string failedBuilders = null;
foreach (CompilerRunner runner in _compilerRunners)
{
ProcessInfo runnerProcess = compilation[(int)runner.Index];
if (runnerProcess != null && !runnerProcess.Succeeded)
{
_compilationFailureBuckets.AddCompilation(runnerProcess);
try
{
File.Copy(runnerProcess.Parameters.InputFileName, runnerProcess.Parameters.OutputFileName);
}
catch (Exception ex)
{
Console.Error.WriteLine("Error copying {0} to {1}: {2}", runnerProcess.Parameters.InputFileName, runnerProcess.Parameters.OutputFileName, ex.Message);
}
if (file == null)
{
file = runnerProcess.Parameters.InputFileName;
failedBuilders = runner.CompilerName;
}
else
{
failedBuilders += "; " + runner.CompilerName;
}
}
}
if (file != null)
{
failedCompilationsPerBuilder.Add(new KeyValuePair <string, string>(file, failedBuilders));
success = false;
}
else
{
successfulCompileCount++;
}
}
}
_compilationMilliseconds = stopwatch.ElapsedMilliseconds;
return(success);
}
public int CompileSerpAssemblies()
{
Console.WriteLine($"Compiling serp in {_options.CompositeScenario} scenario");
string serpRoot = Directory.GetParent(SerpDir).Parent.Parent.Parent.FullName;
string compileOutRoot = Path.Combine(serpRoot, CompileFolder);
if (Directory.Exists(compileOutRoot))
{
Directory.Delete(compileOutRoot, true);
}
// Composite FX, Composite ASP.NET, Composite Serp core, Individual package assemblies
List <ProcessInfo> fileCompilations = new List <ProcessInfo>();
bool combinedComposite = false;
bool compositeFramework = false;
bool compositeAspNet = false;
bool compositeSerpCore = false;
if (_options.CompositeScenario == SerpCompositeScenario.SerpAspNetSharedFramework)
{
compositeFramework = true;
compositeAspNet = true;
compositeSerpCore = true;
}
if (_options.CompositeScenario == SerpCompositeScenario.SerpAspNetSharedFrameworkCompositeAndPackagesComposite)
{
combinedComposite = true;
}
// Single composite image for Serp, Asp, Fx
{
if (combinedComposite)
{
List <string> combinedCompileAssemblies = new List <string>();
HashSet <string> simpleNameList = new HashSet <string>();
combinedCompileAssemblies.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _coreCompileAssemblies));
combinedCompileAssemblies.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _aspCompileAssemblies));
combinedCompileAssemblies.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _frameworkCompileAssemblies));
List <string> combinedCompileAssembliesBackup = BackupAndUseOriginalAssemblies(serpRoot, combinedCompileAssemblies);
string frameworkCompositeDll = Path.Combine(_options.CoreRootDirectory.FullName, FrameworkCompositeFilename);
if (File.Exists(frameworkCompositeDll))
{
File.Delete(frameworkCompositeDll);
}
string frameworkCompositeDllCompile = GetCompileFile(serpRoot, frameworkCompositeDll);
Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions()
{
Composite = true, CompositeRoot = GetBackupFile(serpRoot, SerpDir)
};
var runner = new Crossgen2Runner(_options, crossgen2Options, combinedCompileAssembliesBackup);
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, frameworkCompositeDllCompile, combinedCompileAssembliesBackup));
fileCompilations.Add(compilationProcess);
}
}
if (!combinedComposite)
{
// Composite FX
{
List <string> frameworkCompileAssembliesBackup = BackupAndUseOriginalAssemblies(serpRoot, _frameworkCompileAssemblies);
string frameworkCompositeDll = Path.Combine(_options.CoreRootDirectory.FullName, FrameworkCompositeFilename);
if (File.Exists(frameworkCompositeDll))
{
File.Delete(frameworkCompositeDll);
}
// Always restore the framework from the backup if present first since we run CG2 on it
var backupFrameworkDir = Path.GetDirectoryName(GetBackupFile(serpRoot, frameworkCompositeDll));
var backedUpFiles = Directory.GetFiles(backupFrameworkDir, "*.dll", SearchOption.AllDirectories);
foreach (var file in backedUpFiles)
{
string destinationFile = GetOriginalFile(serpRoot, file);
File.Copy(file, destinationFile, true);
}
if (compositeFramework)
{
string frameworkCompositeDllCompile = GetCompileFile(serpRoot, frameworkCompositeDll);
Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions()
{
Composite = true
};
var runner = new Crossgen2Runner(_options, crossgen2Options, new List <string>());
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, frameworkCompositeDllCompile, frameworkCompileAssembliesBackup));
fileCompilations.Add(compilationProcess);
}
else
{
Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions()
{
Composite = false
};
var runner = new Crossgen2Runner(_options, crossgen2Options, new List <string>());
foreach (string assembly in frameworkCompileAssembliesBackup)
{
string dllCompile = GetCompileFile(serpRoot, assembly);
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, dllCompile, new string[] { assembly }));
fileCompilations.Add(compilationProcess);
}
}
}
// Composite Asp.Net
{
List <string> aspCombinedReferences = new List <string>();
aspCombinedReferences.AddRange(_aspReferenceAssemblies);
aspCombinedReferences.AddRange(_frameworkCompileAssemblies);
List <string> aspCombinedReferencesBackup = BackupAndUseOriginalAssemblies(serpRoot, aspCombinedReferences);
List <string> aspCompileAssembliesBackup = BackupAndUseOriginalAssemblies(serpRoot, _aspCompileAssemblies);
string aspCompositeDll = Path.Combine(_options.AspNetPath.FullName, "asp-r2r.dll");
if (File.Exists(aspCompositeDll))
{
File.Delete(aspCompositeDll);
}
if (compositeAspNet)
{
string aspCompositeDllCompile = GetCompileFile(serpRoot, aspCompositeDll);
Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions()
{
Composite = true, PartialComposite = true
};
var runner = new Crossgen2Runner(_options, crossgen2Options, aspCombinedReferencesBackup);
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, aspCompositeDllCompile, aspCompileAssembliesBackup));
fileCompilations.Add(compilationProcess);
}
else
{
Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions()
{
Composite = false
};
var runner = new Crossgen2Runner(_options, crossgen2Options, aspCombinedReferencesBackup);
foreach (string assembly in aspCompileAssembliesBackup)
{
string dllCompile = GetCompileFile(serpRoot, assembly);
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, dllCompile, new string[] { assembly }));
fileCompilations.Add(compilationProcess);
}
}
}
// Composite Serp core
{
List <string> coreCombinedReferences = new List <string>();
coreCombinedReferences.AddRange(_coreReferenceAssemblies);
coreCombinedReferences.AddRange(_aspReferenceAssemblies);
coreCombinedReferences.AddRange(_frameworkReferenceAssemblies);
List <string> coreCombinedReferencesBackup = BackupAndUseOriginalAssemblies(serpRoot, coreCombinedReferences);
List <string> coreCompileAssembliesBackup = BackupAndUseOriginalAssemblies(serpRoot, _coreCompileAssemblies);
string serpCompositeDll = Path.Combine(BinDir, "serp-r2r.dll");
if (File.Exists(serpCompositeDll))
{
File.Delete(serpCompositeDll);
}
if (compositeSerpCore)
{
string coreCompositeDllCompile = GetCompileFile(serpRoot, serpCompositeDll);
Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions()
{
Composite = true, PartialComposite = true
};
var runner = new Crossgen2Runner(_options, crossgen2Options, coreCombinedReferencesBackup);
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, coreCompositeDllCompile, coreCompileAssembliesBackup));
fileCompilations.Add(compilationProcess);
}
else
{
Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions()
{
Composite = false
};
var runner = new Crossgen2Runner(_options, crossgen2Options, coreCombinedReferencesBackup);
foreach (string assembly in coreCompileAssembliesBackup)
{
string dllCompile = GetCompileFile(serpRoot, assembly);
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, dllCompile, new string[] { assembly }));
fileCompilations.Add(compilationProcess);
}
}
}
}
string packagesCompositeDirectory = null;
// Individual Serp package assemblies
{
List <string> packageCombinedReferences = new List <string>();
HashSet <string> simpleNameList = new HashSet <string>();
packageCombinedReferences.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _packageReferenceAssemblies));
packageCombinedReferences.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _coreReferenceAssemblies));
packageCombinedReferences.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _aspReferenceAssemblies));
packageCombinedReferences.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _frameworkReferenceAssemblies));
List <string> packageCombinedReferencesBackup = BackupAndUseOriginalAssemblies(serpRoot, packageCombinedReferences);
List <string> packageCompileAssembliesBackup = BackupAndUseOriginalAssemblies(serpRoot, _packageCompileAssemblies);
string packagesCompositeDll = Path.Combine(_options.CoreRootDirectory.FullName, PackagesCompositeFilename);
packagesCompositeDirectory = SerpDir;
if (File.Exists(packagesCompositeDll))
{
File.Delete(packagesCompositeDll);
}
if (combinedComposite)
{
string packagesCompositeDllCompile = GetCompileFile(serpRoot, packagesCompositeDll);
Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions()
{
Composite = true, CompositeRoot = GetBackupFile(serpRoot, SerpDir), PartialComposite = true
};
var runner = new Crossgen2Runner(_options, crossgen2Options, packageCombinedReferencesBackup);
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, packagesCompositeDllCompile, packageCompileAssembliesBackup));
fileCompilations.Add(compilationProcess);
}
else
{
Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions()
{
Composite = false
};
var runner = new Crossgen2Runner(_options, crossgen2Options, packageCombinedReferencesBackup);
foreach (string assembly in packageCompileAssembliesBackup)
{
string dllCompile = GetCompileFile(serpRoot, assembly);
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, dllCompile, new string[] { assembly }));
fileCompilations.Add(compilationProcess);
}
}
}
ParallelRunner.Run(fileCompilations, _options.DegreeOfParallelism);
bool success = true;
foreach (var compilationProcess in fileCompilations)
{
if (!compilationProcess.Succeeded)
{
success = false;
Console.WriteLine($"Failed compiling {compilationProcess.Parameters.OutputFileName}");
}
}
if (!success)
{
return(1);
}
if (combinedComposite)
{
// For combined composite, move the component assemblies we added the R2R header from the composite out folder
// to the correct compile tree destination folder so they get copied into the right place
string compositeOutputRootDir = GetCompileFile(serpRoot, _options.CoreRootDirectory.FullName);
string frameworkCompositeDll = Path.Combine(compositeOutputRootDir, FrameworkCompositeFilename);
Debug.Assert(File.Exists(frameworkCompositeDll));
var compiledCompositeFiles = Directory.GetFiles(compositeOutputRootDir, "*.dll", SearchOption.AllDirectories);
foreach (var componentAssembly in compiledCompositeFiles)
{
if (Path.GetFileName(componentAssembly).Equals(FrameworkCompositeFilename))
{
continue;
}
if (Path.GetFileName(componentAssembly).Equals(PackagesCompositeFilename))
{
continue;
}
string assemblyRelativePath = Path.GetRelativePath(compositeOutputRootDir, componentAssembly);
string destinationFile = Path.Combine(SerpDir, assemblyRelativePath);
Debug.Assert(File.Exists(GetBackupFile(serpRoot, destinationFile)));
File.Move(componentAssembly, destinationFile, true);
}
}
// Move everything we compiled to the main directory structure
var compiledFiles = Directory.GetFiles(Path.Combine(serpRoot, CompileFolder), "*.dll", SearchOption.AllDirectories);
foreach (var file in compiledFiles)
{
string destinationFile = GetOriginalFile(serpRoot, file);
File.Move(file, destinationFile, true);
}
return(success ? 0 : 1);
}
public int CompileSerpAssemblies()
{
Console.WriteLine($"Compiling serp in {_options.CompositeScenario} scenario");
string serpRoot = Directory.GetParent(SerpDir).Parent.Parent.Parent.FullName;
string compileOutRoot = Path.Combine(serpRoot, CompileFolder);
if (Directory.Exists(compileOutRoot))
{
Directory.Delete(compileOutRoot, true);
}
// Composite FX, Composite ASP.NET, Composite Serp core, Individual package assemblies
List <ProcessInfo> fileCompilations = new List <ProcessInfo>();
bool compositeFramework = false;
bool compositeAspNet = false;
bool compositeSerpCore = false;
if (_options.CompositeScenario == SerpCompositeScenario.SerpAspNetSharedFramework)
{
compositeFramework = true;
compositeAspNet = true;
compositeSerpCore = true;
}
// Composite FX
{
List <string> frameworkCompileAssembliesBackup = BackupAndUseOriginalAssemblies(serpRoot, _frameworkCompileAssemblies);
string frameworkCompositeDll = Path.Combine(_options.CoreRootDirectory.FullName, "framework-r2r.dll");
if (File.Exists(frameworkCompositeDll))
{
File.Delete(frameworkCompositeDll);
}
// Always restore the framework from the backup if present first since we run CG2 on it
var backupFrameworkDir = Path.GetDirectoryName(GetBackupFile(serpRoot, frameworkCompositeDll));
var backedUpFiles = Directory.GetFiles(backupFrameworkDir, "*.dll", SearchOption.AllDirectories);
foreach (var file in backedUpFiles)
{
string destinationFile = GetOriginalFile(serpRoot, file);
File.Copy(file, destinationFile, true);
}
if (compositeFramework)
{
string frameworkCompositeDllCompile = GetCompileFile(serpRoot, frameworkCompositeDll);
Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions()
{
Composite = true
};
var runner = new Crossgen2Runner(_options, crossgen2Options, new List <string>());
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, frameworkCompositeDllCompile, frameworkCompileAssembliesBackup));
fileCompilations.Add(compilationProcess);
}
else
{
Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions()
{
Composite = false
};
var runner = new Crossgen2Runner(_options, crossgen2Options, new List <string>());
foreach (string assembly in frameworkCompileAssembliesBackup)
{
string dllCompile = GetCompileFile(serpRoot, assembly);
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, dllCompile, new string[] { assembly }));
fileCompilations.Add(compilationProcess);
}
}
}
// Composite Asp.Net
{
List <string> aspCombinedReferencesBackup = BackupAndUseOriginalAssemblies(serpRoot, new List <string>(_frameworkReferenceAssemblies));
List <string> aspCompileAssembliesBackup = BackupAndUseOriginalAssemblies(serpRoot, _aspCompileAssemblies);
string aspCompositeDll = Path.Combine(_options.AspNetPath.FullName, "asp-r2r.dll");
if (File.Exists(aspCompositeDll))
{
File.Delete(aspCompositeDll);
}
if (compositeAspNet)
{
string aspCompositeDllCompile = GetCompileFile(serpRoot, aspCompositeDll);
Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions()
{
Composite = true, PartialComposite = true
};
var runner = new Crossgen2Runner(_options, crossgen2Options, aspCombinedReferencesBackup);
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, aspCompositeDllCompile, aspCompileAssembliesBackup));
fileCompilations.Add(compilationProcess);
}
else
{
Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions()
{
Composite = false
};
var runner = new Crossgen2Runner(_options, crossgen2Options, aspCombinedReferencesBackup);
foreach (string assembly in aspCompileAssembliesBackup)
{
string dllCompile = GetCompileFile(serpRoot, assembly);
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, dllCompile, new string[] { assembly }));
fileCompilations.Add(compilationProcess);
}
}
}
// Composite Serp core
{
List <string> coreCombinedReferences = new List <string>();
coreCombinedReferences.AddRange(_coreReferenceAssemblies);
coreCombinedReferences.AddRange(_aspReferenceAssemblies);
coreCombinedReferences.AddRange(_frameworkReferenceAssemblies);
List <string> coreCombinedReferencesBackup = BackupAndUseOriginalAssemblies(serpRoot, coreCombinedReferences);
List <string> coreCompileAssembliesBackup = BackupAndUseOriginalAssemblies(serpRoot, _coreCompileAssemblies);
string serpCompositeDll = Path.Combine(BinDir, "serp-r2r.dll");
if (File.Exists(serpCompositeDll))
{
File.Delete(serpCompositeDll);
}
if (compositeSerpCore)
{
string coreCompositeDllCompile = GetCompileFile(serpRoot, serpCompositeDll);
Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions()
{
Composite = true, PartialComposite = true
};
var runner = new Crossgen2Runner(_options, crossgen2Options, coreCombinedReferencesBackup);
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, coreCompositeDllCompile, coreCompileAssembliesBackup));
fileCompilations.Add(compilationProcess);
}
else
{
Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions()
{
Composite = false
};
var runner = new Crossgen2Runner(_options, crossgen2Options, coreCombinedReferencesBackup);
foreach (string assembly in coreCompileAssembliesBackup)
{
string dllCompile = GetCompileFile(serpRoot, assembly);
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, dllCompile, new string[] { assembly }));
fileCompilations.Add(compilationProcess);
}
}
}
// Individual Serp package assemblies
{
List <string> packageCombinedReferences = new List <string>();
packageCombinedReferences.AddRange(_packageReferenceAssemblies);
packageCombinedReferences.AddRange(_coreReferenceAssemblies);
packageCombinedReferences.AddRange(_aspReferenceAssemblies);
packageCombinedReferences.AddRange(_frameworkReferenceAssemblies);
List <string> packageCombinedReferencesBackup = BackupAndUseOriginalAssemblies(serpRoot, packageCombinedReferences);
List <string> packageCompileAssembliesBackup = BackupAndUseOriginalAssemblies(serpRoot, _packageCompileAssemblies);
Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions()
{
Composite = false
};
var runner = new Crossgen2Runner(_options, crossgen2Options, packageCombinedReferencesBackup);
foreach (string assembly in packageCompileAssembliesBackup)
{
string dllCompile = GetCompileFile(serpRoot, assembly);
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, dllCompile, new string[] { assembly }));
fileCompilations.Add(compilationProcess);
}
}
ParallelRunner.Run(fileCompilations, _options.DegreeOfParallelism);
bool success = true;
foreach (var compilationProcess in fileCompilations)
{
if (!compilationProcess.Succeeded)
{
success = false;
Console.WriteLine($"Failed compiling {compilationProcess.Parameters.OutputFileName}");
}
}
if (!success)
{
return(1);
}
// Move everything we compiled to the main directory structure
var compiledFiles = Directory.GetFiles(Path.Combine(serpRoot, CompileFolder), "*.dll", SearchOption.AllDirectories);
foreach (var file in compiledFiles)
{
string destinationFile = GetOriginalFile(serpRoot, file);
File.Move(file, destinationFile, true);
}
return(success ? 0 : 1);
}
public static int CompileSerpAssemblies(BuildOptions options)
{
if (options.InputDirectory == null)
{
Console.Error.WriteLine("Specify --response-file or --input-directory containing multiple response files.");
return(1);
}
if (options.CoreRootDirectory == null)
{
Console.Error.WriteLine("--core-root-directory (--cr) is a required argument.");
return(1);
}
// This command does not work in the context of an app, just a loose set of rsp files so don't execute anything we compile
options.NoJit = true;
options.NoEtw = true;
string serpDir = options.InputDirectory.FullName;
if (!File.Exists(Path.Combine(serpDir, "runserp.cmd")))
{
Console.Error.WriteLine($"Error: InputDirectory must point at a SERP build. Could not find {Path.Combine(serpDir, "runserp.cmd")}");
return(1);
}
string whiteListFilePath = Path.Combine(serpDir, "WhitelistDlls.txt");
if (!File.Exists(whiteListFilePath))
{
Console.Error.WriteLine($"File {whiteListFilePath} was not found");
return(1);
}
if (!File.Exists(Path.Combine(options.AspNetPath.FullName, "Microsoft.AspNetCore.dll")))
{
Console.Error.WriteLine($"Error: Asp.NET Core path must contain Microsoft.AspNetCore.dll");
return(1);
}
string binDir = Path.Combine(serpDir, "bin");
// Remove existing native images
foreach (var file in Directory.GetFiles(Path.Combine(serpDir, "App_Data\\Answers\\Services\\Packages"), "*.dll", SearchOption.AllDirectories))
{
if (file.EndsWith(".ni.dll") || file.EndsWith(".ni.exe"))
{
File.Delete(file);
}
}
foreach (var file in Directory.GetFiles(binDir, "*.dll", SearchOption.AllDirectories))
{
if (file.EndsWith(".ni.dll") || file.EndsWith(".ni.exe"))
{
File.Delete(file);
}
}
// Add all assemblies from the various SERP packages (filtered by ShouldInclude)
List <string> binFiles = Directory.GetFiles(Path.Combine(serpDir, "App_Data\\Answers\\Services\\Packages"), "*.dll", SearchOption.AllDirectories)
.Where((string x) => ShouldInclude(x))
.ToList();
// Add a whitelist of assemblies from bin
foreach (string item in new HashSet <string>(File.ReadAllLines(whiteListFilePath)))
{
binFiles.Add(Path.Combine(binDir, item));
}
HashSet <string> referenceAssemblyDirectories = new HashSet <string>();
foreach (var binFile in binFiles)
{
var directory = Path.GetDirectoryName(binFile);
if (!referenceAssemblyDirectories.Contains(directory))
{
referenceAssemblyDirectories.Add(directory);
}
}
// TestILC needs a list of all directories containing assemblies that are referenced from crossgen
List <string> referenceAssemblies = new List <string>();
HashSet <string> simpleNames = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
// Reference all managed assemblies in /bin and /App_Data/answers/services/packages
foreach (string binFile in ResolveReferences(referenceAssemblyDirectories))
{
simpleNames.Add(Path.GetFileNameWithoutExtension(binFile));
referenceAssemblies.Add(binFile);
}
referenceAssemblies.AddRange(ComputeManagedAssemblies.GetManagedAssembliesInFolderNoSimpleNameDuplicates(simpleNames, options.AspNetPath.FullName, "*.dll"));
// Add CoreRoot last because it contains various non-framework assemblies that are duplicated in SERP and we want SERP's to be used
referenceAssemblies.AddRange(ComputeManagedAssemblies.GetManagedAssembliesInFolderNoSimpleNameDuplicates(simpleNames, options.CoreRootDirectory.FullName, "System.*.dll"));
referenceAssemblies.AddRange(ComputeManagedAssemblies.GetManagedAssembliesInFolderNoSimpleNameDuplicates(simpleNames, options.CoreRootDirectory.FullName, "Microsoft.*.dll"));
referenceAssemblies.Add(Path.Combine(options.CoreRootDirectory.FullName, "mscorlib.dll"));
referenceAssemblies.Add(Path.Combine(options.CoreRootDirectory.FullName, "netstandard.dll"));
//
// binFiles is now all the assemblies that we want to compile (either individually or as composite)
// referenceAssemblies is all managed assemblies that are referenceable
//
// Remove all bin files except serp.dll so they're just referenced (eventually we'll be able to compile all these in a single composite)
foreach (string item in new HashSet <string>(File.ReadAllLines(whiteListFilePath)))
{
if (item == "Serp.dll")
{
continue;
}
binFiles.Remove(Path.Combine(binDir, item));
}
List <ProcessInfo> fileCompilations = new List <ProcessInfo>();
if (options.Composite)
{
string serpDll = Path.Combine(binDir, "Serp.dll");
var runner = new CpaotRunner(options, referenceAssemblies);
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, Path.ChangeExtension(serpDll, ".ni.dll"), binFiles));
fileCompilations.Add(compilationProcess);
}
else
{
var runner = new CpaotRunner(options, referenceAssemblies);
foreach (string assemblyName in binFiles)
{
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, Path.ChangeExtension(assemblyName, ".ni.dll"), new string[] { assemblyName }));
fileCompilations.Add(compilationProcess);
}
}
ParallelRunner.Run(fileCompilations, options.DegreeOfParallelism);
bool success = true;
int compilationFailures = 0;
foreach (var compilationProcess in fileCompilations)
{
if (!compilationProcess.Succeeded)
{
success = false;
compilationFailures++;
Console.WriteLine($"Failed compiling {compilationProcess.Parameters.OutputFileName}");
}
}
Console.WriteLine("Serp Compilation Results");
Console.WriteLine($"Total compilations: {fileCompilations.Count}");
Console.WriteLine($"Compilation failures: {compilationFailures}");
return(success ? 0 : 1);
}
/// <summary>
/// Utility mode that allows compilation of a set of assemblies using their existing Crossgen response files.
/// This is currently useful for workloads like Bing which have a large complicated web of binaries in different folders
/// with potentially different sets of reference paths used for different assemblies.
/// </summary>
public static int CompileFromCrossgenRsp(BuildOptions options)
{
if (options.CrossgenResponseFile == null && options.InputDirectory == null)
{
Console.Error.WriteLine("Specify --response-file or --input-directory containing multiple response files.");
return(1);
}
if (options.CoreRootDirectory == null)
{
Console.Error.WriteLine("--core-root-directory (--cr) is a required argument.");
return(1);
}
if (options.OutputDirectory == null)
{
if (options.InputDirectory != null)
{
options.OutputDirectory = options.InputDirectory;
}
else
{
options.OutputDirectory = new DirectoryInfo(Path.GetDirectoryName(options.CrossgenResponseFile.FullName));
}
}
else if (options.InputDirectory != null && options.OutputDirectory.IsParentOf(options.InputDirectory))
{
Console.Error.WriteLine("Error: Input and output folders must be distinct, and the output directory (which gets deleted) better not be a parent of the input directory.");
return(1);
}
// This command does not work in the context of an app, just a loose set of rsp files so don't execute anything we compile
options.NoJit = true;
options.NoEtw = true;
//
// Determine whether we're compiling a single .rsp or a folder of them
//
var responseFiles = new List <string>();
if (options.CrossgenResponseFile != null)
{
responseFiles.Add(options.CrossgenResponseFile.FullName);
}
else
{
responseFiles = Directory.EnumerateFiles(options.InputDirectory.FullName, "*.rsp", SearchOption.TopDirectoryOnly).ToList();
}
Dictionary <string, string> pathReplacements = new Dictionary <string, string>();
if ((options.RewriteOldPath == null) != (options.RewriteNewPath == null))
{
Console.Error.WriteLine("Error: --rewrite-old-path and --rewrite-new-path must both be specified if either is used.");
return(1);
}
if (options.RewriteOldPath != null && options.RewriteNewPath != null)
{
if (options.RewriteOldPath.Length != options.RewriteNewPath.Length)
{
Console.Error.WriteLine("Error: --rewrite-old-path and --rewrite-new-path were specified a different number of times.");
return(1);
}
for (int i = 0; i < options.RewriteNewPath.Length; i++)
{
pathReplacements.Add(options.RewriteOldPath[i].FullName, options.RewriteNewPath[i].FullName);
Console.WriteLine($"Re-writing path {options.RewriteOldPath[i].FullName} as {options.RewriteNewPath[i].FullName}");
}
}
bool success = true;
int compilationFailures = 0;
int totalCompilations = 0;
// Collect all the compilations first
foreach (var inputRsp in responseFiles)
{
var crossgenArguments = CrossgenArguments.ParseFromResponseFile(inputRsp)
.ReplacePaths(pathReplacements);
Console.WriteLine($"{inputRsp} -> {crossgenArguments.InputFile}");
var compilerRunners = options.CompilerRunners(false, crossgenArguments.ReferencePaths);
string responseFileOuputPath = Path.Combine(options.OutputDirectory.FullName, Path.GetFileNameWithoutExtension(inputRsp));
responseFileOuputPath.RecreateDirectory();
List <ProcessInfo> fileCompilations = new List <ProcessInfo>();
foreach (CompilerRunner runner in compilerRunners)
{
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, responseFileOuputPath, new string[] { crossgenArguments.InputFile }));
fileCompilations.Add(compilationProcess);
}
ParallelRunner.Run(fileCompilations, options.DegreeOfParallelism);
totalCompilations++;
foreach (var compilationProcess in fileCompilations)
{
if (!compilationProcess.Succeeded)
{
success = false;
compilationFailures++;
Console.WriteLine($"Failed compiling {compilationProcess.Parameters.OutputFileName}");
}
}
}
Console.WriteLine("Rsp Compilation Results");
Console.WriteLine($"Total compilations: {totalCompilations}");
Console.WriteLine($"Compilation failures: {compilationFailures}");
return(success ? 0 : 1);
}
public bool Compile()
{
CompileFramework();
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
ResolveTestExclusions();
List <ProcessInfo> compilationsToRun = new List <ProcessInfo>();
foreach (BuildFolder folder in FoldersToBuild)
{
foreach (ProcessInfo[] compilation in folder.Compilations)
{
foreach (CompilerRunner runner in _compilerRunners)
{
ProcessInfo compilationProcess = compilation[(int)runner.Index];
if (compilationProcess != null)
{
compilationsToRun.Add(compilationProcess);
}
}
}
}
ParallelRunner.Run(compilationsToRun, _options.DegreeOfParallelism);
bool success = true;
List <KeyValuePair <string, string> > failedCompilationsPerBuilder = new List <KeyValuePair <string, string> >();
int successfulCompileCount = 0;
List <ProcessInfo> r2rDumpExecutionsToRun = new List <ProcessInfo>();
foreach (BuildFolder folder in FoldersToBuild)
{
foreach (ProcessInfo[] compilation in folder.Compilations)
{
string file = null;
string failedBuilders = null;
foreach (CompilerRunner runner in _compilerRunners)
{
ProcessInfo runnerProcess = compilation[(int)runner.Index];
if (runnerProcess == null)
{
// No runner process
}
else if (runnerProcess.Succeeded)
{
if (_options.R2RDumpPath != null)
{
r2rDumpExecutionsToRun.Add(new ProcessInfo(new R2RDumpProcessConstructor(runner, runnerProcess.Parameters.OutputFileName, naked: false)));
r2rDumpExecutionsToRun.Add(new ProcessInfo(new R2RDumpProcessConstructor(runner, runnerProcess.Parameters.OutputFileName, naked: true)));
}
}
else // runner process failed
{
_compilationFailureBuckets.AddCompilation(runnerProcess);
try
{
File.Copy(runnerProcess.Parameters.InputFileName, runnerProcess.Parameters.OutputFileName);
}
catch (Exception ex)
{
Console.Error.WriteLine("Error copying {0} to {1}: {2}", runnerProcess.Parameters.InputFileName, runnerProcess.Parameters.OutputFileName, ex.Message);
}
if (file == null)
{
file = runnerProcess.Parameters.InputFileName;
failedBuilders = runner.CompilerName;
}
else
{
failedBuilders += "; " + runner.CompilerName;
}
}
}
if (file != null)
{
failedCompilationsPerBuilder.Add(new KeyValuePair <string, string>(file, failedBuilders));
success = false;
}
else
{
successfulCompileCount++;
}
}
}
ParallelRunner.Run(r2rDumpExecutionsToRun, _options.DegreeOfParallelism);
foreach (ProcessInfo r2rDumpExecution in r2rDumpExecutionsToRun)
{
if (!r2rDumpExecution.Succeeded)
{
string causeOfFailure;
if (r2rDumpExecution.TimedOut)
{
causeOfFailure = "timed out";
}
else if (r2rDumpExecution.ExitCode != 0)
{
causeOfFailure = $"invalid exit code {r2rDumpExecution.ExitCode}";
}
else
{
causeOfFailure = "Unknown cause of failure";
}
Console.Error.WriteLine("Error running R2R dump on {0}: {1}", r2rDumpExecution.Parameters.InputFileName, causeOfFailure);
success = false;
}
}
_compilationMilliseconds = stopwatch.ElapsedMilliseconds;
return(success);
}
public bool Execute()
{
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
List <ProcessInfo> executionsToRun = new List <ProcessInfo>();
foreach (BuildFolder folder in _buildFolders)
{
AddBuildFolderExecutions(executionsToRun, folder, stopwatch);
}
ParallelRunner.Run(executionsToRun, _logWriter, degreeOfParallelism: _options.Sequential ? 1 : Environment.ProcessorCount);
List <KeyValuePair <string, string> > failedExecutionsPerBuilder = new List <KeyValuePair <string, string> >();
int successfulExecuteCount = 0;
bool success = true;
foreach (BuildFolder folder in _buildFolders)
{
foreach (ProcessInfo[] execution in folder.Executions)
{
string file = null;
string failedBuilders = null;
foreach (CompilerRunner runner in _compilerRunners)
{
ProcessInfo runnerProcess = execution[(int)runner.Index];
if (runnerProcess != null && !runnerProcess.Succeeded)
{
if (file == null)
{
file = runnerProcess.InputFileName;
failedBuilders = runner.CompilerName;
}
else
{
failedBuilders += "; " + runner.CompilerName;
}
}
}
if (file != null)
{
failedExecutionsPerBuilder.Add(new KeyValuePair <string, string>(file, failedBuilders));
success = false;
}
else
{
successfulExecuteCount++;
}
}
}
_logWriter.WriteLine($"Successfully executed {successfulExecuteCount} / {successfulExecuteCount + failedExecutionsPerBuilder.Count} apps in {stopwatch.ElapsedMilliseconds} msecs.");
if (failedExecutionsPerBuilder.Count > 0)
{
int compilerRunnerCount = _compilerRunners.Count();
_logWriter.WriteLine($"Failed to execute {failedExecutionsPerBuilder.Count} apps:");
foreach (KeyValuePair <string, string> assemblyBuilders in failedExecutionsPerBuilder)
{
string assemblySpec = assemblyBuilders.Key;
if (compilerRunnerCount > 1)
{
assemblySpec += " (" + assemblyBuilders.Value + ")";
}
_logWriter.WriteLine(assemblySpec);
}
}
_executionMilliseconds = stopwatch.ElapsedMilliseconds;
return(success);
}
public static int CompileDirectory(DirectoryInfo toolDirectory, DirectoryInfo inputDirectory, DirectoryInfo outputDirectory, bool crossgen, bool cpaot, DirectoryInfo[] referencePath)
{
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
if (toolDirectory == null)
{
Console.WriteLine("--tool-directory is a required argument.");
return(1);
}
if (inputDirectory == null)
{
Console.WriteLine("--input-directory is a required argument.");
return(1);
}
if (outputDirectory == null)
{
outputDirectory = inputDirectory;
}
if (OutputPathIsParentOfInputPath(inputDirectory, outputDirectory))
{
Console.WriteLine("Error: Input and output folders must be distinct, and the output directory (which gets deleted) better not be a parent of the input directory.");
return(1);
}
CompilerRunner runner;
if (cpaot)
{
runner = new CpaotRunner(toolDirectory.ToString(), inputDirectory.ToString(), outputDirectory.ToString(), referencePath?.Select(x => x.ToString())?.ToList());
}
else
{
runner = new CrossgenRunner(toolDirectory.ToString(), inputDirectory.ToString(), outputDirectory.ToString(), referencePath?.Select(x => x.ToString())?.ToList());
}
string runnerOutputPath = runner.GetOutputPath();
if (Directory.Exists(runnerOutputPath))
{
try
{
Directory.Delete(runnerOutputPath, recursive: true);
}
catch (Exception ex) when(
ex is UnauthorizedAccessException ||
ex is DirectoryNotFoundException ||
ex is IOException
)
{
Console.WriteLine($"Error: Could not delete output folder {runnerOutputPath}. {ex.Message}");
return(1);
}
}
Directory.CreateDirectory(runnerOutputPath);
List <ProcessInfo> compilationsToRun = new List <ProcessInfo>();
// Copy unmanaged files (runtime, native dependencies, resources, etc)
foreach (string file in Directory.EnumerateFiles(inputDirectory.FullName))
{
if (ComputeManagedAssemblies.IsManaged(file))
{
ProcessInfo compilationToRun = runner.CompilationProcess(file);
compilationToRun.InputFileName = file;
compilationsToRun.Add(compilationToRun);
}
else
{
// Copy through all other files
File.Copy(file, Path.Combine(runnerOutputPath, Path.GetFileName(file)));
}
}
ParallelRunner.Run(compilationsToRun);
bool success = true;
List <string> failedCompilationAssemblies = new List <string>();
int successfulCompileCount = 0;
foreach (ProcessInfo processInfo in compilationsToRun)
{
if (processInfo.Succeeded)
{
successfulCompileCount++;
}
else
{
File.Copy(processInfo.InputFileName, Path.Combine(runnerOutputPath, Path.GetFileName(processInfo.InputFileName)));
failedCompilationAssemblies.Add(processInfo.InputFileName);
}
}
Console.WriteLine($"Compiled {successfulCompileCount}/{successfulCompileCount + failedCompilationAssemblies.Count} assemblies in {stopwatch.ElapsedMilliseconds} msecs.");
if (failedCompilationAssemblies.Count > 0)
{
Console.WriteLine($"Failed to compile {failedCompilationAssemblies.Count} assemblies:");
foreach (var assembly in failedCompilationAssemblies)
{
Console.WriteLine(assembly);
}
}
return(success ? 0 : 1);
}
public static int CompileDirectory(
DirectoryInfo inputDirectory,
DirectoryInfo outputDirectory,
DirectoryInfo crossgenDirectory,
DirectoryInfo cpaotDirectory,
DirectoryInfo[] referencePath)
{
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
if (inputDirectory == null)
{
Console.WriteLine("--input-directory is a required argument.");
return(1);
}
if (outputDirectory == null)
{
outputDirectory = inputDirectory;
}
if (outputDirectory.IsParentOf(inputDirectory))
{
Console.WriteLine("Error: Input and output folders must be distinct, and the output directory (which gets deleted) better not be a parent of the input directory.");
return(1);
}
List <string> referencePaths = referencePath?.Select(x => x.ToString())?.ToList();
string coreRunPath = null;
foreach (string path in referencePaths)
{
string candidatePath = Path.Combine(path, "CoreRun.exe");
if (File.Exists(candidatePath))
{
coreRunPath = candidatePath;
break;
}
}
if (coreRunPath == null)
{
Console.Error.WriteLine("CoreRun.exe not found in reference folders, execution won't run");
}
List <CompilerRunner> runners = new List <CompilerRunner>();
if (cpaotDirectory != null)
{
runners.Add(new CpaotRunner(cpaotDirectory.ToString(), inputDirectory.ToString(), outputDirectory.ToString(), referencePaths));
}
if (crossgenDirectory != null)
{
runners.Add(new CrossgenRunner(crossgenDirectory.ToString(), inputDirectory.ToString(), outputDirectory.ToString(), referencePaths));
}
List <string> compilationInputFiles = new List <string>();
List <string> passThroughFiles = new List <string>();
string mainExecutable = null;
// Copy unmanaged files (runtime, native dependencies, resources, etc)
foreach (string file in Directory.EnumerateFiles(inputDirectory.FullName))
{
bool isManagedAssembly = ComputeManagedAssemblies.IsManaged(file);
if (isManagedAssembly)
{
compilationInputFiles.Add(file);
}
else
{
passThroughFiles.Add(file);
}
if (Path.GetExtension(file).Equals(".exe", StringComparison.OrdinalIgnoreCase))
{
mainExecutable = file;
}
}
foreach (CompilerRunner runner in runners)
{
string runnerOutputPath = runner.GetOutputPath();
runnerOutputPath.RecreateDirectory();
foreach (string file in passThroughFiles)
{
File.Copy(file, Path.Combine(runnerOutputPath, Path.GetFileName(file)));
}
}
List <ProcessInfo> compilationsToRun = new List <ProcessInfo>();
Application application = new Application(compilationInputFiles, mainExecutable, runners, coreRunPath);
foreach (ProcessInfo[] compilation in application.Compilations)
{
foreach (CompilerRunner runner in runners)
{
compilationsToRun.Add(compilation[(int)runner.Index]);
}
}
compilationsToRun.Sort((a, b) => b.CompilationCostHeuristic.CompareTo(a.CompilationCostHeuristic));
ParallelRunner.Run(compilationsToRun);
bool success = true;
List <KeyValuePair <string, string> > failedCompilationsPerBuilder = new List <KeyValuePair <string, string> >();
int successfulCompileCount = 0;
foreach (ProcessInfo[] compilation in application.Compilations)
{
string file = null;
string failedBuilders = null;
foreach (CompilerRunner runner in runners)
{
ProcessInfo runnerProcess = compilation[(int)runner.Index];
if (!runnerProcess.Succeeded)
{
File.Copy(runnerProcess.InputFileName, runnerProcess.OutputFileName);
if (file == null)
{
file = runnerProcess.InputFileName;
failedBuilders = runner.CompilerName;
}
else
{
failedBuilders += "; " + runner.CompilerName;
}
}
}
if (file != null)
{
failedCompilationsPerBuilder.Add(new KeyValuePair <string, string>(file, failedBuilders));
success = false;
}
else
{
successfulCompileCount++;
}
}
Console.WriteLine($"Compiled {successfulCompileCount} / {successfulCompileCount + failedCompilationsPerBuilder.Count} assemblies in {stopwatch.ElapsedMilliseconds} msecs.");
if (failedCompilationsPerBuilder.Count > 0)
{
Console.WriteLine($"Failed to compile {failedCompilationsPerBuilder.Count} assemblies:");
foreach (KeyValuePair <string, string> assemblyBuilders in failedCompilationsPerBuilder)
{
string assemblySpec = assemblyBuilders.Key;
if (runners.Count > 1)
{
assemblySpec += " (" + assemblyBuilders.Value + ")";
}
Console.WriteLine(assemblySpec);
}
}
if (coreRunPath != null)
{
List <ProcessInfo> executionsToRun = new List <ProcessInfo>();
foreach (CompilerRunner runner in runners)
{
bool compilationsSucceeded = application.Compilations.All(comp => comp[(int)runner.Index].Succeeded);
if (compilationsSucceeded)
{
ProcessInfo executionProcess = application.Execution[(int)runner.Index];
if (executionProcess != null)
{
executionsToRun.Add(executionProcess);
}
}
}
ParallelRunner.Run(executionsToRun);
}
return(success ? 0 : 1);
}
public int CompileSerpAssemblies()
{
Console.WriteLine("Compiling serp in " + (_options.Composite ? "composite" : "single assembly") + " mode");
string serpRoot = Directory.GetParent(SerpDir).Parent.Parent.Parent.FullName;
string compileOutRoot = Path.Combine(serpRoot, CompileFolder);
if (Directory.Exists(compileOutRoot))
{
Directory.Delete(compileOutRoot, true);
}
List <ProcessInfo> fileCompilations = new List <ProcessInfo>();
// Single composite image for all of Serp
if (_options.Composite)
{
List <string> combinedCompileAssemblies = new List <string>();
HashSet <string> simpleNameList = new HashSet <string>();
combinedCompileAssemblies.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _coreCompileAssemblies));
combinedCompileAssemblies.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _aspCompileAssemblies));
combinedCompileAssemblies.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _frameworkCompileAssemblies));
combinedCompileAssemblies.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _packageCompileAssemblies));
List <string> combinedCompileAssembliesBackup = BackupAndUseOriginalAssemblies(serpRoot, combinedCompileAssemblies);
string compositeDllPath = Path.Combine(BinDir, SerpCompositeFilename);
if (File.Exists(compositeDllPath))
{
File.Delete(compositeDllPath);
}
string compositeDllCompile = GetCompileFile(serpRoot, compositeDllPath);
Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions()
{
Composite = true, CompositeRoot = GetBackupFile(serpRoot, SerpDir), PartialComposite = true
};
var runner = new Crossgen2Runner(_options, crossgen2Options, combinedCompileAssembliesBackup);
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, compositeDllCompile, combinedCompileAssembliesBackup));
fileCompilations.Add(compilationProcess);
}
else
{
// Framework assemblies
{
List <string> frameworkCompileAssembliesBackup = BackupAndUseOriginalAssemblies(serpRoot, _frameworkCompileAssemblies);
Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions()
{
Composite = false
};
var runner = new Crossgen2Runner(_options, crossgen2Options, new List <string>());
foreach (string assembly in frameworkCompileAssembliesBackup)
{
string dllCompile = GetCompileFile(serpRoot, assembly);
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, dllCompile, new string[] { assembly }));
fileCompilations.Add(compilationProcess);
}
}
// Asp.Net
{
List <string> aspCombinedReferences = new List <string>();
aspCombinedReferences.AddRange(_aspReferenceAssemblies);
aspCombinedReferences.AddRange(_frameworkCompileAssemblies);
List <string> aspCombinedReferencesBackup = BackupAndUseOriginalAssemblies(serpRoot, aspCombinedReferences);
List <string> aspCompileAssembliesBackup = BackupAndUseOriginalAssemblies(serpRoot, _aspCompileAssemblies);
Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions()
{
Composite = false
};
var runner = new Crossgen2Runner(_options, crossgen2Options, aspCombinedReferencesBackup);
foreach (string assembly in aspCompileAssembliesBackup)
{
string dllCompile = GetCompileFile(serpRoot, assembly);
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, dllCompile, new string[] { assembly }));
fileCompilations.Add(compilationProcess);
}
}
// Serp core
{
List <string> coreCombinedReferences = new List <string>();
coreCombinedReferences.AddRange(_coreReferenceAssemblies);
coreCombinedReferences.AddRange(_aspReferenceAssemblies);
coreCombinedReferences.AddRange(_frameworkReferenceAssemblies);
List <string> coreCombinedReferencesBackup = BackupAndUseOriginalAssemblies(serpRoot, coreCombinedReferences);
List <string> coreCompileAssembliesBackup = BackupAndUseOriginalAssemblies(serpRoot, _coreCompileAssemblies);
Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions()
{
Composite = false
};
var runner = new Crossgen2Runner(_options, crossgen2Options, coreCombinedReferencesBackup);
foreach (string assembly in coreCompileAssembliesBackup)
{
string dllCompile = GetCompileFile(serpRoot, assembly);
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, dllCompile, new string[] { assembly }));
fileCompilations.Add(compilationProcess);
}
}
// Individual Serp package assemblies
{
List <string> packageCombinedReferences = new List <string>();
HashSet <string> simpleNameList = new HashSet <string>();
packageCombinedReferences.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _packageReferenceAssemblies));
packageCombinedReferences.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _coreReferenceAssemblies));
packageCombinedReferences.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _aspReferenceAssemblies));
packageCombinedReferences.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _frameworkReferenceAssemblies));
List <string> packageCombinedReferencesBackup = BackupAndUseOriginalAssemblies(serpRoot, packageCombinedReferences);
List <string> packageCompileAssembliesBackup = BackupAndUseOriginalAssemblies(serpRoot, _packageCompileAssemblies);
Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions()
{
Composite = false
};
var runner = new Crossgen2Runner(_options, crossgen2Options, packageCombinedReferencesBackup);
foreach (string assembly in packageCompileAssembliesBackup)
{
string dllCompile = GetCompileFile(serpRoot, assembly);
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, dllCompile, new string[] { assembly }));
fileCompilations.Add(compilationProcess);
}
}
}
ParallelRunner.Run(fileCompilations, _options.DegreeOfParallelism);
bool success = true;
foreach (var compilationProcess in fileCompilations)
{
if (!compilationProcess.Succeeded)
{
success = false;
Console.WriteLine($"Failed compiling {compilationProcess.Parameters.OutputFileName}");
}
}
if (!success)
{
return(1);
}
if (_options.Composite)
{
// For combined composite, move the component assemblies we added the R2R header from the composite out folder
// to the correct compile tree destination folder so they get copied into the right place
string compositeOutputRootDir = GetCompileFile(serpRoot, BinDir);
string frameworkCompositeDll = Path.Combine(compositeOutputRootDir, SerpCompositeFilename);
Debug.Assert(File.Exists(frameworkCompositeDll));
var compiledCompositeFiles = Directory.GetFiles(compositeOutputRootDir, "*.dll", SearchOption.AllDirectories);
foreach (var componentAssembly in compiledCompositeFiles)
{
if (Path.GetFileName(componentAssembly).Equals(SerpCompositeFilename))
{
continue;
}
string assemblyRelativePath = Path.GetRelativePath(compositeOutputRootDir, componentAssembly);
string destinationFile = Path.Combine(SerpDir, assemblyRelativePath);
Debug.Assert(File.Exists(GetBackupFile(serpRoot, destinationFile)));
File.Move(componentAssembly, destinationFile, true);
}
}
// Move everything we compiled to the main directory structure
var compiledFiles = Directory.GetFiles(Path.Combine(serpRoot, CompileFolder), "*.dll", SearchOption.AllDirectories);
foreach (var file in compiledFiles)
{
string destinationFile = GetOriginalFile(serpRoot, file);
File.Move(file, destinationFile, true);
}
return(success ? 0 : 1);
}
