public override bool ExecuteActions(List <Action> ActionsToExecute)
{
bool XGEResult = true;
// Batch up XGE execution by actions with the same output event handler.
List <Action> ActionBatch = new List <Action>();
ActionBatch.Add(ActionsToExecute[0]);
for (int ActionIndex = 1; ActionIndex < ActionsToExecute.Count && XGEResult; ++ActionIndex)
{
Action CurrentAction = ActionsToExecute[ActionIndex];
if (CurrentAction.OutputEventHandler == ActionBatch[0].OutputEventHandler)
{
ActionBatch.Add(CurrentAction);
}
else
{
XGEResult = XGE.ExecuteActionBatch(ActionBatch);
ActionBatch.Clear();
ActionBatch.Add(CurrentAction);
}
}
if (ActionBatch.Count > 0 && XGEResult)
{
XGEResult = XGE.ExecuteActionBatch(ActionBatch);
ActionBatch.Clear();
}
return(XGEResult);
}
public static ExecutionResult ExecuteActions(List <Action> Actions)
{
ExecutionResult XGEResult = ExecutionResult.TasksSucceeded;
if (Actions.Count > 0)
{
// Write the actions to execute to a XGE task file.
string XGETaskFilePath = Path.Combine(BuildConfiguration.BaseIntermediatePath, "XGETasks.xml");
WriteTaskFile(Actions, XGETaskFilePath, ProgressWriter.bWriteMarkup);
if (BuildConfiguration.bXGEExport)
{
SaveXGEFile(XGETaskFilePath);
}
else
{
// Try to execute the XGE tasks, and if XGE is available, skip the local execution fallback.
if (Telemetry.IsAvailable())
{
try
{
const string BuilderKey = "HKEY_LOCAL_MACHINE\\SOFTWARE\\Wow6432Node\\Xoreax\\IncrediBuild\\Builder";
string CPUUtilization = Registry.GetValue(BuilderKey, "ForceCPUCount", "").ToString();
string AvoidTaskExecutionOnLocalMachine = Registry.GetValue(BuilderKey, "AvoidLocalExec", "").ToString();
string RestartRemoteProcessesOnLocalMachine = Registry.GetValue(BuilderKey, "AllowDoubleTargets", "").ToString();
string LimitMaxNumberOfCores = Registry.GetValue(BuilderKey, "MaxHelpers", "").ToString();
string WriteOutputToDiskInBackground = Registry.GetValue(BuilderKey, "LazyOutputWriter_Beta", "").ToString();
string MaxConcurrentPDBs = Registry.GetValue(BuilderKey, "MaxConcurrentPDBs", "").ToString();
string EnabledAsHelper = Registry.GetValue(BuilderKey, "LastEnabled", "").ToString();
Telemetry.SendEvent("XGESettings.2",
"CPUUtilization", CPUUtilization,
"AvoidTaskExecutionOnLocalMachine", AvoidTaskExecutionOnLocalMachine,
"RestartRemoteProcessesOnLocalMachine", RestartRemoteProcessesOnLocalMachine,
"LimitMaxNumberOfCores", LimitMaxNumberOfCores,
"WriteOutputToDiskInBackground", WriteOutputToDiskInBackground,
"MaxConcurrentPDBs", MaxConcurrentPDBs,
"EnabledAsHelper", EnabledAsHelper);
}
catch
{
}
}
XGEResult = XGE.ExecuteTaskFileWithProgressMarkup(XGETaskFilePath, Actions.Count, (Sender, Args) =>
{
if (Actions[0].OutputEventHandler != null)
{
Actions[0].OutputEventHandler(Sender, Args);
}
else
{
Console.WriteLine(Args.Data);
}
});
}
}
return(XGEResult);
}
/// <summary>
/// Executes a list of actions.
/// </summary>
public static void ExecuteActions(BuildConfiguration BuildConfiguration, List <Action> ActionsToExecute, List <TargetDescriptor> TargetDescriptors = null)
{
Log.TraceInformation("Execute actions num:" + ActionsToExecute.Count + " target descriptors num:" + TargetDescriptors.Count);
if (ActionsToExecute.Count == 0)
{
Log.TraceInformation("Target is up to date");
}
else
{
// Figure out which executor to use
ActionExecutor Executor;
if (BuildConfiguration.bAllowHybridExecutor && HybridExecutor.IsAvailable())
{
Executor = new HybridExecutor();
}
else if (BuildConfiguration.bAllowXGE && XGE.IsAvailable())
{
Executor = new XGE();
}
else if (BuildConfiguration.bAllowDistcc)
{
Executor = new Distcc();
}
else if (BuildConfiguration.bAllowSNDBS && SNDBS.IsAvailable())
{
Executor = new SNDBS();
}
else if (BuildConfiguration.bAllowParallelExecutor && ParallelExecutor.IsAvailable())
{
Executor = new ParallelExecutor();
}
else
{
Executor = new LocalExecutor();
}
//if (BuildConfiguration.bAllowFastBuild && FastBuild_v1.IsAvailable())
//{
// Executor = new FastBuild_v1(Executor, TargetDescriptors);
//}
if (BuildConfiguration.bAllowFastBuild && FastBuild_v2.IsAvailable())
{
Executor = new FastBuild_v2(Executor, TargetDescriptors);
}
// Execute the build
Stopwatch Timer = Stopwatch.StartNew();
if (!Executor.ExecuteActions(ActionsToExecute, BuildConfiguration.bLogDetailedActionStats))
{
throw new CompilationResultException(CompilationResult.OtherCompilationError);
}
Log.TraceInformation("Total time in {0} executor: {1:0.00} seconds", Executor.Name, Timer.Elapsed.TotalSeconds);
// Reset the file info for all the produced items
foreach (Action BuildAction in ActionsToExecute)
{
foreach (FileItem ProducedItem in BuildAction.ProducedItems)
{
ProducedItem.ResetCachedInfo();
}
}
// Verify the link outputs were created (seems to happen with Win64 compiles)
foreach (Action BuildAction in ActionsToExecute)
{
if (BuildAction.ActionType == ActionType.Link)
{
foreach (FileItem Item in BuildAction.ProducedItems)
{
if (!Item.Exists)
{
throw new BuildException("Failed to produce item: {0}", Item.AbsolutePath);
}
}
}
}
}
}
/// <summary>
/// Build a list of targets with a given set of makefiles.
/// </summary>
/// <param name="Makefiles">Makefiles created with CreateMakefiles</param>
/// <param name="TargetDescriptors">Target descriptors</param>
/// <param name="BuildConfiguration">Current build configuration</param>
/// <param name="WorkingSet">The source file working set</param>
/// <param name="Options">Additional options for the build</param>
/// <param name="WriteOutdatedActionsFile">Files to write the list of outdated actions to (rather than building them)</param>
/// <returns>Result from the compilation</returns>
static void Build(TargetMakefile[] Makefiles, List <TargetDescriptor> TargetDescriptors, BuildConfiguration BuildConfiguration, ISourceFileWorkingSet WorkingSet, BuildOptions Options, FileReference WriteOutdatedActionsFile)
{
// Export the actions for each target
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
TargetDescriptor TargetDescriptor = TargetDescriptors[TargetIdx];
foreach (FileReference WriteActionFile in TargetDescriptor.WriteActionFiles)
{
Log.TraceInformation("Writing actions to {0}", WriteActionFile);
ActionGraph.ExportJson(Makefiles[TargetIdx].Actions, WriteActionFile);
}
}
// Execute the build
if ((Options & BuildOptions.SkipBuild) == 0)
{
// Make sure that none of the actions conflict with any other (producing output files differently, etc...)
ActionGraph.CheckForConflicts(Makefiles.SelectMany(x => x.Actions));
// Check we don't exceed the nominal max path length
using (Timeline.ScopeEvent("ActionGraph.CheckPathLengths"))
{
ActionGraph.CheckPathLengths(BuildConfiguration, Makefiles.SelectMany(x => x.Actions));
}
// Clean up any previous hot reload runs, and reapply the current state if it's already active
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
HotReload.Setup(TargetDescriptors[TargetIdx], Makefiles[TargetIdx], BuildConfiguration);
}
// Merge the action graphs together
List <Action> MergedActions;
if (TargetDescriptors.Count == 1)
{
MergedActions = new List <Action>(Makefiles[0].Actions);
}
else
{
MergedActions = MergeActionGraphs(TargetDescriptors, Makefiles);
}
// Gather all the prerequisite actions that are part of the targets
HashSet <FileItem> MergedOutputItems = new HashSet <FileItem>();
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
GatherOutputItems(TargetDescriptors[TargetIdx], Makefiles[TargetIdx], MergedOutputItems);
}
// Link all the actions together
ActionGraph.Link(MergedActions);
// Get all the actions that are prerequisites for these targets. This forms the list of actions that we want executed.
List <Action> PrerequisiteActions = ActionGraph.GatherPrerequisiteActions(MergedActions, MergedOutputItems);
// Create the action history
ActionHistory History = new ActionHistory();
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
using (Timeline.ScopeEvent("Reading action history"))
{
TargetDescriptor TargetDescriptor = TargetDescriptors[TargetIdx];
if (TargetDescriptor.ProjectFile != null)
{
History.Mount(TargetDescriptor.ProjectFile.Directory);
}
}
}
// Figure out which actions need to be built
Dictionary <Action, bool> ActionToOutdatedFlag = new Dictionary <Action, bool>();
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
TargetDescriptor TargetDescriptor = TargetDescriptors[TargetIdx];
// Create the dependencies cache
CppDependencyCache CppDependencies;
using (Timeline.ScopeEvent("Reading dependency cache"))
{
CppDependencies = CppDependencyCache.CreateHierarchy(TargetDescriptor.ProjectFile, TargetDescriptor.Name, TargetDescriptor.Platform, TargetDescriptor.Configuration, Makefiles[TargetIdx].TargetType, TargetDescriptor.Architecture);
}
// Plan the actions to execute for the build. For single file compiles, always rebuild the source file regardless of whether it's out of date.
if (TargetDescriptor.SpecificFilesToCompile.Count == 0)
{
ActionGraph.GatherAllOutdatedActions(PrerequisiteActions, History, ActionToOutdatedFlag, CppDependencies, BuildConfiguration.bIgnoreOutdatedImportLibraries);
}
else
{
foreach (FileReference SpecificFile in TargetDescriptor.SpecificFilesToCompile)
{
foreach (Action PrerequisiteAction in PrerequisiteActions.Where(x => x.PrerequisiteItems.Any(y => y.Location == SpecificFile)))
{
ActionToOutdatedFlag[PrerequisiteAction] = true;
}
}
}
}
// Link the action graph again to sort it
List <Action> MergedActionsToExecute = ActionToOutdatedFlag.Where(x => x.Value).Select(x => x.Key).ToList();
ActionGraph.Link(MergedActionsToExecute);
// Allow hot reload to override the actions
int HotReloadTargetIdx = -1;
for (int Idx = 0; Idx < TargetDescriptors.Count; Idx++)
{
if (TargetDescriptors[Idx].HotReloadMode != HotReloadMode.Disabled)
{
if (HotReloadTargetIdx != -1)
{
throw new BuildException("Unable to perform hot reload with multiple targets.");
}
else
{
MergedActionsToExecute = HotReload.PatchActionsForTarget(BuildConfiguration, TargetDescriptors[Idx], Makefiles[Idx], PrerequisiteActions, MergedActionsToExecute);
}
HotReloadTargetIdx = Idx;
}
}
// Make sure we're not modifying any engine files
if ((Options & BuildOptions.NoEngineChanges) != 0)
{
List <FileItem> EngineChanges = MergedActionsToExecute.SelectMany(x => x.ProducedItems).Where(x => x.Location.IsUnderDirectory(UnrealBuildTool.EngineDirectory)).Distinct().OrderBy(x => x.FullName).ToList();
if (EngineChanges.Count > 0)
{
StringBuilder Result = new StringBuilder("Building would modify the following engine files:\n");
foreach (FileItem EngineChange in EngineChanges)
{
Result.AppendFormat("\n{0}", EngineChange.FullName);
}
Result.Append("\n\nPlease rebuild from an IDE instead.");
Log.TraceError("{0}", Result.ToString());
throw new CompilationResultException(CompilationResult.FailedDueToEngineChange);
}
}
// Make sure the appropriate executor is selected
foreach (TargetDescriptor TargetDescriptor in TargetDescriptors)
{
UEBuildPlatform BuildPlatform = UEBuildPlatform.GetBuildPlatform(TargetDescriptor.Platform);
BuildConfiguration.bAllowXGE &= BuildPlatform.CanUseXGE();
BuildConfiguration.bAllowDistcc &= BuildPlatform.CanUseDistcc();
BuildConfiguration.bAllowSNDBS &= BuildPlatform.CanUseSNDBS();
}
// Delete produced items that are outdated.
ActionGraph.DeleteOutdatedProducedItems(MergedActionsToExecute);
// Save all the action histories now that files have been removed. We have to do this after deleting produced items to ensure that any
// items created during the build don't have the wrong command line.
History.Save();
// Create directories for the outdated produced items.
ActionGraph.CreateDirectoriesForProducedItems(MergedActionsToExecute);
// Execute the actions
if ((Options & BuildOptions.XGEExport) != 0)
{
OutputToolchainInfo(TargetDescriptors, Makefiles);
// Just export to an XML file
using (Timeline.ScopeEvent("XGE.ExportActions()"))
{
XGE.ExportActions(MergedActionsToExecute);
}
}
else if (WriteOutdatedActionsFile != null)
{
OutputToolchainInfo(TargetDescriptors, Makefiles);
// Write actions to an output file
using (Timeline.ScopeEvent("ActionGraph.WriteActions"))
{
ActionGraph.ExportJson(MergedActionsToExecute, WriteOutdatedActionsFile);
}
}
else
{
// Execute the actions
if (MergedActionsToExecute.Count == 0)
{
if (TargetDescriptors.Any(x => !x.bQuiet))
{
Log.TraceInformation((TargetDescriptors.Count == 1)? "Target is up to date" : "Targets are up to date");
}
}
else
{
if (TargetDescriptors.Any(x => !x.bQuiet))
{
Log.TraceInformation("Building {0}...", StringUtils.FormatList(TargetDescriptors.Select(x => x.Name).Distinct()));
}
OutputToolchainInfo(TargetDescriptors, Makefiles);
using (Timeline.ScopeEvent("ActionGraph.ExecuteActions()"))
{
ActionGraph.ExecuteActions(BuildConfiguration, MergedActionsToExecute);
}
}
// Run the deployment steps
foreach (TargetMakefile Makefile in Makefiles)
{
if (Makefile.bDeployAfterCompile)
{
TargetReceipt Receipt = TargetReceipt.Read(Makefile.ReceiptFile);
Log.TraceInformation("Deploying {0} {1} {2}...", Receipt.TargetName, Receipt.Platform, Receipt.Configuration);
UEBuildPlatform.GetBuildPlatform(Receipt.Platform).Deploy(Receipt);
}
}
}
}
}
/// <summary>
/// Build a list of targets
/// </summary>
/// <param name="TargetDescriptors">Target descriptors</param>
/// <param name="BuildConfiguration">Current build configuration</param>
/// <param name="WorkingSet">The source file working set</param>
/// <param name="Options">Additional options for the build</param>
/// <param name="LiveCodingManifest">Path to write the live coding manifest to</param>
/// <param name="WriteOutdatedActionsFile">Files to write the list of outdated actions to (rather than building them)</param>
/// <returns>Result from the compilation</returns>
public static void Build(List <TargetDescriptor> TargetDescriptors, BuildConfiguration BuildConfiguration, ISourceFileWorkingSet WorkingSet, BuildOptions Options, FileReference LiveCodingManifest, FileReference WriteOutdatedActionsFile)
{
// Create a makefile for each target
TargetMakefile[] Makefiles = new TargetMakefile[TargetDescriptors.Count];
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
Makefiles[TargetIdx] = CreateMakefile(BuildConfiguration, TargetDescriptors[TargetIdx], WorkingSet);
}
// Output the Live Coding manifest
if (LiveCodingManifest != null)
{
List <Action> AllActions = Makefiles.SelectMany(x => x.Actions).ToList();
HotReload.WriteLiveCodingManifest(LiveCodingManifest, AllActions);
}
// Export the actions for each target
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
TargetDescriptor TargetDescriptor = TargetDescriptors[TargetIdx];
foreach (FileReference WriteActionFile in TargetDescriptor.WriteActionFiles)
{
Log.TraceInformation("Writing actions to {0}", WriteActionFile);
ActionGraph.ExportJson(Makefiles[TargetIdx].Actions, WriteActionFile);
}
}
// Execute the build
if ((Options & BuildOptions.SkipBuild) == 0)
{
// Make sure that none of the actions conflict with any other (producing output files differently, etc...)
ActionGraph.CheckForConflicts(Makefiles.SelectMany(x => x.Actions));
// Find all the actions to be executed
HashSet <Action>[] ActionsToExecute = new HashSet <Action> [TargetDescriptors.Count];
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
ActionsToExecute[TargetIdx] = GetActionsForTarget(BuildConfiguration, TargetDescriptors[TargetIdx], Makefiles[TargetIdx]);
}
// If there are multiple targets being built, merge the actions together
List <Action> MergedActionsToExecute;
if (TargetDescriptors.Count == 1)
{
MergedActionsToExecute = new List <Action>(ActionsToExecute[0]);
}
else
{
MergedActionsToExecute = MergeActionGraphs(TargetDescriptors, ActionsToExecute);
}
// Link all the actions together
ActionGraph.Link(MergedActionsToExecute);
// Make sure the appropriate executor is selected
foreach (TargetDescriptor TargetDescriptor in TargetDescriptors)
{
UEBuildPlatform BuildPlatform = UEBuildPlatform.GetBuildPlatform(TargetDescriptor.Platform);
BuildConfiguration.bAllowXGE &= BuildPlatform.CanUseXGE();
BuildConfiguration.bAllowDistcc &= BuildPlatform.CanUseDistcc();
BuildConfiguration.bAllowSNDBS &= BuildPlatform.CanUseSNDBS();
}
// Delete produced items that are outdated.
ActionGraph.DeleteOutdatedProducedItems(MergedActionsToExecute);
// Save all the action histories now that files have been removed. We have to do this after deleting produced items to ensure that any
// items created during the build don't have the wrong command line.
ActionHistory.SaveAll();
// Create directories for the outdated produced items.
ActionGraph.CreateDirectoriesForProducedItems(MergedActionsToExecute);
// Execute the actions
if ((Options & BuildOptions.XGEExport) != 0)
{
// Just export to an XML file
using (Timeline.ScopeEvent("XGE.ExportActions()"))
{
XGE.ExportActions(MergedActionsToExecute);
}
}
else if (WriteOutdatedActionsFile != null)
{
// Write actions to an output file
using (Timeline.ScopeEvent("ActionGraph.WriteActions"))
{
ActionGraph.ExportJson(MergedActionsToExecute, WriteOutdatedActionsFile);
}
}
else
{
// Execute the actions
if (MergedActionsToExecute.Count == 0)
{
if ((Options & BuildOptions.Quiet) == 0)
{
Log.TraceInformation((TargetDescriptors.Count == 1)? "Target is up to date" : "Targets are up to date");
}
}
else
{
if ((Options & BuildOptions.Quiet) != 0)
{
Log.TraceInformation("Building {0}...", StringUtils.FormatList(TargetDescriptors.Select(x => x.Name).Distinct()));
}
OutputToolchainInfo(TargetDescriptors, Makefiles);
using (Timeline.ScopeEvent("ActionGraph.ExecuteActions()"))
{
ActionGraph.ExecuteActions(BuildConfiguration, MergedActionsToExecute);
}
}
// Run the deployment steps
foreach (TargetMakefile Makefile in Makefiles)
{
if (Makefile.bDeployAfterCompile)
{
TargetReceipt Receipt = TargetReceipt.Read(Makefile.ReceiptFile);
Log.TraceInformation("Deploying {0} {1} {2}...", Receipt.TargetName, Receipt.Platform, Receipt.Configuration);
UEBuildPlatform.GetBuildPlatform(Receipt.Platform).Deploy(Receipt);
}
}
}
}
}
public static ExecutionResult ExecuteActions(List <Action> Actions)
{
ExecutionResult XGEResult = ExecutionResult.TasksSucceeded;
if (Actions.Count > 0)
{
// Write the actions to execute to a XGE task file.
string XGETaskFilePath = Path.Combine(BuildConfiguration.BaseIntermediatePath, "XGETasks.xml");
WriteTaskFile(Actions, XGETaskFilePath, ProgressWriter.bWriteMarkup);
if (BuildConfiguration.bXGEExport)
{
SaveXGEFile(XGETaskFilePath);
}
else
{
// Try to execute the XGE tasks, and if XGE is available, skip the local execution fallback.
if (Telemetry.IsAvailable())
{
try
{
const string BuilderKey = "HKEY_LOCAL_MACHINE\\SOFTWARE\\Wow6432Node\\Xoreax\\IncrediBuild\\Builder";
string CPUUtilization = Registry.GetValue(BuilderKey, "ForceCPUCount", "").ToString();
string AvoidTaskExecutionOnLocalMachine = Registry.GetValue(BuilderKey, "AvoidLocalExec", "").ToString();
string RestartRemoteProcessesOnLocalMachine = Registry.GetValue(BuilderKey, "AllowDoubleTargets", "").ToString();
string LimitMaxNumberOfCores = Registry.GetValue(BuilderKey, "MaxHelpers", "").ToString();
string WriteOutputToDiskInBackground = Registry.GetValue(BuilderKey, "LazyOutputWriter_Beta", "").ToString();
string MaxConcurrentPDBs = Registry.GetValue(BuilderKey, "MaxConcurrentPDBs", "").ToString();
string EnabledAsHelper = Registry.GetValue(BuilderKey, "LastEnabled", "").ToString();
Telemetry.SendEvent("XGESettings.2",
"CPUUtilization", CPUUtilization,
"AvoidTaskExecutionOnLocalMachine", AvoidTaskExecutionOnLocalMachine,
"RestartRemoteProcessesOnLocalMachine", RestartRemoteProcessesOnLocalMachine,
"LimitMaxNumberOfCores", LimitMaxNumberOfCores,
"WriteOutputToDiskInBackground", WriteOutputToDiskInBackground,
"MaxConcurrentPDBs", MaxConcurrentPDBs,
"EnabledAsHelper", EnabledAsHelper);
}
catch
{
}
// Add a custom output handler to determine the build duration for each task and map it back to the action that generated it.
XGEResult = XGE.ExecuteTaskFileWithProgressMarkup(XGETaskFilePath, Actions.Count, (sender, args) =>
{
// sometimes the args comes in as null
var match = XGEDurationRegex.Match(args.Data ?? "");
/*
* // This is considered fragile and risky parsing of undocumented XGE output, so protect this code from taking down UBT
* try
* {
* // Use LINQ to evaluate the terms so we don't end up with a
* // huge nested conditional expression just to verify the input.
* foreach (var PCHEvent in
* // first, convert the input line into an enumerable
* from RegexMatch in new[] { match }
* where RegexMatch.Success
* let Filename = RegexMatch.Groups["Filename"].Value
* // find the mapped action. (On PS4 at least, XGE appears to use the full filenane, so we need to strip it back down.
* let Action = Actions.Find(a => a.StatusDescription == Path.GetFileName(Filename) && a.ActionType == ActionType.Compile)
* // We should ALWAYS find the action, so maybe this should throw if we can't
* where Action != null
* // see if the mapped action produces a PCH file
* // (there is currently no compile environment left around by which to tell absolutely, so we infer by the extension. Ugh).
* let ActionProducedPCH = Action.ProducedItems.Find(fileItem => new[] { ".PCH", ".GCH" }.Contains(Path.GetExtension(fileItem.AbsolutePath).ToUpperInvariant()))
* where ActionProducedPCH != null
* // we found a valid PCH action and output file, so parse the duration and send an event.
* let durationMatchStr = RegexMatch.Groups["Duration"].Value
* // if there's no hour designator, add one so .NET can parse the time.
* let durationStr = durationMatchStr.Count(c => c == ':') == 1 ? "0:" + durationMatchStr : durationMatchStr
* let duration = TimeSpan.Parse(durationStr)
* select new
* {
* // actually use the filename here because it is coerced to be PCH.MODULE.HEADER.cpp.
* // This allows us an easy way to determine shared status (or source module) and the real header source.
* FileName = Filename,//Path.GetFileName(ActionProducedPCH.AbsolutePath),
* // Get the length from the OS as the FileItem.Length is really for when the file is used as an input,
* // so the stored length is absent for a new for or out of date at best.
* GeneratedFileLength = new FileInfo(ActionProducedPCH.AbsolutePath).Length,
* GeneratedFileDuration = duration,
* })
* {
* // If we had a valid match for a PCH item, send an event.
* Telemetry.SendEvent("PCHTime.2",
* "ExecutorType", "XGE",
* "Filename", PCHEvent.FileName,
* "FileSize", PCHEvent.GeneratedFileLength.ToString(),
* "Duration", PCHEvent.GeneratedFileDuration.TotalSeconds.ToString("0.00"));
* }
* }
* catch (Exception ex)
* {
* // report that something went wrong so we can diagnose.
* Telemetry.SendEvent("PCHTimeError.2",
* "OutputLine", args.Data,
* "Exception", ex.ToString());
* }
*/
// XGE outputs the duration info in a format that makes VC think it's an error file/line notation if the full filename is used.
// So munge it a bit so we don't confuse VC.
var updatedData = match.Success ? args.Data.Replace('(', '[').Replace(')', ']') : args.Data;
// forward the output on to the normal handler or the console like normal
if (Actions[0].OutputEventHandler != null)
{
DataReceivedEventArgs EventArgs = ConstructDataReceivedEventArgs(updatedData);
Actions[0].OutputEventHandler(sender, EventArgs);
}
else
{
Console.WriteLine(updatedData);
}
});
}
else
{
XGEResult = XGE.ExecuteTaskFileWithProgressMarkup(XGETaskFilePath, Actions.Count, (Sender, Args) =>
{
if (Actions[0].OutputEventHandler != null)
{
Actions[0].OutputEventHandler(Sender, Args);
}
else
{
Console.WriteLine(Args.Data);
}
});
}
}
}
return(XGEResult);
}
/// <summary>
/// Gets the path for the XGE console executable
/// </summary>
/// <param name="OutXgConsoleExe">On success, receives the path to the XGE console executable</param>
/// <returns>True if the path was found, false otherwise</returns>
public static bool TryGetXgConsoleExecutable(out string OutXgConsoleExe)
{
return(XGE.TryGetXgConsoleExecutable(out OutXgConsoleExe));
}
public static ExecutionResult ExecuteActions(List <Action> Actions)
{
ExecutionResult XGEResult = ExecutionResult.TasksSucceeded;
if (Actions.Count > 0)
{
// Write the actions to execute to a XGE task file.
string XGETaskFilePath = Path.Combine(BuildConfiguration.BaseIntermediatePath, "XGETasks.xml");
WriteTaskFile(Actions, XGETaskFilePath, ProgressWriter.bWriteMarkup);
if (BuildConfiguration.bXGEExport)
{
SaveXGEFile(XGETaskFilePath);
}
else
{
// Try to execute the XGE tasks, and if XGE is available, skip the local execution fallback.
if (Telemetry.IsAvailable())
{
// Add a custom output handler to determine the build duration for each task and map it back to the action that generated it.
XGEResult = XGE.ExecuteTaskFileWithProgressMarkup(XGETaskFilePath, Actions.Count, (sender, args) =>
{
// sometimes the args comes in as null
var match = XGEDurationRegex.Match(args.Data ?? "");
// This is considered fragile and risky parsing of undocumented XGE output, so protect this code from taking down UBT
try
{
// Use LINQ to evaluate the terms so we don't end up with a
// huge nested conditional expression just to verify the input.
foreach (var PCHEvent in
// first, convert the input line into an enumerable
from RegexMatch in new[] { match }
where RegexMatch.Success
let Filename = RegexMatch.Groups["Filename"].Value
// find the mapped action. (On PS4 at least, XGE appears to use the full filenane, so we need to strip it back down.
let Action = Actions.Find(a => a.StatusDescription == Path.GetFileName(Filename) && a.ActionType == ActionType.Compile)
// We should ALWAYS find the action, so maybe this should throw if we can't
where Action != null
// see if the mapped action produces a PCH file
// (there is currently no compile environment left around by which to tell absolutely, so we infer by the extension. Ugh).
let ActionProducedPCH = Action.ProducedItems.Find(fileItem => new[] { ".PCH", ".GCH" }.Contains(Path.GetExtension(fileItem.AbsolutePath).ToUpperInvariant()))
where ActionProducedPCH != null
// we found a valid PCH action and output file, so parse the duration and send an event.
let durationMatchStr = RegexMatch.Groups["Duration"].Value
// if there's no hour designator, add one so .NET can parse the time.
let durationStr = durationMatchStr.Count(c => c == ':') == 1 ? "0:" + durationMatchStr : durationMatchStr
let duration = TimeSpan.Parse(durationStr)
select new
{
// actually use the filename here because it is coerced to be PCH.MODULE.HEADER.cpp.
// This allows us an easy way to determine shared status (or source module) and the real header source.
FileName = Filename, //Path.GetFileName(ActionProducedPCH.AbsolutePath),
// Get the length from the OS as the FileItem.Length is really for when the file is used as an input,
// so the stored length is absent for a new for or out of date at best.
GeneratedFileLength = new FileInfo(ActionProducedPCH.AbsolutePath).Length,
GeneratedFileDuration = duration,
})
{
// If we had a valid match for a PCH item, send an event.
Telemetry.SendEvent("PCHTime.2",
"ExecutorType", "XGE",
"Filename", PCHEvent.FileName,
"FileSize", PCHEvent.GeneratedFileLength.ToString(),
"Duration", PCHEvent.GeneratedFileDuration.TotalSeconds.ToString("0.00"));
}
}
catch (Exception ex)
{
// report that something went wrong so we can diagnose.
Telemetry.SendEvent("PCHTimeError.2",
"OutputLine", args.Data,
"Exception", ex.ToString());
}
// XGE outputs the duration info in a format that makes VC think it's an error file/line notation if the full filename is used.
// So munge it a bit so we don't confuse VC.
var updatedData = match.Success ? args.Data.Replace('(', '[').Replace(')', ']') : args.Data;
// forward the output on to the normal handler or the console like normal
if (Actions[0].OutputEventHandler != null)
{
DataReceivedEventArgs EventArgs = ConstructDataReceivedEventArgs(updatedData);
Actions[0].OutputEventHandler(sender, EventArgs);
}
else
{
Console.WriteLine(updatedData);
}
});
}
else
{
XGEResult = XGE.ExecuteTaskFileWithProgressMarkup(XGETaskFilePath, Actions.Count, (Sender, Args) =>
{
if (Actions[0].OutputEventHandler != null)
{
Actions[0].OutputEventHandler(Sender, Args);
}
else
{
Console.WriteLine(Args.Data);
}
});
}
}
}
return(XGEResult);
}
/// <summary>
/// Build a list of targets
/// </summary>
/// <param name="TargetDescriptors">Target descriptors</param>
/// <param name="BuildConfiguration">Current build configuration</param>
/// <param name="WorkingSet">The source file working set</param>
/// <param name="Options">Additional options for the build</param>
/// <param name="WriteOutdatedActionsFile">Files to write the list of outdated actions to (rather than building them)</param>
/// <returns>Result from the compilation</returns>
public static void Build(List <TargetDescriptor> TargetDescriptors, BuildConfiguration BuildConfiguration, ISourceFileWorkingSet WorkingSet, BuildOptions Options, FileReference WriteOutdatedActionsFile)
{
// Create a makefile for each target
TargetMakefile[] Makefiles = new TargetMakefile[TargetDescriptors.Count];
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
Makefiles[TargetIdx] = CreateMakefile(BuildConfiguration, TargetDescriptors[TargetIdx], WorkingSet);
}
// Export the actions for each target
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
TargetDescriptor TargetDescriptor = TargetDescriptors[TargetIdx];
foreach (FileReference WriteActionFile in TargetDescriptor.WriteActionFiles)
{
Log.TraceInformation("Writing actions to {0}", WriteActionFile);
ActionGraph.ExportJson(Makefiles[TargetIdx].Actions, WriteActionFile);
}
}
// Execute the build
if ((Options & BuildOptions.SkipBuild) == 0)
{
// Make sure that none of the actions conflict with any other (producing output files differently, etc...)
ActionGraph.CheckForConflicts(Makefiles.SelectMany(x => x.Actions));
// Check we don't exceed the nominal max path length
using (Timeline.ScopeEvent("ActionGraph.CheckPathLengths"))
{
ActionGraph.CheckPathLengths(BuildConfiguration, Makefiles.SelectMany(x => x.Actions));
}
// Find all the actions to be executed
HashSet <Action>[] ActionsToExecute = new HashSet <Action> [TargetDescriptors.Count];
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
ActionsToExecute[TargetIdx] = GetActionsForTarget(BuildConfiguration, TargetDescriptors[TargetIdx], Makefiles[TargetIdx]);
}
// If there are multiple targets being built, merge the actions together
List <Action> MergedActionsToExecute;
if (TargetDescriptors.Count == 1)
{
MergedActionsToExecute = new List <Action>(ActionsToExecute[0]);
}
else
{
MergedActionsToExecute = MergeActionGraphs(TargetDescriptors, ActionsToExecute);
}
// Link all the actions together
ActionGraph.Link(MergedActionsToExecute);
// Make sure we're not modifying any engine files
if ((Options & BuildOptions.NoEngineChanges) != 0)
{
List <FileItem> EngineChanges = MergedActionsToExecute.SelectMany(x => x.ProducedItems).Where(x => x.Location.IsUnderDirectory(UnrealBuildTool.EngineDirectory)).Distinct().OrderBy(x => x.FullName).ToList();
if (EngineChanges.Count > 0)
{
StringBuilder Result = new StringBuilder("Building would modify the following engine files:\n");
foreach (FileItem EngineChange in EngineChanges)
{
Result.AppendFormat("\n{0}", EngineChange.FullName);
}
Result.Append("\n\nPlease rebuild from an IDE instead.");
Log.TraceError("{0}", Result.ToString());
throw new CompilationResultException(CompilationResult.FailedDueToEngineChange);
}
}
// Make sure the appropriate executor is selected
foreach (TargetDescriptor TargetDescriptor in TargetDescriptors)
{
UEBuildPlatform BuildPlatform = UEBuildPlatform.GetBuildPlatform(TargetDescriptor.Platform);
BuildConfiguration.bAllowXGE &= BuildPlatform.CanUseXGE();
BuildConfiguration.bAllowDistcc &= BuildPlatform.CanUseDistcc();
BuildConfiguration.bAllowSNDBS &= BuildPlatform.CanUseSNDBS();
}
// Delete produced items that are outdated.
ActionGraph.DeleteOutdatedProducedItems(MergedActionsToExecute);
// Save all the action histories now that files have been removed. We have to do this after deleting produced items to ensure that any
// items created during the build don't have the wrong command line.
ActionHistory.SaveAll();
// Create directories for the outdated produced items.
ActionGraph.CreateDirectoriesForProducedItems(MergedActionsToExecute);
// Execute the actions
if ((Options & BuildOptions.XGEExport) != 0)
{
OutputToolchainInfo(TargetDescriptors, Makefiles);
// Just export to an XML file
using (Timeline.ScopeEvent("XGE.ExportActions()"))
{
XGE.ExportActions(MergedActionsToExecute);
}
}
else if (WriteOutdatedActionsFile != null)
{
OutputToolchainInfo(TargetDescriptors, Makefiles);
// Write actions to an output file
using (Timeline.ScopeEvent("ActionGraph.WriteActions"))
{
ActionGraph.ExportJson(MergedActionsToExecute, WriteOutdatedActionsFile);
}
}
else
{
// Execute the actions
if (MergedActionsToExecute.Count == 0)
{
if (TargetDescriptors.Any(x => !x.bQuiet))
{
Log.TraceInformation((TargetDescriptors.Count == 1)? "Target is up to date" : "Targets are up to date");
}
}
else
{
if (TargetDescriptors.Any(x => !x.bQuiet))
{
Log.TraceInformation("Building {0}...", StringUtils.FormatList(TargetDescriptors.Select(x => x.Name).Distinct()));
}
OutputToolchainInfo(TargetDescriptors, Makefiles);
using (Timeline.ScopeEvent("ActionGraph.ExecuteActions()"))
{
ActionGraph.ExecuteActions(BuildConfiguration, MergedActionsToExecute);
}
}
// Run the deployment steps
foreach (TargetMakefile Makefile in Makefiles)
{
if (Makefile.bDeployAfterCompile)
{
TargetReceipt Receipt = TargetReceipt.Read(Makefile.ReceiptFile);
Log.TraceInformation("Deploying {0} {1} {2}...", Receipt.TargetName, Receipt.Platform, Receipt.Configuration);
UEBuildPlatform.GetBuildPlatform(Receipt.Platform).Deploy(Receipt);
}
}
}
}
}
/// <summary>
/// Check whether the given platform supports XGE
/// </summary>
/// <param name="Platform">Platform to check</param>
/// <returns>True if the platform supports XGE</returns>
public static bool CanUseXGE(UnrealTargetPlatform Platform)
{
return(UEBuildPlatform.IsPlatformAvailable(Platform) && UEBuildPlatform.GetBuildPlatform(Platform).CanUseXGE() && XGE.IsAvailable());
}
/// <summary>
/// Tests whether this executor can be used
/// </summary>
/// <returns>True if the executor may be used</returns>
public static bool IsAvailable()
{
return(XGE.IsAvailable() && ParallelExecutor.IsAvailable());
}