protected override bool WriteProjectFiles(PlatformProjectGeneratorCollection PlatformProjectGenerators)
{
using (ProgressWriter Progress = new ProgressWriter("Writing project files...", true))
{
List <ProjectFile> ProjectsToGenerate = new List <ProjectFile>(GeneratedProjectFiles);
if (ProjectNames.Any())
{
ProjectsToGenerate = ProjectsToGenerate.Where(it =>
ProjectNames.Contains(it.ProjectFilePath.GetFileNameWithoutAnyExtensions())).ToList();
}
int TotalProjectFileCount = ProjectsToGenerate.Count;
HashSet <UnrealTargetPlatform> PlatformsToGenerate = new HashSet <UnrealTargetPlatform>(SupportedPlatforms);
if (Platforms.Any())
{
PlatformsToGenerate.IntersectWith(Platforms);
}
List <UnrealTargetPlatform> FilteredPlatforms = PlatformsToGenerate.Where(it =>
{
if (UEBuildPlatform.IsPlatformAvailable(it))
{
return(true);
}
Log.TraceWarning(
"Platform {0} is not a valid platform to build. Check that the SDK is installed properly",
it);
Log.TraceWarning("Platform will be ignored in project file generation");
return(false);
}).ToList();
HashSet <UnrealTargetConfiguration> ConfigurationsToGenerate = new HashSet <UnrealTargetConfiguration>(SupportedConfigurations);
if (TargetConfigurations.Any())
{
ConfigurationsToGenerate.IntersectWith(TargetConfigurations);
}
for (int ProjectFileIndex = 0; ProjectFileIndex < ProjectsToGenerate.Count; ++ProjectFileIndex)
{
ProjectFile CurProject = ProjectsToGenerate[ProjectFileIndex];
if (!CurProject.WriteProjectFile(FilteredPlatforms, ConfigurationsToGenerate.ToList(),
PlatformProjectGenerators))
{
return(false);
}
Progress.Write(ProjectFileIndex + 1, TotalProjectFileCount);
}
Progress.Write(TotalProjectFileCount, TotalProjectFileCount);
}
return(true);
}
/// <summary>
/// Executes the tasks in the specified file, parsing progress markup as part of the output.
/// </summary>
bool ExecuteTaskFileWithProgressMarkup(string TaskFilePath, int NumActions)
{
using (ProgressWriter Writer = new ProgressWriter("Compiling C++ source files...", false))
{
int NumCompletedActions = 0;
// Create a wrapper delegate that will parse the output actions
DataReceivedEventHandler EventHandlerWrapper = (Sender, Args) =>
{
if (Args.Data != null)
{
string Text = Args.Data;
if (Text.StartsWith(ProgressMarkupPrefix))
{
Writer.Write(++NumCompletedActions, NumActions);
// Strip out anything that is just an XGE timer. Some programs don't output anything except the progress text.
Text = Args.Data.Substring(ProgressMarkupPrefix.Length);
if (Text.StartsWith(" (") && Text.EndsWith(")"))
{
return;
}
}
Log.TraceInformation(Text);
}
};
// Run through the standard XGE executor
return(ExecuteTaskFile(TaskFilePath, EventHandlerWrapper, NumActions));
}
}
protected override bool WriteProjectFiles(PlatformProjectGeneratorCollection PlatformProjectGenerators)
{
using (ProgressWriter Progress = new ProgressWriter("Writing project files...", true))
{
List <ProjectFile> projectsToGenerate = new List <ProjectFile>(GeneratedProjectFiles);
if (ProjectNames.Any())
{
projectsToGenerate = projectsToGenerate.Where(it =>
ProjectNames.Contains(it.ProjectFilePath.GetFileNameWithoutAnyExtensions())).ToList();
}
int TotalProjectFileCount = projectsToGenerate.Count;
HashSet <UnrealTargetPlatform> platformsToGenerate = new HashSet <UnrealTargetPlatform>(SupportedPlatforms);
if (Platforms.Any())
{
platformsToGenerate.IntersectWith(Platforms);
}
HashSet <UnrealTargetConfiguration> configurationsToGenerate = new HashSet <UnrealTargetConfiguration>(SupportedConfigurations);
if (TargetConfigurations.Any())
{
configurationsToGenerate.IntersectWith(TargetConfigurations);
}
for (int ProjectFileIndex = 0; ProjectFileIndex < projectsToGenerate.Count; ++ProjectFileIndex)
{
ProjectFile CurProject = projectsToGenerate[ProjectFileIndex];
if (!CurProject.WriteProjectFile(platformsToGenerate.ToList(), configurationsToGenerate.ToList(),
PlatformProjectGenerators))
{
return(false);
}
Progress.Write(ProjectFileIndex + 1, TotalProjectFileCount);
}
Progress.Write(TotalProjectFileCount, TotalProjectFileCount);
}
return(true);
}
/// <summary>
/// Executes the tasks in the specified file, parsing progress markup as part of the output.
/// </summary>
bool ExecuteTaskFileWithProgressMarkup(string TaskFilePath, int NumActions, DataReceivedEventHandler OutputEventHandler)
{
using (ProgressWriter Writer = new ProgressWriter("Compiling C++ source files...", false))
{
int NumCompletedActions = 0;
// Create a wrapper delegate that will parse the output actions
DataReceivedEventHandler EventHandlerWrapper = (Sender, Args) =>
{
if (Args.Data != null && Args.Data.StartsWith(ProgressMarkupPrefix))
{
Writer.Write(++NumCompletedActions, NumActions);
Args = ConstructDataReceivedEventArgs(Args.Data.Substring(ProgressMarkupPrefix.Length));
}
if (OutputEventHandler != null)
{
OutputEventHandler(Sender, Args);
}
};
// Run through the standard XGE executor
return(ExecuteTaskFile(TaskFilePath, EventHandlerWrapper, NumActions));
}
}
/// <summary>
/// Builds and runs the header tool and touches the header directories.
/// Performs any early outs if headers need no changes, given the UObject modules, tool path, game name, and configuration
/// </summary>
public static bool ExecuteHeaderToolIfNecessary(UEToolChain ToolChain, UEBuildTarget Target, CPPEnvironment GlobalCompileEnvironment, List <UHTModuleInfo> UObjectModules, FileReference ModuleInfoFileName, ref ECompilationResult UHTResult)
{
if (ProgressWriter.bWriteMarkup)
{
Log.WriteLine(LogEventType.Console, "@progress push 5%");
}
using (ProgressWriter Progress = new ProgressWriter("Generating code...", false))
{
// We never want to try to execute the header tool when we're already trying to build it!
var bIsBuildingUHT = Target.GetTargetName().Equals("UnrealHeaderTool", StringComparison.InvariantCultureIgnoreCase);
var RootLocalPath = Path.GetFullPath(ProjectFileGenerator.RootRelativePath);
// check if UHT is out of date
DateTime HeaderToolTimestamp = DateTime.MaxValue;
bool bHaveHeaderTool = !bIsBuildingUHT && GetHeaderToolTimestamp(out HeaderToolTimestamp);
// ensure the headers are up to date
bool bUHTNeedsToRun = (UEBuildConfiguration.bForceHeaderGeneration == true || !bHaveHeaderTool || AreGeneratedCodeFilesOutOfDate(UObjectModules, HeaderToolTimestamp));
if (bUHTNeedsToRun || UnrealBuildTool.IsGatheringBuild)
{
// Since code files are definitely out of date, we'll now finish computing information about the UObject modules for UHT. We
// want to save this work until we know that UHT actually needs to be run to speed up best-case iteration times.
if (UnrealBuildTool.IsGatheringBuild) // In assembler-only mode, PCH info is loaded from our UBTMakefile!
{
foreach (var UHTModuleInfo in UObjectModules)
{
// Only cache the PCH name if we don't already have one. When running in 'gather only' mode, this will have already been cached
if (string.IsNullOrEmpty(UHTModuleInfo.PCH))
{
UHTModuleInfo.PCH = "";
// We need to figure out which PCH header this module is including, so that UHT can inject an include statement for it into any .cpp files it is synthesizing
var DependencyModuleCPP = (UEBuildModuleCPP)Target.GetModuleByName(UHTModuleInfo.ModuleName);
var ModuleCompileEnvironment = DependencyModuleCPP.CreateModuleCompileEnvironment(GlobalCompileEnvironment);
DependencyModuleCPP.CachePCHUsageForModuleSourceFiles(ModuleCompileEnvironment);
if (DependencyModuleCPP.ProcessedDependencies.UniquePCHHeaderFile != null)
{
UHTModuleInfo.PCH = DependencyModuleCPP.ProcessedDependencies.UniquePCHHeaderFile.AbsolutePath;
}
}
}
}
}
// @todo ubtmake: Optimization: Ideally we could avoid having to generate this data in the case where UHT doesn't even need to run! Can't we use the existing copy? (see below use of Manifest)
UHTManifest Manifest = new UHTManifest(Target, RootLocalPath, ToolChain.ConvertPath(RootLocalPath + '\\'), UObjectModules);
if (!bIsBuildingUHT && bUHTNeedsToRun)
{
// Always build UnrealHeaderTool if header regeneration is required, unless we're running within a Rocket ecosystem or hot-reloading
if (UnrealBuildTool.RunningRocket() == false &&
UEBuildConfiguration.bDoNotBuildUHT == false &&
UEBuildConfiguration.bHotReloadFromIDE == false &&
!(bHaveHeaderTool && !UnrealBuildTool.IsGatheringBuild && UnrealBuildTool.IsAssemblingBuild)) // If running in "assembler only" mode, we assume UHT is already up to date for much faster iteration!
{
// If it is out of date or not there it will be built.
// If it is there and up to date, it will add 0.8 seconds to the build time.
Log.TraceInformation("Building UnrealHeaderTool...");
var UBTArguments = new StringBuilder();
UBTArguments.Append("UnrealHeaderTool");
// Which desktop platform do we need to compile UHT for?
UBTArguments.Append(" " + BuildHostPlatform.Current.Platform.ToString());
// NOTE: We force Development configuration for UHT so that it runs quickly, even when compiling debug
UBTArguments.Append(" " + UnrealTargetConfiguration.Development.ToString());
// NOTE: We disable mutex when launching UBT from within UBT to compile UHT
UBTArguments.Append(" -NoMutex");
if (UnrealBuildTool.CommandLineContains("-noxge"))
{
UBTArguments.Append(" -noxge");
}
// Propagate command-line option
if (UnrealBuildTool.CommandLineContains("-2015"))
{
UBTArguments.Append(" -2015");
}
if (UnrealBuildTool.CommandLineContains("-2013"))
{
UBTArguments.Append(" -2013");
}
// Add UHT plugins to UBT command line as external plugins
if (Target.UnrealHeaderToolPlugins != null && Target.UnrealHeaderToolPlugins.Count > 0)
{
foreach (PluginInfo Plugin in Target.UnrealHeaderToolPlugins)
{
UBTArguments.Append(" -PLUGIN \"" + Plugin.File + "\"");
}
}
if (RunExternalExecutable(UnrealBuildTool.GetUBTPath(), UBTArguments.ToString()) != 0)
{
return(false);
}
}
Progress.Write(1, 3);
var ActualTargetName = String.IsNullOrEmpty(Target.GetTargetName()) ? "UE4" : Target.GetTargetName();
Log.TraceInformation("Parsing headers for {0}", ActualTargetName);
string HeaderToolPath = GetHeaderToolPath();
if (!File.Exists(HeaderToolPath))
{
throw new BuildException("Unable to generate headers because UnrealHeaderTool binary was not found ({0}).", Path.GetFullPath(HeaderToolPath));
}
// Disable extensions when serializing to remove the $type fields
Directory.CreateDirectory(ModuleInfoFileName.Directory.FullName);
System.IO.File.WriteAllText(ModuleInfoFileName.FullName, fastJSON.JSON.Instance.ToJSON(Manifest, new fastJSON.JSONParameters {
UseExtensions = false
}));
string CmdLine = (Target.ProjectFile != null) ? "\"" + Target.ProjectFile.FullName + "\"" : Target.GetTargetName();
CmdLine += " \"" + ModuleInfoFileName + "\" -LogCmds=\"loginit warning, logexit warning, logdatabase error\" -Unattended -WarningsAsErrors";
if (UnrealBuildTool.IsEngineInstalled())
{
CmdLine += " -installed";
}
if (UEBuildConfiguration.bFailIfGeneratedCodeChanges)
{
CmdLine += " -FailIfGeneratedCodeChanges";
}
if (!bInvalidateUHTMakefile && BuildConfiguration.bUseUHTMakefiles)
{
CmdLine += " -UseMakefiles";
}
if (!UEBuildConfiguration.bCompileAgainstEngine)
{
CmdLine += " -NoEnginePlugins";
}
Log.TraceInformation(" Running UnrealHeaderTool {0}", CmdLine);
Stopwatch s = new Stopwatch();
s.Start();
UHTResult = (ECompilationResult)RunExternalExecutable(ExternalExecution.GetHeaderToolPath(), CmdLine);
s.Stop();
if (UHTResult != ECompilationResult.Succeeded)
{
// On Linux and Mac, the shell will return 128+signal number exit codes if UHT gets a signal (e.g. crashes or is interrupted)
if ((BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Linux ||
BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Mac) &&
(int)(UHTResult) >= 128
)
{
// SIGINT is 2, so 128 + SIGINT is 130
UHTResult = ((int)(UHTResult) == 130) ? ECompilationResult.Canceled : ECompilationResult.CrashOrAssert;
}
Log.TraceInformation("Error: Failed to generate code for {0} - error code: {2} ({1})", ActualTargetName, (int)UHTResult, UHTResult.ToString());
return(false);
}
Log.TraceInformation("Reflection code generated for {0} in {1} seconds", ActualTargetName, s.Elapsed.TotalSeconds);
if (BuildConfiguration.bPrintPerformanceInfo)
{
Log.TraceInformation("UnrealHeaderTool took {1}", ActualTargetName, (double)s.ElapsedMilliseconds / 1000.0);
}
// Now that UHT has successfully finished generating code, we need to update all cached FileItems in case their last write time has changed.
// Otherwise UBT might not detect changes UHT made.
DateTime StartTime = DateTime.UtcNow;
FileItem.ResetInfos();
double ResetDuration = (DateTime.UtcNow - StartTime).TotalSeconds;
Log.TraceVerbose("FileItem.ResetInfos() duration: {0}s", ResetDuration);
}
else
{
Log.TraceVerbose("Generated code is up to date.");
}
Progress.Write(2, 3);
// There will never be generated code if we're building UHT, so this should never be called.
if (!bIsBuildingUHT)
{
// Allow generated code to be sync'd to remote machines if needed. This needs to be done even if UHT did not run because
// generated headers include other generated headers using absolute paths which in case of building remotely are already
// the remote machine absolute paths. Because of that parsing headers will not result in finding all includes properly.
// @todo ubtmake: Need to figure out what this does in the assembler case, and whether we need to run it
ToolChain.PostCodeGeneration(Manifest);
}
// touch the directories
UpdateDirectoryTimestamps(UObjectModules);
Progress.Write(3, 3);
}
if (ProgressWriter.bWriteMarkup)
{
Log.WriteLine(LogEventType.Console, "@progress pop");
}
return(true);
}
/// <summary>
/// Executes the specified actions locally.
/// </summary>
/// <returns>True if all the tasks successfully executed, or false if any of them failed.</returns>
public override bool ExecuteActions(List <Action> Actions, bool bLogDetailedActionStats)
{
// Time to sleep after each iteration of the loop in order to not busy wait.
const float LoopSleepTime = 0.1f;
// Use WMI to figure out physical cores, excluding hyper threading.
int NumCores = Utils.GetPhysicalProcessorCount();
if (NumCores == -1)
{
NumCores = System.Environment.ProcessorCount;
}
// The number of actions to execute in parallel is trying to keep the CPU busy enough in presence of I/O stalls.
int MaxActionsToExecuteInParallel = 0;
if (NumCores < System.Environment.ProcessorCount && ProcessorCountMultiplier != 1.0)
{
// The CPU has more logical cores than physical ones, aka uses hyper-threading.
// Use multiplier if provided
MaxActionsToExecuteInParallel = (int)(NumCores * ProcessorCountMultiplier);
}
else if (NumCores < System.Environment.ProcessorCount && NumCores > 4)
{
// The CPU has more logical cores than physical ones, aka uses hyper-threading.
// Use average of logical and physical if we have "lots of cores"
MaxActionsToExecuteInParallel = (int)(NumCores + System.Environment.ProcessorCount) / 2;
}
// No hyper-threading. Only kicking off a task per CPU to keep machine responsive.
else
{
MaxActionsToExecuteInParallel = NumCores;
}
if (Utils.IsRunningOnMono)
{
// heuristic: give each action at least 1.5GB of RAM (some clang instances will need more, actually)
long MinMemoryPerActionMB = 3 * 1024 / 2;
long PhysicalRAMAvailableMB = (new PerformanceCounter("Mono Memory", "Total Physical Memory").RawValue) / (1024 * 1024);
int MaxActionsAffordedByMemory = (int)(Math.Max(1, (PhysicalRAMAvailableMB) / MinMemoryPerActionMB));
MaxActionsToExecuteInParallel = Math.Min(MaxActionsToExecuteInParallel, MaxActionsAffordedByMemory);
}
MaxActionsToExecuteInParallel = Math.Max(1, Math.Min(MaxActionsToExecuteInParallel, MaxProcessorCount));
Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel);
Dictionary <Action, ActionThread> ActionThreadDictionary = new Dictionary <Action, ActionThread>();
int JobNumber = 1;
using (ProgressWriter ProgressWriter = new ProgressWriter("Compiling C++ source code...", false))
{
int ProgressValue = 0;
while (true)
{
// Count the number of pending and still executing actions.
int NumUnexecutedActions = 0;
int NumExecutingActions = 0;
foreach (Action Action in Actions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
NumUnexecutedActions++;
}
else if (ActionThread != null)
{
if (ActionThread.bComplete == false)
{
NumUnexecutedActions++;
NumExecutingActions++;
}
}
}
// Update the current progress
int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions;
if (ProgressValue != NewProgressValue)
{
ProgressWriter.Write(ProgressValue, Actions.Count + 1);
ProgressValue = NewProgressValue;
}
// If there aren't any pending actions left, we're done executing.
if (NumUnexecutedActions == 0)
{
break;
}
// If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
// prerequisites.
foreach (Action Action in Actions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel))
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems)
{
if (PrerequisiteItem.ProducingAction != null && Actions.Contains(PrerequisiteItem.ProducingAction))
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue(PrerequisiteItem.ProducingAction, out PrerequisiteProcess);
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
ActionThreadDictionary.Add(Action, null);
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count);
JobNumber++;
try
{
ActionThread.Run();
}
catch (Exception ex)
{
throw new BuildException(ex, "Failed to start thread for action: {0} {1}\r\n{2}", Action.CommandPath, Action.CommandArguments, ex.ToString());
}
ActionThreadDictionary.Add(Action, ActionThread);
NumExecutingActions++;
}
}
}
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime));
}
}
Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "-------- Begin Detailed Action Stats ----------------------------------------------------------");
Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "^Action Type^Duration (seconds)^Tool^Task^Using PCH");
double TotalThreadSeconds = 0;
// Check whether any of the tasks failed and log action stats if wanted.
bool bSuccess = true;
double TotalBuildProjectTime = 0;
double TotalCompileTime = 0;
double TotalCreateAppBundleTime = 0;
double TotalGenerateDebugInfoTime = 0;
double TotalLinkTime = 0;
double TotalOtherActionsTime = 0;
foreach (KeyValuePair <Action, ActionThread> ActionProcess in ActionThreadDictionary)
{
Action Action = ActionProcess.Key;
ActionThread ActionThread = ActionProcess.Value;
// Check for pending actions, preemptive failure
if (ActionThread == null)
{
bSuccess = false;
continue;
}
// Check for executed action but general failure
if (ActionThread.ExitCode != 0)
{
bSuccess = false;
}
// Log CPU time, tool and task.
double ThreadSeconds = Action.Duration.TotalSeconds;
Log.WriteLineIf(bLogDetailedActionStats,
LogEventType.Console,
"^{0}^{1:0.00}^{2}^{3}^{4}",
Action.ActionType.ToString(),
ThreadSeconds,
Path.GetFileName(Action.CommandPath),
Action.StatusDescription,
Action.bIsUsingPCH);
// Update statistics
switch (Action.ActionType)
{
case ActionType.BuildProject:
TotalBuildProjectTime += ThreadSeconds;
break;
case ActionType.Compile:
TotalCompileTime += ThreadSeconds;
break;
case ActionType.CreateAppBundle:
TotalCreateAppBundleTime += ThreadSeconds;
break;
case ActionType.GenerateDebugInfo:
TotalGenerateDebugInfoTime += ThreadSeconds;
break;
case ActionType.Link:
TotalLinkTime += ThreadSeconds;
break;
default:
TotalOtherActionsTime += ThreadSeconds;
break;
}
// Keep track of total thread seconds spent on tasks.
TotalThreadSeconds += ThreadSeconds;
}
Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "-------- End Detailed Actions Stats -----------------------------------------------------------");
// Log total CPU seconds and numbers of processors involved in tasks.
Log.WriteLineIf(bLogDetailedActionStats || UnrealBuildTool.bPrintDebugInfo,
LogEventType.Console, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds);
// Log detailed stats
Log.WriteLineIf(bLogDetailedActionStats || UnrealBuildTool.bPrintDebugInfo,
LogEventType.Console,
"Cumulative action seconds ({0} processors): {1:0.00} building projects, {2:0.00} compiling, {3:0.00} creating app bundles, {4:0.00} generating debug info, {5:0.00} linking, {6:0.00} other",
System.Environment.ProcessorCount,
TotalBuildProjectTime,
TotalCompileTime,
TotalCreateAppBundleTime,
TotalGenerateDebugInfoTime,
TotalLinkTime,
TotalOtherActionsTime
);
return(bSuccess);
}
/// <summary>
/// Executes the specified actions locally.
/// </summary>
/// <returns>True if all the tasks successfully executed, or false if any of them failed.</returns>
public override bool ExecuteActions(List <Action> Actions, bool bLogDetailedActionStats)
{
// Time to sleep after each iteration of the loop in order to not busy wait.
const float LoopSleepTime = 0.1f;
// The number of actions to execute in parallel is trying to keep the CPU busy enough in presence of I/O stalls.
int MaxActionsToExecuteInParallel = GetMaxActionsToExecuteInParallel();
Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel);
Dictionary <Action, ActionThread> ActionThreadDictionary = new Dictionary <Action, ActionThread>();
int JobNumber = 1;
using (ProgressWriter ProgressWriter = new ProgressWriter("Compiling C++ source code...", false))
{
int ProgressValue = 0;
while (true)
{
// Count the number of pending and still executing actions.
int NumUnexecutedActions = 0;
int NumExecutingActions = 0;
foreach (Action Action in Actions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
NumUnexecutedActions++;
}
else if (ActionThread != null)
{
if (ActionThread.bComplete == false)
{
NumUnexecutedActions++;
NumExecutingActions++;
}
}
}
// Update the current progress
int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions;
if (ProgressValue != NewProgressValue)
{
ProgressWriter.Write(ProgressValue, Actions.Count + 1);
ProgressValue = NewProgressValue;
}
// If there aren't any pending actions left, we're done executing.
if (NumUnexecutedActions == 0)
{
break;
}
// If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
// prerequisites.
foreach (Action Action in Actions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel))
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (Action PrerequisiteAction in Action.PrerequisiteActions)
{
if (Actions.Contains(PrerequisiteAction))
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue(PrerequisiteAction, out PrerequisiteProcess);
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
ActionThreadDictionary.Add(Action, null);
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count);
JobNumber++;
try
{
ActionThread.Run();
}
catch (Exception ex)
{
throw new BuildException(ex, "Failed to start thread for action: {0} {1}\r\n{2}", Action.CommandPath, Action.CommandArguments, ex.ToString());
}
ActionThreadDictionary.Add(Action, ActionThread);
NumExecutingActions++;
}
}
}
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime));
}
}
Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "-------- Begin Detailed Action Stats ----------------------------------------------------------");
Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "^Action Type^Duration (seconds)^Tool^Task^Using PCH");
double TotalThreadSeconds = 0;
// Check whether any of the tasks failed and log action stats if wanted.
bool bSuccess = true;
double TotalBuildProjectTime = 0;
double TotalCompileTime = 0;
double TotalCreateAppBundleTime = 0;
double TotalGenerateDebugInfoTime = 0;
double TotalLinkTime = 0;
double TotalOtherActionsTime = 0;
foreach (KeyValuePair <Action, ActionThread> ActionProcess in ActionThreadDictionary)
{
Action Action = ActionProcess.Key;
ActionThread ActionThread = ActionProcess.Value;
// Check for pending actions, preemptive failure
if (ActionThread == null)
{
bSuccess = false;
continue;
}
// Check for executed action but general failure
if (ActionThread.ExitCode != 0)
{
bSuccess = false;
}
// Log CPU time, tool and task.
double ThreadSeconds = Action.Duration.TotalSeconds;
Log.WriteLineIf(bLogDetailedActionStats,
LogEventType.Console,
"^{0}^{1:0.00}^{2}^{3}^{4}",
Action.ActionType.ToString(),
ThreadSeconds,
Action.CommandPath.GetFileName(),
Action.StatusDescription,
Action.bIsUsingPCH);
// Update statistics
switch (Action.ActionType)
{
case ActionType.BuildProject:
TotalBuildProjectTime += ThreadSeconds;
break;
case ActionType.Compile:
TotalCompileTime += ThreadSeconds;
break;
case ActionType.CreateAppBundle:
TotalCreateAppBundleTime += ThreadSeconds;
break;
case ActionType.GenerateDebugInfo:
TotalGenerateDebugInfoTime += ThreadSeconds;
break;
case ActionType.Link:
TotalLinkTime += ThreadSeconds;
break;
default:
TotalOtherActionsTime += ThreadSeconds;
break;
}
// Keep track of total thread seconds spent on tasks.
TotalThreadSeconds += ThreadSeconds;
}
Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "-------- End Detailed Actions Stats -----------------------------------------------------------");
// Log total CPU seconds and numbers of processors involved in tasks.
Log.WriteLineIf(bLogDetailedActionStats,
LogEventType.Console, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds);
// Log detailed stats
Log.WriteLineIf(bLogDetailedActionStats,
LogEventType.Console,
"Cumulative action seconds ({0} processors): {1:0.00} building projects, {2:0.00} compiling, {3:0.00} creating app bundles, {4:0.00} generating debug info, {5:0.00} linking, {6:0.00} other",
System.Environment.ProcessorCount,
TotalBuildProjectTime,
TotalCompileTime,
TotalCreateAppBundleTime,
TotalGenerateDebugInfoTime,
TotalLinkTime,
TotalOtherActionsTime
);
return(bSuccess);
}
public override bool ExecuteActions(List <Action> Actions, bool bLogDetailedActionStats)
{
bool bDistccResult = true;
if (Actions.Count > 0)
{
// Time to sleep after each iteration of the loop in order to not busy wait.
const float LoopSleepTime = 0.1f;
int MaxActionsToExecuteInParallel = 0;
string UserDir = Environment.GetEnvironmentVariable("HOME");
string HostsInfo = UserDir + "/.dmucs/hosts-info";
string NumUBTBuildTasks = Environment.GetEnvironmentVariable("NumUBTBuildTasks");
Int32 MaxUBTBuildTasks = MaxActionsToExecuteInParallel;
if (Int32.TryParse(NumUBTBuildTasks, out MaxUBTBuildTasks))
{
MaxActionsToExecuteInParallel = MaxUBTBuildTasks;
}
else if (BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Mac)
{
using (System.Diagnostics.Process DefaultsProcess = new System.Diagnostics.Process())
{
try
{
DefaultsProcess.StartInfo.FileName = "/usr/bin/defaults";
DefaultsProcess.StartInfo.CreateNoWindow = true;
DefaultsProcess.StartInfo.UseShellExecute = false;
DefaultsProcess.StartInfo.RedirectStandardOutput = true;
DefaultsProcess.StartInfo.Arguments = "read com.apple.dt.Xcode IDEBuildOperationMaxNumberOfConcurrentCompileTasks";
DefaultsProcess.Start();
string Output = DefaultsProcess.StandardOutput.ReadToEnd();
DefaultsProcess.WaitForExit();
if (DefaultsProcess.ExitCode == 0 && Int32.TryParse(Output, out MaxUBTBuildTasks))
{
MaxActionsToExecuteInParallel = MaxUBTBuildTasks;
}
}
catch (Exception)
{
}
}
}
else if (System.IO.File.Exists(HostsInfo))
{
System.IO.StreamReader File = new System.IO.StreamReader(HostsInfo);
string Line = null;
while ((Line = File.ReadLine()) != null)
{
string[] HostInfo = Line.Split(' ');
if (HostInfo.Count() == 3)
{
int NumCPUs = 0;
if (System.Int32.TryParse(HostInfo[1], out NumCPUs))
{
MaxActionsToExecuteInParallel += NumCPUs;
}
}
}
File.Close();
}
else
{
MaxActionsToExecuteInParallel = System.Environment.ProcessorCount;
}
if (bAllowDistccLocalFallback == false)
{
Environment.SetEnvironmentVariable("DISTCC_FALLBACK", "0");
}
if (bVerboseDistccOutput == true)
{
Environment.SetEnvironmentVariable("DISTCC_VERBOSE", "1");
}
FileReference DistccExecutable = new FileReference(DistccExecutablesPath + "/distcc");
FileReference GetHostExecutable = new FileReference(DistccExecutablesPath + "/gethost");
Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel, DistccExecutable, GetHostExecutable);
Dictionary <Action, ActionThread> ActionThreadDictionary = new Dictionary <Action, ActionThread>();
int JobNumber = 1;
using (ProgressWriter ProgressWriter = new ProgressWriter("Compiling source code...", false))
{
int ProgressValue = 0;
while (true)
{
// Count the number of pending and still executing actions.
int NumUnexecutedActions = 0;
int NumExecutingActions = 0;
foreach (Action Action in Actions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
NumUnexecutedActions++;
}
else if (ActionThread != null)
{
if (ActionThread.bComplete == false)
{
NumUnexecutedActions++;
NumExecutingActions++;
}
}
}
// Update the current progress
int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions;
if (ProgressValue != NewProgressValue)
{
ProgressWriter.Write(ProgressValue, Actions.Count + 1);
ProgressValue = NewProgressValue;
}
// If there aren't any pending actions left, we're done executing.
if (NumUnexecutedActions == 0)
{
break;
}
// If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
// prerequisites.
foreach (Action Action in Actions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel))
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (Action PrerequisiteAction in Action.PrerequisiteActions)
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue(PrerequisiteAction, out PrerequisiteProcess);
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
ActionThreadDictionary.Add(Action, null);
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
if (Action.ActionType == ActionType.Compile || Action.ActionType == ActionType.Link)
{
string TypeCommand = String.IsNullOrEmpty(DMUCSDistProp) ? "" : (" --type " + DMUCSDistProp);
string NewCommandArguments = "--server " + DMUCSCoordinator + TypeCommand + " --wait -1 \"" + DistccExecutable + "\" " + Action.CommandPath + " " + Action.CommandArguments;
if (Action.ActionType == ActionType.Compile)
{
// Distcc separates preprocessing and compilation which means we must silence these warnings
NewCommandArguments += " -Wno-constant-logical-operand";
NewCommandArguments += " -Wno-parentheses-equality";
}
Action.CommandPath = GetHostExecutable;
Action.CommandArguments = NewCommandArguments;
}
ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count);
JobNumber++;
ActionThread.Run();
ActionThreadDictionary.Add(Action, ActionThread);
NumExecutingActions++;
}
}
}
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime));
}
}
Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "-------- Begin Detailed Action Stats ----------------------------------------------------------");
Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "^Action Type^Duration (seconds)^Tool^Task^Using PCH");
double TotalThreadSeconds = 0;
// Check whether any of the tasks failed and log action stats if wanted.
foreach (KeyValuePair <Action, ActionThread> ActionProcess in ActionThreadDictionary)
{
Action Action = ActionProcess.Key;
ActionThread ActionThread = ActionProcess.Value;
// Check for pending actions, preemptive failure
if (ActionThread == null)
{
bDistccResult = false;
continue;
}
// Check for executed action but general failure
if (ActionThread.ExitCode != 0)
{
bDistccResult = false;
}
// Log CPU time, tool and task.
double ThreadSeconds = Action.Duration.TotalSeconds;
Log.WriteLineIf(bLogDetailedActionStats,
LogEventType.Console,
"^{0}^{1:0.00}^{2}^{3}^{4}",
Action.ActionType.ToString(),
ThreadSeconds,
Action.CommandPath.GetFileName(),
Action.StatusDescription,
Action.bIsUsingPCH);
// Keep track of total thread seconds spent on tasks.
TotalThreadSeconds += ThreadSeconds;
}
Log.WriteLineIf(bLogDetailedActionStats,
LogEventType.Console,
"-------- End Detailed Actions Stats -----------------------------------------------------------");
// Log total CPU seconds and numbers of processors involved in tasks.
Log.WriteLineIf(bLogDetailedActionStats,
LogEventType.Console, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds);
}
return(bDistccResult);
}
public static bool ExecuteActions(List <Action> Actions)
{
bool bDistccResult = true;
if (Actions.Count > 0)
{
// Time to sleep after each iteration of the loop in order to not busy wait.
const float LoopSleepTime = 0.1f;
int MaxActionsToExecuteInParallel = 0;
string UserDir = Environment.GetEnvironmentVariable("HOME");
string HostsInfo = UserDir + "/.dmucs/hosts-info";
System.IO.StreamReader File = new System.IO.StreamReader(HostsInfo);
string Line = null;
while ((Line = File.ReadLine()) != null)
{
var HostInfo = Line.Split(' ');
if (HostInfo.Count() == 3)
{
int NumCPUs = 0;
if (System.Int32.TryParse(HostInfo [1], out NumCPUs))
{
MaxActionsToExecuteInParallel += NumCPUs;
}
}
}
File.Close();
if (BuildConfiguration.bAllowDistccLocalFallback == false)
{
Environment.SetEnvironmentVariable("DISTCC_FALLBACK", "0");
}
string DistccExecutable = BuildConfiguration.DistccExecutablesPath + "/distcc";
string GetHostExecutable = BuildConfiguration.DistccExecutablesPath + "/gethost";
Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel, DistccExecutable, GetHostExecutable);
Dictionary <Action, ActionThread> ActionThreadDictionary = new Dictionary <Action, ActionThread>();
int JobNumber = 1;
using (ProgressWriter ProgressWriter = new ProgressWriter("Compiling source code...", false))
{
int ProgressValue = 0;
while (true)
{
// Count the number of pending and still executing actions.
int NumUnexecutedActions = 0;
int NumExecutingActions = 0;
foreach (Action Action in Actions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
NumUnexecutedActions++;
}
else if (ActionThread != null)
{
if (ActionThread.bComplete == false)
{
NumUnexecutedActions++;
NumExecutingActions++;
}
}
}
// Update the current progress
int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions;
if (ProgressValue != NewProgressValue)
{
ProgressWriter.Write(ProgressValue, Actions.Count + 1);
ProgressValue = NewProgressValue;
}
// If there aren't any pending actions left, we're done executing.
if (NumUnexecutedActions == 0)
{
break;
}
// If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
// prerequisites.
foreach (Action Action in Actions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel))
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems)
{
if (PrerequisiteItem.ProducingAction != null && Actions.Contains(PrerequisiteItem.ProducingAction))
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue(PrerequisiteItem.ProducingAction, out PrerequisiteProcess);
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
ActionThreadDictionary.Add(Action, null);
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
if ((Action.ActionType == ActionType.Compile || Action.ActionType == ActionType.Link) && DistccExecutable != null && GetHostExecutable != null)
{
string NewCommandArguments = "--wait -1 \"" + DistccExecutable + "\" " + Action.CommandPath + " " + Action.CommandArguments;
Action.CommandPath = GetHostExecutable;
Action.CommandArguments = NewCommandArguments;
}
ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count);
JobNumber++;
ActionThread.Run();
ActionThreadDictionary.Add(Action, ActionThread);
NumExecutingActions++;
}
}
}
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime));
}
}
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "-------- Begin Detailed Action Stats ----------------------------------------------------------");
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "^Action Type^Duration (seconds)^Tool^Task^Using PCH");
double TotalThreadSeconds = 0;
// Check whether any of the tasks failed and log action stats if wanted.
foreach (KeyValuePair <Action, ActionThread> ActionProcess in ActionThreadDictionary)
{
Action Action = ActionProcess.Key;
ActionThread ActionThread = ActionProcess.Value;
// Check for pending actions, preemptive failure
if (ActionThread == null)
{
bDistccResult = false;
continue;
}
// Check for executed action but general failure
if (ActionThread.ExitCode != 0)
{
bDistccResult = false;
}
// Log CPU time, tool and task.
double ThreadSeconds = Action.Duration.TotalSeconds;
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats,
TraceEventType.Information,
"^{0}^{1:0.00}^{2}^{3}^{4}",
Action.ActionType.ToString(),
ThreadSeconds,
Path.GetFileName(Action.CommandPath),
Action.StatusDescription,
Action.bIsUsingPCH);
// Update statistics
switch (Action.ActionType)
{
case ActionType.BuildProject:
UnrealBuildTool.TotalBuildProjectTime += ThreadSeconds;
break;
case ActionType.Compile:
UnrealBuildTool.TotalCompileTime += ThreadSeconds;
break;
case ActionType.CreateAppBundle:
UnrealBuildTool.TotalCreateAppBundleTime += ThreadSeconds;
break;
case ActionType.GenerateDebugInfo:
UnrealBuildTool.TotalGenerateDebugInfoTime += ThreadSeconds;
break;
case ActionType.Link:
UnrealBuildTool.TotalLinkTime += ThreadSeconds;
break;
default:
UnrealBuildTool.TotalOtherActionsTime += ThreadSeconds;
break;
}
// Keep track of total thread seconds spent on tasks.
TotalThreadSeconds += ThreadSeconds;
}
Log.TraceInformation("-------- End Detailed Actions Stats -----------------------------------------------------------");
// Log total CPU seconds and numbers of processors involved in tasks.
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats || BuildConfiguration.bPrintDebugInfo,
TraceEventType.Information, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds);
}
return(bDistccResult);
}
/// <summary>
/// Executes the specified actions locally.
/// </summary>
/// <returns>True if all the tasks successfully executed, or false if any of them failed.</returns>
public override bool ExecuteActions(List <Action> InputActions, bool bLogDetailedActionStats)
{
// Figure out how many processors to use
int MaxProcesses = Math.Min((int)(Environment.ProcessorCount * ProcessorCountMultiplier), MaxProcessorCount);
Log.TraceInformation("Building {0} {1} with {2} {3}...", InputActions.Count, (InputActions.Count == 1) ? "action" : "actions", MaxProcesses, (MaxProcesses == 1)? "process" : "processes");
// Create actions with all our internal metadata
List <BuildAction> Actions = new List <BuildAction>();
for (int Idx = 0; Idx < InputActions.Count; Idx++)
{
BuildAction Action = new BuildAction();
Action.SortIndex = Idx;
Action.Inner = InputActions[Idx];
Actions.Add(Action);
}
// Build a map of items to their producing actions
Dictionary <FileItem, BuildAction> FileToProducingAction = new Dictionary <FileItem, BuildAction>();
foreach (BuildAction Action in Actions)
{
foreach (FileItem ProducedItem in Action.Inner.ProducedItems)
{
FileToProducingAction[ProducedItem] = Action;
}
}
// Update all the actions with all their dependencies
foreach (BuildAction Action in Actions)
{
foreach (FileItem PrerequisiteItem in Action.Inner.PrerequisiteItems)
{
BuildAction Dependency;
if (FileToProducingAction.TryGetValue(PrerequisiteItem, out Dependency))
{
Action.Dependencies.Add(Dependency);
Dependency.Dependants.Add(Action);
}
}
}
// Figure out the recursive dependency count
HashSet <BuildAction> VisitedActions = new HashSet <BuildAction>();
foreach (BuildAction Action in Actions)
{
Action.MissingDependencyCount = Action.Dependencies.Count;
RecursiveIncDependents(Action, VisitedActions);
}
// Create the list of things to process
List <BuildAction> QueuedActions = new List <BuildAction>();
foreach (BuildAction Action in Actions)
{
if (Action.MissingDependencyCount == 0)
{
QueuedActions.Add(Action);
}
}
// Execute the actions
using (ScopedLogIndent Indent = new ScopedLogIndent(" "))
{
// Create a job object for all the child processes
bool bResult = true;
Dictionary <BuildAction, Thread> ExecutingActions = new Dictionary <BuildAction, Thread>();
List <BuildAction> CompletedActions = new List <BuildAction>();
using (ManagedProcessGroup ProcessGroup = new ManagedProcessGroup())
{
using (AutoResetEvent CompletedEvent = new AutoResetEvent(false))
{
int NumCompletedActions = 0;
using (ProgressWriter ProgressWriter = new ProgressWriter("Compiling C++ source code...", false))
{
while (QueuedActions.Count > 0 || ExecutingActions.Count > 0)
{
// Sort the actions by the number of things dependent on them
QueuedActions.Sort((A, B) => (A.TotalDependantCount == B.TotalDependantCount)? (B.SortIndex - A.SortIndex) : (B.TotalDependantCount - A.TotalDependantCount));
// Create threads up to the maximum number of actions
while (ExecutingActions.Count < MaxProcesses && QueuedActions.Count > 0)
{
BuildAction Action = QueuedActions[QueuedActions.Count - 1];
QueuedActions.RemoveAt(QueuedActions.Count - 1);
Thread ExecutingThread = new Thread(() => { ExecuteAction(ProcessGroup, Action, CompletedActions, CompletedEvent); });
ExecutingThread.Name = String.Format("Build:{0}", Action.Inner.StatusDescription);
ExecutingThread.Start();
ExecutingActions.Add(Action, ExecutingThread);
}
// Wait for something to finish
CompletedEvent.WaitOne();
// Wait for something to finish and flush it to the log
lock (CompletedActions)
{
foreach (BuildAction CompletedAction in CompletedActions)
{
// Join the thread
Thread CompletedThread = ExecutingActions[CompletedAction];
CompletedThread.Join();
ExecutingActions.Remove(CompletedAction);
// Update the progress
NumCompletedActions++;
ProgressWriter.Write(NumCompletedActions, InputActions.Count);
// Write it to the log
if (CompletedAction.LogLines.Count > 0)
{
Log.TraceInformation("[{0}/{1}] {2}", NumCompletedActions, InputActions.Count, CompletedAction.LogLines[0]);
for (int LineIdx = 1; LineIdx < CompletedAction.LogLines.Count; LineIdx++)
{
Log.TraceInformation("{0}", CompletedAction.LogLines[LineIdx]);
}
}
// Check the exit code
if (CompletedAction.ExitCode == 0)
{
// Mark all the dependents as done
foreach (BuildAction DependantAction in CompletedAction.Dependants)
{
if (--DependantAction.MissingDependencyCount == 0)
{
QueuedActions.Add(DependantAction);
}
}
}
else
{
// Update the exit code if it's not already set
if (bResult && CompletedAction.ExitCode != 0)
{
bResult = false;
}
}
}
CompletedActions.Clear();
}
// If we've already got a non-zero exit code, clear out the list of queued actions so nothing else will run
if (!bResult && bStopCompilationAfterErrors)
{
QueuedActions.Clear();
}
}
}
}
}
return(bResult);
}
}
/**
* Executes the specified actions locally.
* @return True if all the tasks successfully executed, or false if any of them failed.
*/
public static bool ExecuteActions(List <Action> Actions)
{
// Time to sleep after each iteration of the loop in order to not busy wait.
const float LoopSleepTime = 0.1f;
// Use WMI to figure out physical cores, excluding hyper threading.
int NumCores = 0;
if (!Utils.IsRunningOnMono)
{
try
{
using (var Mos = new System.Management.ManagementObjectSearcher("Select * from Win32_Processor"))
{
var MosCollection = Mos.Get();
foreach (var Item in MosCollection)
{
NumCores += int.Parse(Item["NumberOfCores"].ToString());
}
}
}
catch (Exception Ex)
{
Log.TraceWarning("Unable to get the number of Cores: {0}", Ex.ToString());
Log.TraceWarning("Falling back to processor count.");
}
}
// On some systems this requires a hot fix to work so we fall back to using the (logical) processor count.
if (NumCores == 0)
{
NumCores = System.Environment.ProcessorCount;
}
// The number of actions to execute in parallel is trying to keep the CPU busy enough in presence of I/O stalls.
int MaxActionsToExecuteInParallel = 0;
// The CPU has more logical cores than physical ones, aka uses hyper-threading.
if (NumCores < System.Environment.ProcessorCount)
{
MaxActionsToExecuteInParallel = (int)(NumCores * BuildConfiguration.ProcessorCountMultiplier);
}
// No hyper-threading. Only kicking off a task per CPU to keep machine responsive.
else
{
MaxActionsToExecuteInParallel = NumCores;
}
if (Utils.IsRunningOnMono)
{
// heuristic: give each action at least 1.5GB of RAM (some clang instances will need more, actually)
long MinMemoryPerActionMB = 3 * 1024 / 2;
long PhysicalRAMAvailableMB = (new PerformanceCounter("Mono Memory", "Total Physical Memory").RawValue) / (1024 * 1024);
int MaxActionsAffordedByMemory = (int)(Math.Max(1, (PhysicalRAMAvailableMB) / MinMemoryPerActionMB));
MaxActionsToExecuteInParallel = Math.Min(MaxActionsToExecuteInParallel, MaxActionsAffordedByMemory);
}
MaxActionsToExecuteInParallel = Math.Min(MaxActionsToExecuteInParallel, BuildConfiguration.MaxProcessorCount);
Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel);
Dictionary <Action, ActionThread> ActionThreadDictionary = new Dictionary <Action, ActionThread>();
int JobNumber = 1;
using (ProgressWriter ProgressWriter = new ProgressWriter("Compiling source code...", false))
{
int ProgressValue = 0;
while (true)
{
// Count the number of pending and still executing actions.
int NumUnexecutedActions = 0;
int NumExecutingActions = 0;
foreach (Action Action in Actions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
NumUnexecutedActions++;
}
else if (ActionThread != null)
{
if (ActionThread.bComplete == false)
{
NumUnexecutedActions++;
NumExecutingActions++;
}
}
}
// Update the current progress
int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions;
if (ProgressValue != NewProgressValue)
{
ProgressWriter.Write(ProgressValue, Actions.Count + 1);
ProgressValue = NewProgressValue;
}
// If there aren't any pending actions left, we're done executing.
if (NumUnexecutedActions == 0)
{
break;
}
// If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
// prerequisites.
foreach (Action Action in Actions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel))
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems)
{
if (PrerequisiteItem.ProducingAction != null && Actions.Contains(PrerequisiteItem.ProducingAction))
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue(PrerequisiteItem.ProducingAction, out PrerequisiteProcess);
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
ActionThreadDictionary.Add(Action, null);
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count);
JobNumber++;
ActionThread.Run();
ActionThreadDictionary.Add(Action, ActionThread);
NumExecutingActions++;
}
}
}
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime));
}
}
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "-------- Begin Detailed Action Stats ----------------------------------------------------------");
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "^Action Type^Duration (seconds)^Tool^Task^Using PCH");
double TotalThreadSeconds = 0;
// Check whether any of the tasks failed and log action stats if wanted.
bool bSuccess = true;
foreach (KeyValuePair <Action, ActionThread> ActionProcess in ActionThreadDictionary)
{
Action Action = ActionProcess.Key;
ActionThread ActionThread = ActionProcess.Value;
// Check for pending actions, preemptive failure
if (ActionThread == null)
{
bSuccess = false;
continue;
}
// Check for executed action but general failure
if (ActionThread.ExitCode != 0)
{
bSuccess = false;
}
// Log CPU time, tool and task.
double ThreadSeconds = Action.Duration.TotalSeconds;
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats,
TraceEventType.Information,
"^{0}^{1:0.00}^{2}^{3}^{4}",
Action.ActionType.ToString(),
ThreadSeconds,
Path.GetFileName(Action.CommandPath),
Action.StatusDescription,
Action.bIsUsingPCH);
// Update statistics
switch (Action.ActionType)
{
case ActionType.BuildProject:
UnrealBuildTool.TotalBuildProjectTime += ThreadSeconds;
break;
case ActionType.Compile:
UnrealBuildTool.TotalCompileTime += ThreadSeconds;
break;
case ActionType.CreateAppBundle:
UnrealBuildTool.TotalCreateAppBundleTime += ThreadSeconds;
break;
case ActionType.GenerateDebugInfo:
UnrealBuildTool.TotalGenerateDebugInfoTime += ThreadSeconds;
break;
case ActionType.Link:
UnrealBuildTool.TotalLinkTime += ThreadSeconds;
break;
default:
UnrealBuildTool.TotalOtherActionsTime += ThreadSeconds;
break;
}
// Keep track of total thread seconds spent on tasks.
TotalThreadSeconds += ThreadSeconds;
}
Log.TraceInformation("-------- End Detailed Actions Stats -----------------------------------------------------------");
// Log total CPU seconds and numbers of processors involved in tasks.
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats || BuildConfiguration.bPrintDebugInfo,
TraceEventType.Information, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds);
return(bSuccess);
}
