internal static ExecutionResult ExecuteLocalActions(List<Action> InLocalActions, Dictionary<Action, ActionThread> InActionThreadDictionary, int TotalNumJobs)
{
// Time to sleep after each iteration of the loop in order to not busy wait.
const float LoopSleepTime = 0.1f;
ExecutionResult LocalActionsResult = ExecutionResult.TasksSucceeded;
while (true)
{
// Count the number of pending and still executing actions.
int NumUnexecutedActions = 0;
int NumExecutingActions = 0;
foreach (Action Action in InLocalActions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = InActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
NumUnexecutedActions++;
}
else if (ActionThread != null)
{
if (ActionThread.bComplete == false)
{
NumUnexecutedActions++;
NumExecutingActions++;
}
}
}
// If there aren't any pending actions left, we're done executing.
if (NumUnexecutedActions == 0)
{
break;
}
// If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
// prerequisites.
foreach (Action Action in InLocalActions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = InActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel))
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems)
{
if (PrerequisiteItem.ProducingAction != null && InLocalActions.Contains(PrerequisiteItem.ProducingAction))
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = InActionThreadDictionary.TryGetValue(PrerequisiteItem.ProducingAction, out PrerequisiteProcess);
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
InActionThreadDictionary.Add(Action, null);
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
ActionThread ActionThread = new ActionThread(Action, JobNumber, TotalNumJobs);
ActionThread.Run();
InActionThreadDictionary.Add(Action, ActionThread);
NumExecutingActions++;
JobNumber++;
}
}
}
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime));
}
return LocalActionsResult;
}
/// <summary>
/// Executes the specified actions locally.
/// </summary>
/// <returns>True if all the tasks successfully executed, or false if any of them failed.</returns>
public override bool ExecuteActions(List <Action> Actions, bool bLogDetailedActionStats)
{
// Time to sleep after each iteration of the loop in order to not busy wait.
const float LoopSleepTime = 0.1f;
// Use WMI to figure out physical cores, excluding hyper threading.
int NumCores = Utils.GetPhysicalProcessorCount();
if (NumCores == -1)
{
NumCores = System.Environment.ProcessorCount;
}
// The number of actions to execute in parallel is trying to keep the CPU busy enough in presence of I/O stalls.
int MaxActionsToExecuteInParallel = 0;
if (NumCores < System.Environment.ProcessorCount && ProcessorCountMultiplier != 1.0)
{
// The CPU has more logical cores than physical ones, aka uses hyper-threading.
// Use multiplier if provided
MaxActionsToExecuteInParallel = (int)(NumCores * ProcessorCountMultiplier);
}
else if (NumCores < System.Environment.ProcessorCount && NumCores > 4)
{
// The CPU has more logical cores than physical ones, aka uses hyper-threading.
// Use average of logical and physical if we have "lots of cores"
MaxActionsToExecuteInParallel = (int)(NumCores + System.Environment.ProcessorCount) / 2;
}
// No hyper-threading. Only kicking off a task per CPU to keep machine responsive.
else
{
MaxActionsToExecuteInParallel = NumCores;
}
if (Utils.IsRunningOnMono)
{
// heuristic: give each action at least 1.5GB of RAM (some clang instances will need more, actually)
long MinMemoryPerActionMB = 3 * 1024 / 2;
long PhysicalRAMAvailableMB = (new PerformanceCounter("Mono Memory", "Total Physical Memory").RawValue) / (1024 * 1024);
int MaxActionsAffordedByMemory = (int)(Math.Max(1, (PhysicalRAMAvailableMB) / MinMemoryPerActionMB));
MaxActionsToExecuteInParallel = Math.Min(MaxActionsToExecuteInParallel, MaxActionsAffordedByMemory);
}
MaxActionsToExecuteInParallel = Math.Max(1, Math.Min(MaxActionsToExecuteInParallel, MaxProcessorCount));
Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel);
Dictionary <Action, ActionThread> ActionThreadDictionary = new Dictionary <Action, ActionThread>();
int JobNumber = 1;
using (ProgressWriter ProgressWriter = new ProgressWriter("Compiling C++ source code...", false))
{
int ProgressValue = 0;
while (true)
{
// Count the number of pending and still executing actions.
int NumUnexecutedActions = 0;
int NumExecutingActions = 0;
foreach (Action Action in Actions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
NumUnexecutedActions++;
}
else if (ActionThread != null)
{
if (ActionThread.bComplete == false)
{
NumUnexecutedActions++;
NumExecutingActions++;
}
}
}
// Update the current progress
int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions;
if (ProgressValue != NewProgressValue)
{
ProgressWriter.Write(ProgressValue, Actions.Count + 1);
ProgressValue = NewProgressValue;
}
// If there aren't any pending actions left, we're done executing.
if (NumUnexecutedActions == 0)
{
break;
}
// If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
// prerequisites.
foreach (Action Action in Actions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel))
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems)
{
if (PrerequisiteItem.ProducingAction != null && Actions.Contains(PrerequisiteItem.ProducingAction))
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue(PrerequisiteItem.ProducingAction, out PrerequisiteProcess);
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
ActionThreadDictionary.Add(Action, null);
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count);
JobNumber++;
try
{
ActionThread.Run();
}
catch (Exception ex)
{
throw new BuildException(ex, "Failed to start thread for action: {0} {1}\r\n{2}", Action.CommandPath, Action.CommandArguments, ex.ToString());
}
ActionThreadDictionary.Add(Action, ActionThread);
NumExecutingActions++;
}
}
}
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime));
}
}
Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "-------- Begin Detailed Action Stats ----------------------------------------------------------");
Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "^Action Type^Duration (seconds)^Tool^Task^Using PCH");
double TotalThreadSeconds = 0;
// Check whether any of the tasks failed and log action stats if wanted.
bool bSuccess = true;
double TotalBuildProjectTime = 0;
double TotalCompileTime = 0;
double TotalCreateAppBundleTime = 0;
double TotalGenerateDebugInfoTime = 0;
double TotalLinkTime = 0;
double TotalOtherActionsTime = 0;
foreach (KeyValuePair <Action, ActionThread> ActionProcess in ActionThreadDictionary)
{
Action Action = ActionProcess.Key;
ActionThread ActionThread = ActionProcess.Value;
// Check for pending actions, preemptive failure
if (ActionThread == null)
{
bSuccess = false;
continue;
}
// Check for executed action but general failure
if (ActionThread.ExitCode != 0)
{
bSuccess = false;
}
// Log CPU time, tool and task.
double ThreadSeconds = Action.Duration.TotalSeconds;
Log.WriteLineIf(bLogDetailedActionStats,
LogEventType.Console,
"^{0}^{1:0.00}^{2}^{3}^{4}",
Action.ActionType.ToString(),
ThreadSeconds,
Path.GetFileName(Action.CommandPath),
Action.StatusDescription,
Action.bIsUsingPCH);
// Update statistics
switch (Action.ActionType)
{
case ActionType.BuildProject:
TotalBuildProjectTime += ThreadSeconds;
break;
case ActionType.Compile:
TotalCompileTime += ThreadSeconds;
break;
case ActionType.CreateAppBundle:
TotalCreateAppBundleTime += ThreadSeconds;
break;
case ActionType.GenerateDebugInfo:
TotalGenerateDebugInfoTime += ThreadSeconds;
break;
case ActionType.Link:
TotalLinkTime += ThreadSeconds;
break;
default:
TotalOtherActionsTime += ThreadSeconds;
break;
}
// Keep track of total thread seconds spent on tasks.
TotalThreadSeconds += ThreadSeconds;
}
Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "-------- End Detailed Actions Stats -----------------------------------------------------------");
// Log total CPU seconds and numbers of processors involved in tasks.
Log.WriteLineIf(bLogDetailedActionStats || UnrealBuildTool.bPrintDebugInfo,
LogEventType.Console, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds);
// Log detailed stats
Log.WriteLineIf(bLogDetailedActionStats || UnrealBuildTool.bPrintDebugInfo,
LogEventType.Console,
"Cumulative action seconds ({0} processors): {1:0.00} building projects, {2:0.00} compiling, {3:0.00} creating app bundles, {4:0.00} generating debug info, {5:0.00} linking, {6:0.00} other",
System.Environment.ProcessorCount,
TotalBuildProjectTime,
TotalCompileTime,
TotalCreateAppBundleTime,
TotalGenerateDebugInfoTime,
TotalLinkTime,
TotalOtherActionsTime
);
return(bSuccess);
}
internal bool ExecuteLocalActions(List <Action> InLocalActions, Dictionary <Action, ActionThread> InActionThreadDictionary, int TotalNumJobs)
{
// Time to sleep after each iteration of the loop in order to not busy wait.
const float LoopSleepTime = 0.1f;
bool LocalActionsResult = true;
while (true)
{
// Count the number of pending and still executing actions.
int NumUnexecutedActions = 0;
int NumExecutingActions = 0;
foreach (Action Action in InLocalActions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = InActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
NumUnexecutedActions++;
}
else if (ActionThread != null)
{
if (ActionThread.bComplete == false)
{
NumUnexecutedActions++;
NumExecutingActions++;
}
}
}
// If there aren't any pending actions left, we're done executing.
if (NumUnexecutedActions == 0)
{
break;
}
// If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
// prerequisites.
foreach (Action Action in InLocalActions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = InActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel))
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (Action PrerequisiteAction in Action.PrerequisiteActions)
{
if (InLocalActions.Contains(PrerequisiteAction))
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = InActionThreadDictionary.TryGetValue(PrerequisiteAction, out PrerequisiteProcess);
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
InActionThreadDictionary.Add(Action, null);
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
ActionThread ActionThread = new ActionThread(Action, JobNumber, TotalNumJobs);
ActionThread.Run();
InActionThreadDictionary.Add(Action, ActionThread);
NumExecutingActions++;
JobNumber++;
}
}
}
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime));
}
return(LocalActionsResult);
}
public override bool ExecuteActions(List <Action> Actions, bool bLogDetailedActionStats)
{
bool bDistccResult = true;
if (Actions.Count > 0)
{
// Time to sleep after each iteration of the loop in order to not busy wait.
const float LoopSleepTime = 0.1f;
int MaxActionsToExecuteInParallel = 0;
string UserDir = Environment.GetEnvironmentVariable("HOME");
string HostsInfo = UserDir + "/.dmucs/hosts-info";
string NumUBTBuildTasks = Environment.GetEnvironmentVariable("NumUBTBuildTasks");
Int32 MaxUBTBuildTasks = MaxActionsToExecuteInParallel;
if (Int32.TryParse(NumUBTBuildTasks, out MaxUBTBuildTasks))
{
MaxActionsToExecuteInParallel = MaxUBTBuildTasks;
}
else if (BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Mac)
{
using (System.Diagnostics.Process DefaultsProcess = new System.Diagnostics.Process())
{
try
{
DefaultsProcess.StartInfo.FileName = "/usr/bin/defaults";
DefaultsProcess.StartInfo.CreateNoWindow = true;
DefaultsProcess.StartInfo.UseShellExecute = false;
DefaultsProcess.StartInfo.RedirectStandardOutput = true;
DefaultsProcess.StartInfo.Arguments = "read com.apple.dt.Xcode IDEBuildOperationMaxNumberOfConcurrentCompileTasks";
DefaultsProcess.Start();
string Output = DefaultsProcess.StandardOutput.ReadToEnd();
DefaultsProcess.WaitForExit();
if (DefaultsProcess.ExitCode == 0 && Int32.TryParse(Output, out MaxUBTBuildTasks))
{
MaxActionsToExecuteInParallel = MaxUBTBuildTasks;
}
}
catch (Exception)
{
}
}
}
else if (System.IO.File.Exists(HostsInfo))
{
System.IO.StreamReader File = new System.IO.StreamReader(HostsInfo);
string Line = null;
while ((Line = File.ReadLine()) != null)
{
string[] HostInfo = Line.Split(' ');
if (HostInfo.Count() == 3)
{
int NumCPUs = 0;
if (System.Int32.TryParse(HostInfo[1], out NumCPUs))
{
MaxActionsToExecuteInParallel += NumCPUs;
}
}
}
File.Close();
}
else
{
MaxActionsToExecuteInParallel = System.Environment.ProcessorCount;
}
if (bAllowDistccLocalFallback == false)
{
Environment.SetEnvironmentVariable("DISTCC_FALLBACK", "0");
}
if (bVerboseDistccOutput == true)
{
Environment.SetEnvironmentVariable("DISTCC_VERBOSE", "1");
}
FileReference DistccExecutable = new FileReference(DistccExecutablesPath + "/distcc");
FileReference GetHostExecutable = new FileReference(DistccExecutablesPath + "/gethost");
Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel, DistccExecutable, GetHostExecutable);
Dictionary <Action, ActionThread> ActionThreadDictionary = new Dictionary <Action, ActionThread>();
int JobNumber = 1;
using (ProgressWriter ProgressWriter = new ProgressWriter("Compiling source code...", false))
{
int ProgressValue = 0;
while (true)
{
// Count the number of pending and still executing actions.
int NumUnexecutedActions = 0;
int NumExecutingActions = 0;
foreach (Action Action in Actions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
NumUnexecutedActions++;
}
else if (ActionThread != null)
{
if (ActionThread.bComplete == false)
{
NumUnexecutedActions++;
NumExecutingActions++;
}
}
}
// Update the current progress
int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions;
if (ProgressValue != NewProgressValue)
{
ProgressWriter.Write(ProgressValue, Actions.Count + 1);
ProgressValue = NewProgressValue;
}
// If there aren't any pending actions left, we're done executing.
if (NumUnexecutedActions == 0)
{
break;
}
// If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
// prerequisites.
foreach (Action Action in Actions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel))
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (Action PrerequisiteAction in Action.PrerequisiteActions)
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue(PrerequisiteAction, out PrerequisiteProcess);
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
ActionThreadDictionary.Add(Action, null);
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
if (Action.ActionType == ActionType.Compile || Action.ActionType == ActionType.Link)
{
string TypeCommand = String.IsNullOrEmpty(DMUCSDistProp) ? "" : (" --type " + DMUCSDistProp);
string NewCommandArguments = "--server " + DMUCSCoordinator + TypeCommand + " --wait -1 \"" + DistccExecutable + "\" " + Action.CommandPath + " " + Action.CommandArguments;
if (Action.ActionType == ActionType.Compile)
{
// Distcc separates preprocessing and compilation which means we must silence these warnings
NewCommandArguments += " -Wno-constant-logical-operand";
NewCommandArguments += " -Wno-parentheses-equality";
}
Action.CommandPath = GetHostExecutable;
Action.CommandArguments = NewCommandArguments;
}
ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count);
JobNumber++;
ActionThread.Run();
ActionThreadDictionary.Add(Action, ActionThread);
NumExecutingActions++;
}
}
}
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime));
}
}
Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "-------- Begin Detailed Action Stats ----------------------------------------------------------");
Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "^Action Type^Duration (seconds)^Tool^Task^Using PCH");
double TotalThreadSeconds = 0;
// Check whether any of the tasks failed and log action stats if wanted.
foreach (KeyValuePair <Action, ActionThread> ActionProcess in ActionThreadDictionary)
{
Action Action = ActionProcess.Key;
ActionThread ActionThread = ActionProcess.Value;
// Check for pending actions, preemptive failure
if (ActionThread == null)
{
bDistccResult = false;
continue;
}
// Check for executed action but general failure
if (ActionThread.ExitCode != 0)
{
bDistccResult = false;
}
// Log CPU time, tool and task.
double ThreadSeconds = Action.Duration.TotalSeconds;
Log.WriteLineIf(bLogDetailedActionStats,
LogEventType.Console,
"^{0}^{1:0.00}^{2}^{3}^{4}",
Action.ActionType.ToString(),
ThreadSeconds,
Action.CommandPath.GetFileName(),
Action.StatusDescription,
Action.bIsUsingPCH);
// Keep track of total thread seconds spent on tasks.
TotalThreadSeconds += ThreadSeconds;
}
Log.WriteLineIf(bLogDetailedActionStats,
LogEventType.Console,
"-------- End Detailed Actions Stats -----------------------------------------------------------");
// Log total CPU seconds and numbers of processors involved in tasks.
Log.WriteLineIf(bLogDetailedActionStats,
LogEventType.Console, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds);
}
return(bDistccResult);
}
/// <summary>
/// Executes the specified actions locally.
/// </summary>
/// <returns>True if all the tasks successfully executed, or false if any of them failed.</returns>
public override bool ExecuteActions(List <Action> Actions, bool bLogDetailedActionStats)
{
// Time to sleep after each iteration of the loop in order to not busy wait.
const float LoopSleepTime = 0.1f;
// The number of actions to execute in parallel is trying to keep the CPU busy enough in presence of I/O stalls.
int MaxActionsToExecuteInParallel = GetMaxActionsToExecuteInParallel();
Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel);
Dictionary <Action, ActionThread> ActionThreadDictionary = new Dictionary <Action, ActionThread>();
int JobNumber = 1;
using (ProgressWriter ProgressWriter = new ProgressWriter("Compiling C++ source code...", false))
{
int ProgressValue = 0;
while (true)
{
// Count the number of pending and still executing actions.
int NumUnexecutedActions = 0;
int NumExecutingActions = 0;
foreach (Action Action in Actions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
NumUnexecutedActions++;
}
else if (ActionThread != null)
{
if (ActionThread.bComplete == false)
{
NumUnexecutedActions++;
NumExecutingActions++;
}
}
}
// Update the current progress
int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions;
if (ProgressValue != NewProgressValue)
{
ProgressWriter.Write(ProgressValue, Actions.Count + 1);
ProgressValue = NewProgressValue;
}
// If there aren't any pending actions left, we're done executing.
if (NumUnexecutedActions == 0)
{
break;
}
// If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
// prerequisites.
foreach (Action Action in Actions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel))
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (Action PrerequisiteAction in Action.PrerequisiteActions)
{
if (Actions.Contains(PrerequisiteAction))
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue(PrerequisiteAction, out PrerequisiteProcess);
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
ActionThreadDictionary.Add(Action, null);
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count);
JobNumber++;
try
{
ActionThread.Run();
}
catch (Exception ex)
{
throw new BuildException(ex, "Failed to start thread for action: {0} {1}\r\n{2}", Action.CommandPath, Action.CommandArguments, ex.ToString());
}
ActionThreadDictionary.Add(Action, ActionThread);
NumExecutingActions++;
}
}
}
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime));
}
}
Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "-------- Begin Detailed Action Stats ----------------------------------------------------------");
Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "^Action Type^Duration (seconds)^Tool^Task^Using PCH");
double TotalThreadSeconds = 0;
// Check whether any of the tasks failed and log action stats if wanted.
bool bSuccess = true;
double TotalBuildProjectTime = 0;
double TotalCompileTime = 0;
double TotalCreateAppBundleTime = 0;
double TotalGenerateDebugInfoTime = 0;
double TotalLinkTime = 0;
double TotalOtherActionsTime = 0;
foreach (KeyValuePair <Action, ActionThread> ActionProcess in ActionThreadDictionary)
{
Action Action = ActionProcess.Key;
ActionThread ActionThread = ActionProcess.Value;
// Check for pending actions, preemptive failure
if (ActionThread == null)
{
bSuccess = false;
continue;
}
// Check for executed action but general failure
if (ActionThread.ExitCode != 0)
{
bSuccess = false;
}
// Log CPU time, tool and task.
double ThreadSeconds = Action.Duration.TotalSeconds;
Log.WriteLineIf(bLogDetailedActionStats,
LogEventType.Console,
"^{0}^{1:0.00}^{2}^{3}^{4}",
Action.ActionType.ToString(),
ThreadSeconds,
Action.CommandPath.GetFileName(),
Action.StatusDescription,
Action.bIsUsingPCH);
// Update statistics
switch (Action.ActionType)
{
case ActionType.BuildProject:
TotalBuildProjectTime += ThreadSeconds;
break;
case ActionType.Compile:
TotalCompileTime += ThreadSeconds;
break;
case ActionType.CreateAppBundle:
TotalCreateAppBundleTime += ThreadSeconds;
break;
case ActionType.GenerateDebugInfo:
TotalGenerateDebugInfoTime += ThreadSeconds;
break;
case ActionType.Link:
TotalLinkTime += ThreadSeconds;
break;
default:
TotalOtherActionsTime += ThreadSeconds;
break;
}
// Keep track of total thread seconds spent on tasks.
TotalThreadSeconds += ThreadSeconds;
}
Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "-------- End Detailed Actions Stats -----------------------------------------------------------");
// Log total CPU seconds and numbers of processors involved in tasks.
Log.WriteLineIf(bLogDetailedActionStats,
LogEventType.Console, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds);
// Log detailed stats
Log.WriteLineIf(bLogDetailedActionStats,
LogEventType.Console,
"Cumulative action seconds ({0} processors): {1:0.00} building projects, {2:0.00} compiling, {3:0.00} creating app bundles, {4:0.00} generating debug info, {5:0.00} linking, {6:0.00} other",
System.Environment.ProcessorCount,
TotalBuildProjectTime,
TotalCompileTime,
TotalCreateAppBundleTime,
TotalGenerateDebugInfoTime,
TotalLinkTime,
TotalOtherActionsTime
);
return(bSuccess);
}
/**
* Executes the specified actions locally.
* @return True if all the tasks successfully executed, or false if any of them failed.
*/
public static bool ExecuteActions(List<Action> Actions)
{
// Time to sleep after each iteration of the loop in order to not busy wait.
const float LoopSleepTime = 0.1f;
// Use WMI to figure out physical cores, excluding hyper threading.
int NumCores = 0;
if (!Utils.IsRunningOnMono)
{
try
{
using (var Mos = new System.Management.ManagementObjectSearcher("Select * from Win32_Processor"))
{
var MosCollection = Mos.Get();
foreach (var Item in MosCollection)
{
NumCores += int.Parse(Item["NumberOfCores"].ToString());
}
}
}
catch (Exception Ex)
{
Log.TraceWarning("Unable to get the number of Cores: {0}", Ex.ToString());
Log.TraceWarning("Falling back to processor count.");
}
}
// On some systems this requires a hot fix to work so we fall back to using the (logical) processor count.
if( NumCores == 0 )
{
NumCores = System.Environment.ProcessorCount;
}
// The number of actions to execute in parallel is trying to keep the CPU busy enough in presence of I/O stalls.
int MaxActionsToExecuteInParallel = 0;
if (NumCores < System.Environment.ProcessorCount && BuildConfiguration.ProcessorCountMultiplier != 1.0)
{
// The CPU has more logical cores than physical ones, aka uses hyper-threading.
// Use multiplier if provided
MaxActionsToExecuteInParallel = (int)(NumCores * BuildConfiguration.ProcessorCountMultiplier);
}
else if (NumCores < System.Environment.ProcessorCount && NumCores > 4)
{
// The CPU has more logical cores than physical ones, aka uses hyper-threading.
// Use average of logical and physical if we have "lots of cores"
MaxActionsToExecuteInParallel = (int)(NumCores + System.Environment.ProcessorCount) / 2;
}
// No hyper-threading. Only kicking off a task per CPU to keep machine responsive.
else
{
MaxActionsToExecuteInParallel = NumCores;
}
if (Utils.IsRunningOnMono)
{
// heuristic: give each action at least 1.5GB of RAM (some clang instances will need more, actually)
long MinMemoryPerActionMB = 3 * 1024 / 2;
long PhysicalRAMAvailableMB = (new PerformanceCounter ("Mono Memory", "Total Physical Memory").RawValue) / (1024 * 1024);
int MaxActionsAffordedByMemory = (int)(Math.Max(1, (PhysicalRAMAvailableMB) / MinMemoryPerActionMB));
MaxActionsToExecuteInParallel = Math.Min(MaxActionsToExecuteInParallel, MaxActionsAffordedByMemory);
}
MaxActionsToExecuteInParallel = Math.Max( 1, Math.Min(MaxActionsToExecuteInParallel, BuildConfiguration.MaxProcessorCount) );
Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel);
Dictionary<Action, ActionThread> ActionThreadDictionary = new Dictionary<Action, ActionThread>();
int JobNumber = 1;
using(ProgressWriter ProgressWriter = new ProgressWriter("Compiling C++ source code...", false))
{
int ProgressValue = 0;
while(true)
{
// Count the number of pending and still executing actions.
int NumUnexecutedActions = 0;
int NumExecutingActions = 0;
foreach (Action Action in Actions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
NumUnexecutedActions++;
}
else if (ActionThread != null)
{
if (ActionThread.bComplete == false)
{
NumUnexecutedActions++;
NumExecutingActions++;
}
}
}
// Update the current progress
int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions;
if (ProgressValue != NewProgressValue)
{
ProgressWriter.Write(ProgressValue, Actions.Count + 1);
ProgressValue = NewProgressValue;
}
// If there aren't any pending actions left, we're done executing.
if (NumUnexecutedActions == 0)
{
break;
}
// If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
// prerequisites.
foreach (Action Action in Actions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
if (NumExecutingActions < Math.Max(1,MaxActionsToExecuteInParallel))
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems)
{
if (PrerequisiteItem.ProducingAction != null && Actions.Contains(PrerequisiteItem.ProducingAction))
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue( PrerequisiteItem.ProducingAction, out PrerequisiteProcess );
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
ActionThreadDictionary.Add( Action, null );
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count);
JobNumber++;
ActionThread.Run();
ActionThreadDictionary.Add(Action, ActionThread);
NumExecutingActions++;
}
}
}
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime));
}
}
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, LogEventType.Console, "-------- Begin Detailed Action Stats ----------------------------------------------------------");
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, LogEventType.Console, "^Action Type^Duration (seconds)^Tool^Task^Using PCH");
double TotalThreadSeconds = 0;
// Check whether any of the tasks failed and log action stats if wanted.
bool bSuccess = true;
foreach (KeyValuePair<Action, ActionThread> ActionProcess in ActionThreadDictionary)
{
Action Action = ActionProcess.Key;
ActionThread ActionThread = ActionProcess.Value;
// Check for pending actions, preemptive failure
if (ActionThread == null)
{
bSuccess = false;
continue;
}
// Check for executed action but general failure
if (ActionThread.ExitCode != 0)
{
bSuccess = false;
}
// Log CPU time, tool and task.
double ThreadSeconds = Action.Duration.TotalSeconds;
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats,
LogEventType.Console,
"^{0}^{1:0.00}^{2}^{3}^{4}",
Action.ActionType.ToString(),
ThreadSeconds,
Path.GetFileName(Action.CommandPath),
Action.StatusDescription,
Action.bIsUsingPCH);
// Update statistics
switch (Action.ActionType)
{
case ActionType.BuildProject:
UnrealBuildTool.TotalBuildProjectTime += ThreadSeconds;
break;
case ActionType.Compile:
UnrealBuildTool.TotalCompileTime += ThreadSeconds;
break;
case ActionType.CreateAppBundle:
UnrealBuildTool.TotalCreateAppBundleTime += ThreadSeconds;
break;
case ActionType.GenerateDebugInfo:
UnrealBuildTool.TotalGenerateDebugInfoTime += ThreadSeconds;
break;
case ActionType.Link:
UnrealBuildTool.TotalLinkTime += ThreadSeconds;
break;
default:
UnrealBuildTool.TotalOtherActionsTime += ThreadSeconds;
break;
}
// Keep track of total thread seconds spent on tasks.
TotalThreadSeconds += ThreadSeconds;
}
Log.TraceInformation("-------- End Detailed Actions Stats -----------------------------------------------------------");
// Log total CPU seconds and numbers of processors involved in tasks.
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats || BuildConfiguration.bPrintDebugInfo,
LogEventType.Console, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds);
return bSuccess;
}
public static bool ExecuteActions(List <Action> Actions)
{
bool bDistccResult = true;
if (Actions.Count > 0)
{
// Time to sleep after each iteration of the loop in order to not busy wait.
const float LoopSleepTime = 0.1f;
int MaxActionsToExecuteInParallel = 0;
string UserDir = Environment.GetEnvironmentVariable("HOME");
string HostsInfo = UserDir + "/.dmucs/hosts-info";
System.IO.StreamReader File = new System.IO.StreamReader(HostsInfo);
string Line = null;
while ((Line = File.ReadLine()) != null)
{
var HostInfo = Line.Split(' ');
if (HostInfo.Count() == 3)
{
int NumCPUs = 0;
if (System.Int32.TryParse(HostInfo [1], out NumCPUs))
{
MaxActionsToExecuteInParallel += NumCPUs;
}
}
}
File.Close();
if (BuildConfiguration.bAllowDistccLocalFallback == false)
{
Environment.SetEnvironmentVariable("DISTCC_FALLBACK", "0");
}
string DistccExecutable = BuildConfiguration.DistccExecutablesPath + "/distcc";
string GetHostExecutable = BuildConfiguration.DistccExecutablesPath + "/gethost";
Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel, DistccExecutable, GetHostExecutable);
Dictionary <Action, ActionThread> ActionThreadDictionary = new Dictionary <Action, ActionThread>();
int JobNumber = 1;
using (ProgressWriter ProgressWriter = new ProgressWriter("Compiling source code...", false))
{
int ProgressValue = 0;
while (true)
{
// Count the number of pending and still executing actions.
int NumUnexecutedActions = 0;
int NumExecutingActions = 0;
foreach (Action Action in Actions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
NumUnexecutedActions++;
}
else if (ActionThread != null)
{
if (ActionThread.bComplete == false)
{
NumUnexecutedActions++;
NumExecutingActions++;
}
}
}
// Update the current progress
int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions;
if (ProgressValue != NewProgressValue)
{
ProgressWriter.Write(ProgressValue, Actions.Count + 1);
ProgressValue = NewProgressValue;
}
// If there aren't any pending actions left, we're done executing.
if (NumUnexecutedActions == 0)
{
break;
}
// If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
// prerequisites.
foreach (Action Action in Actions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel))
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems)
{
if (PrerequisiteItem.ProducingAction != null && Actions.Contains(PrerequisiteItem.ProducingAction))
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue(PrerequisiteItem.ProducingAction, out PrerequisiteProcess);
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
ActionThreadDictionary.Add(Action, null);
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
if ((Action.ActionType == ActionType.Compile || Action.ActionType == ActionType.Link) && DistccExecutable != null && GetHostExecutable != null)
{
string NewCommandArguments = "--wait -1 \"" + DistccExecutable + "\" " + Action.CommandPath + " " + Action.CommandArguments;
Action.CommandPath = GetHostExecutable;
Action.CommandArguments = NewCommandArguments;
}
ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count);
JobNumber++;
ActionThread.Run();
ActionThreadDictionary.Add(Action, ActionThread);
NumExecutingActions++;
}
}
}
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime));
}
}
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "-------- Begin Detailed Action Stats ----------------------------------------------------------");
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "^Action Type^Duration (seconds)^Tool^Task^Using PCH");
double TotalThreadSeconds = 0;
// Check whether any of the tasks failed and log action stats if wanted.
foreach (KeyValuePair <Action, ActionThread> ActionProcess in ActionThreadDictionary)
{
Action Action = ActionProcess.Key;
ActionThread ActionThread = ActionProcess.Value;
// Check for pending actions, preemptive failure
if (ActionThread == null)
{
bDistccResult = false;
continue;
}
// Check for executed action but general failure
if (ActionThread.ExitCode != 0)
{
bDistccResult = false;
}
// Log CPU time, tool and task.
double ThreadSeconds = Action.Duration.TotalSeconds;
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats,
TraceEventType.Information,
"^{0}^{1:0.00}^{2}^{3}^{4}",
Action.ActionType.ToString(),
ThreadSeconds,
Path.GetFileName(Action.CommandPath),
Action.StatusDescription,
Action.bIsUsingPCH);
// Update statistics
switch (Action.ActionType)
{
case ActionType.BuildProject:
UnrealBuildTool.TotalBuildProjectTime += ThreadSeconds;
break;
case ActionType.Compile:
UnrealBuildTool.TotalCompileTime += ThreadSeconds;
break;
case ActionType.CreateAppBundle:
UnrealBuildTool.TotalCreateAppBundleTime += ThreadSeconds;
break;
case ActionType.GenerateDebugInfo:
UnrealBuildTool.TotalGenerateDebugInfoTime += ThreadSeconds;
break;
case ActionType.Link:
UnrealBuildTool.TotalLinkTime += ThreadSeconds;
break;
default:
UnrealBuildTool.TotalOtherActionsTime += ThreadSeconds;
break;
}
// Keep track of total thread seconds spent on tasks.
TotalThreadSeconds += ThreadSeconds;
}
Log.TraceInformation("-------- End Detailed Actions Stats -----------------------------------------------------------");
// Log total CPU seconds and numbers of processors involved in tasks.
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats || BuildConfiguration.bPrintDebugInfo,
TraceEventType.Information, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds);
}
return(bDistccResult);
}
public static bool ExecuteActions(List<Action> Actions)
{
bool bDistccResult = true;
if (Actions.Count > 0)
{
// Time to sleep after each iteration of the loop in order to not busy wait.
const float LoopSleepTime = 0.1f;
int MaxActionsToExecuteInParallel = 0;
string UserDir = Environment.GetEnvironmentVariable("HOME");
string HostsInfo = UserDir + "/.dmucs/hosts-info";
string NumUBTBuildTasks = Environment.GetEnvironmentVariable("NumUBTBuildTasks");
Int32 MaxUBTBuildTasks = MaxActionsToExecuteInParallel;
if(Int32.TryParse(NumUBTBuildTasks, out MaxUBTBuildTasks))
{
MaxActionsToExecuteInParallel = MaxUBTBuildTasks;
}
else if (BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Mac)
{
using (System.Diagnostics.Process DefaultsProcess = new System.Diagnostics.Process())
{
try
{
DefaultsProcess.StartInfo.FileName = "/usr/bin/defaults";
DefaultsProcess.StartInfo.CreateNoWindow = true;
DefaultsProcess.StartInfo.UseShellExecute = false;
DefaultsProcess.StartInfo.RedirectStandardOutput = true;
DefaultsProcess.StartInfo.Arguments = "com.apple.dt.Xcode IDEBuildOperationMaxNumberOfConcurrentCompileTasks";
DefaultsProcess.Start();
string Output = DefaultsProcess.StandardOutput.ReadToEnd();
DefaultsProcess.WaitForExit();
if(DefaultsProcess.ExitCode == 0 && Int32.TryParse(Output, out MaxUBTBuildTasks))
{
MaxActionsToExecuteInParallel = MaxUBTBuildTasks;
}
}
catch (Exception)
{
}
}
}
else if (System.IO.File.Exists (HostsInfo))
{
System.IO.StreamReader File = new System.IO.StreamReader(HostsInfo);
string Line = null;
while((Line = File.ReadLine()) != null)
{
var HostInfo = Line.Split(' ');
if (HostInfo.Count () == 3)
{
int NumCPUs = 0;
if (System.Int32.TryParse (HostInfo [1], out NumCPUs))
{
MaxActionsToExecuteInParallel += NumCPUs;
}
}
}
File.Close();
}
else
{
MaxActionsToExecuteInParallel = System.Environment.ProcessorCount;
}
if (BuildConfiguration.bAllowDistccLocalFallback == false)
{
Environment.SetEnvironmentVariable("DISTCC_FALLBACK", "0");
}
if (BuildConfiguration.bVerboseDistccOutput == true)
{
Environment.SetEnvironmentVariable("DISTCC_VERBOSE", "1");
}
string DistccExecutable = BuildConfiguration.DistccExecutablesPath + "/distcc";
string GetHostExecutable = BuildConfiguration.DistccExecutablesPath + "/gethost";
Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel, DistccExecutable, GetHostExecutable);
Dictionary<Action, ActionThread> ActionThreadDictionary = new Dictionary<Action, ActionThread>();
int JobNumber = 1;
using(ProgressWriter ProgressWriter = new ProgressWriter("Compiling source code...", false))
{
int ProgressValue = 0;
while(true)
{
// Count the number of pending and still executing actions.
int NumUnexecutedActions = 0;
int NumExecutingActions = 0;
foreach (Action Action in Actions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
NumUnexecutedActions++;
}
else if (ActionThread != null)
{
if (ActionThread.bComplete == false)
{
NumUnexecutedActions++;
NumExecutingActions++;
}
}
}
// Update the current progress
int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions;
if (ProgressValue != NewProgressValue)
{
ProgressWriter.Write(ProgressValue, Actions.Count + 1);
ProgressValue = NewProgressValue;
}
// If there aren't any pending actions left, we're done executing.
if (NumUnexecutedActions == 0)
{
break;
}
// If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
// prerequisites.
foreach (Action Action in Actions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
if (NumExecutingActions < Math.Max(1,MaxActionsToExecuteInParallel))
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems)
{
if (PrerequisiteItem.ProducingAction != null && Actions.Contains(PrerequisiteItem.ProducingAction))
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue( PrerequisiteItem.ProducingAction, out PrerequisiteProcess );
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
ActionThreadDictionary.Add( Action, null );
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
if ((Action.ActionType == ActionType.Compile || Action.ActionType == ActionType.Link) && DistccExecutable != null && GetHostExecutable != null)
{
string TypeCommand = String.IsNullOrEmpty(BuildConfiguration.DMUCSDistProp) ? "" : (" --type " + BuildConfiguration.DMUCSDistProp);
string NewCommandArguments = "--server " + BuildConfiguration.DMUCSCoordinator + TypeCommand + " --wait -1 \"" + DistccExecutable + "\" " + Action.CommandPath + " " + Action.CommandArguments;
Action.CommandPath = GetHostExecutable;
Action.CommandArguments = NewCommandArguments;
}
ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count);
JobNumber++;
ActionThread.Run();
ActionThreadDictionary.Add(Action, ActionThread);
NumExecutingActions++;
}
}
}
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime));
}
}
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, LogEventType.Console, "-------- Begin Detailed Action Stats ----------------------------------------------------------");
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, LogEventType.Console, "^Action Type^Duration (seconds)^Tool^Task^Using PCH");
double TotalThreadSeconds = 0;
// Check whether any of the tasks failed and log action stats if wanted.
foreach (KeyValuePair<Action, ActionThread> ActionProcess in ActionThreadDictionary)
{
Action Action = ActionProcess.Key;
ActionThread ActionThread = ActionProcess.Value;
// Check for pending actions, preemptive failure
if (ActionThread == null)
{
bDistccResult = false;
continue;
}
// Check for executed action but general failure
if (ActionThread.ExitCode != 0)
{
bDistccResult = false;
}
// Log CPU time, tool and task.
double ThreadSeconds = Action.Duration.TotalSeconds;
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats,
LogEventType.Console,
"^{0}^{1:0.00}^{2}^{3}^{4}",
Action.ActionType.ToString(),
ThreadSeconds,
Path.GetFileName(Action.CommandPath),
Action.StatusDescription,
Action.bIsUsingPCH);
// Update statistics
switch (Action.ActionType)
{
case ActionType.BuildProject:
UnrealBuildTool.TotalBuildProjectTime += ThreadSeconds;
break;
case ActionType.Compile:
UnrealBuildTool.TotalCompileTime += ThreadSeconds;
break;
case ActionType.CreateAppBundle:
UnrealBuildTool.TotalCreateAppBundleTime += ThreadSeconds;
break;
case ActionType.GenerateDebugInfo:
UnrealBuildTool.TotalGenerateDebugInfoTime += ThreadSeconds;
break;
case ActionType.Link:
UnrealBuildTool.TotalLinkTime += ThreadSeconds;
break;
default:
UnrealBuildTool.TotalOtherActionsTime += ThreadSeconds;
break;
}
// Keep track of total thread seconds spent on tasks.
TotalThreadSeconds += ThreadSeconds;
}
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats || BuildConfiguration.bPrintDebugInfo,
LogEventType.Console,
"-------- End Detailed Actions Stats -----------------------------------------------------------");
// Log total CPU seconds and numbers of processors involved in tasks.
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats || BuildConfiguration.bPrintDebugInfo,
LogEventType.Console, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds);
}
return bDistccResult;
}
/**
* Executes the specified actions locally.
* @return True if all the tasks successfully executed, or false if any of them failed.
*/
public static bool ExecuteActions(List <Action> Actions)
{
// Time to sleep after each iteration of the loop in order to not busy wait.
const float LoopSleepTime = 0.1f;
// Use WMI to figure out physical cores, excluding hyper threading.
int NumCores = 0;
if (!Utils.IsRunningOnMono)
{
try
{
using (var Mos = new System.Management.ManagementObjectSearcher("Select * from Win32_Processor"))
{
var MosCollection = Mos.Get();
foreach (var Item in MosCollection)
{
NumCores += int.Parse(Item["NumberOfCores"].ToString());
}
}
}
catch (Exception Ex)
{
Log.TraceWarning("Unable to get the number of Cores: {0}", Ex.ToString());
Log.TraceWarning("Falling back to processor count.");
}
}
// On some systems this requires a hot fix to work so we fall back to using the (logical) processor count.
if (NumCores == 0)
{
NumCores = System.Environment.ProcessorCount;
}
// The number of actions to execute in parallel is trying to keep the CPU busy enough in presence of I/O stalls.
int MaxActionsToExecuteInParallel = 0;
// The CPU has more logical cores than physical ones, aka uses hyper-threading.
if (NumCores < System.Environment.ProcessorCount)
{
MaxActionsToExecuteInParallel = (int)(NumCores * BuildConfiguration.ProcessorCountMultiplier);
}
// No hyper-threading. Only kicking off a task per CPU to keep machine responsive.
else
{
MaxActionsToExecuteInParallel = NumCores;
}
if (Utils.IsRunningOnMono)
{
// heuristic: give each action at least 1.5GB of RAM (some clang instances will need more, actually)
long MinMemoryPerActionMB = 3 * 1024 / 2;
long PhysicalRAMAvailableMB = (new PerformanceCounter("Mono Memory", "Total Physical Memory").RawValue) / (1024 * 1024);
int MaxActionsAffordedByMemory = (int)(Math.Max(1, (PhysicalRAMAvailableMB) / MinMemoryPerActionMB));
MaxActionsToExecuteInParallel = Math.Min(MaxActionsToExecuteInParallel, MaxActionsAffordedByMemory);
}
MaxActionsToExecuteInParallel = Math.Min(MaxActionsToExecuteInParallel, BuildConfiguration.MaxProcessorCount);
Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel);
Dictionary <Action, ActionThread> ActionThreadDictionary = new Dictionary <Action, ActionThread>();
int JobNumber = 1;
using (ProgressWriter ProgressWriter = new ProgressWriter("Compiling source code...", false))
{
int ProgressValue = 0;
while (true)
{
// Count the number of pending and still executing actions.
int NumUnexecutedActions = 0;
int NumExecutingActions = 0;
foreach (Action Action in Actions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
NumUnexecutedActions++;
}
else if (ActionThread != null)
{
if (ActionThread.bComplete == false)
{
NumUnexecutedActions++;
NumExecutingActions++;
}
}
}
// Update the current progress
int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions;
if (ProgressValue != NewProgressValue)
{
ProgressWriter.Write(ProgressValue, Actions.Count + 1);
ProgressValue = NewProgressValue;
}
// If there aren't any pending actions left, we're done executing.
if (NumUnexecutedActions == 0)
{
break;
}
// If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
// prerequisites.
foreach (Action Action in Actions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel))
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems)
{
if (PrerequisiteItem.ProducingAction != null && Actions.Contains(PrerequisiteItem.ProducingAction))
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue(PrerequisiteItem.ProducingAction, out PrerequisiteProcess);
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
ActionThreadDictionary.Add(Action, null);
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count);
JobNumber++;
ActionThread.Run();
ActionThreadDictionary.Add(Action, ActionThread);
NumExecutingActions++;
}
}
}
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime));
}
}
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "-------- Begin Detailed Action Stats ----------------------------------------------------------");
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "^Action Type^Duration (seconds)^Tool^Task^Using PCH");
double TotalThreadSeconds = 0;
// Check whether any of the tasks failed and log action stats if wanted.
bool bSuccess = true;
foreach (KeyValuePair <Action, ActionThread> ActionProcess in ActionThreadDictionary)
{
Action Action = ActionProcess.Key;
ActionThread ActionThread = ActionProcess.Value;
// Check for pending actions, preemptive failure
if (ActionThread == null)
{
bSuccess = false;
continue;
}
// Check for executed action but general failure
if (ActionThread.ExitCode != 0)
{
bSuccess = false;
}
// Log CPU time, tool and task.
double ThreadSeconds = Action.Duration.TotalSeconds;
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats,
TraceEventType.Information,
"^{0}^{1:0.00}^{2}^{3}^{4}",
Action.ActionType.ToString(),
ThreadSeconds,
Path.GetFileName(Action.CommandPath),
Action.StatusDescription,
Action.bIsUsingPCH);
// Update statistics
switch (Action.ActionType)
{
case ActionType.BuildProject:
UnrealBuildTool.TotalBuildProjectTime += ThreadSeconds;
break;
case ActionType.Compile:
UnrealBuildTool.TotalCompileTime += ThreadSeconds;
break;
case ActionType.CreateAppBundle:
UnrealBuildTool.TotalCreateAppBundleTime += ThreadSeconds;
break;
case ActionType.GenerateDebugInfo:
UnrealBuildTool.TotalGenerateDebugInfoTime += ThreadSeconds;
break;
case ActionType.Link:
UnrealBuildTool.TotalLinkTime += ThreadSeconds;
break;
default:
UnrealBuildTool.TotalOtherActionsTime += ThreadSeconds;
break;
}
// Keep track of total thread seconds spent on tasks.
TotalThreadSeconds += ThreadSeconds;
}
Log.TraceInformation("-------- End Detailed Actions Stats -----------------------------------------------------------");
// Log total CPU seconds and numbers of processors involved in tasks.
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats || BuildConfiguration.bPrintDebugInfo,
TraceEventType.Information, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds);
return(bSuccess);
}
/**
* Executes the specified actions locally.
* @return True if all the tasks successfully executed, or false if any of them failed.
*/
public static bool ExecuteActions(List<Action> Actions)
{
// Time to sleep after each iteration of the loop in order to not busy wait.
const float LoopSleepTime = 0.1f;
// Use WMI to figure out physical cores, excluding hyper threading.
int NumCores = 0;
if (!Utils.IsRunningOnMono)
{
foreach(var Item in new System.Management.ManagementObjectSearcher("Select * from Win32_Processor").Get())
{
NumCores += int.Parse(Item["NumberOfCores"].ToString());
}
}
// On some systems this requires a hot fix to work so we fall back to using the (logical) processor count.
if( NumCores == 0 )
{
NumCores = System.Environment.ProcessorCount;
}
// The number of actions to execute in parallel is trying to keep the CPU busy enough in presence of I/O stalls.
int MaxActionsToExecuteInParallel = 0;
// The CPU has more logical cores than physical ones, aka uses hyper-threading.
if( NumCores < System.Environment.ProcessorCount )
{
MaxActionsToExecuteInParallel = (int) (NumCores * BuildConfiguration.ProcessorCountMultiplier);
}
// No hyper-threading. Only kicking off a task per CPU to keep machine responsive.
else
{
MaxActionsToExecuteInParallel = NumCores;
}
if( Utils.IsRunningOnMono )
{
// @todo iosmerge: this should be looking at available memory as well since some of our
// Macs have a poor memory/CPU ratio
MaxActionsToExecuteInParallel /= 2;
}
Dictionary<Action, ActionThread> ActionThreadDictionary = new Dictionary<Action, ActionThread>();
while(true)
{
// Count the number of pending and still executing actions.
int NumUnexecutedActions = 0;
int NumExecutingActions = 0;
foreach (Action Action in Actions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
NumUnexecutedActions++;
}
else if (ActionThread != null)
{
if (ActionThread.bComplete == false)
{
NumUnexecutedActions++;
NumExecutingActions++;
}
}
}
// If there aren't any pending actions left, we're done executing.
if (NumUnexecutedActions == 0)
{
break;
}
// If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
// prerequisites.
foreach (Action Action in Actions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
if (NumExecutingActions < Math.Max(1,MaxActionsToExecuteInParallel))
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems)
{
if (PrerequisiteItem.ProducingAction != null && Actions.Contains(PrerequisiteItem.ProducingAction))
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue( PrerequisiteItem.ProducingAction, out PrerequisiteProcess );
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
ActionThreadDictionary.Add( Action, null );
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
ActionThread ActionThread = new ActionThread(Action);
ActionThread.Run();
ActionThreadDictionary.Add(Action, ActionThread);
NumExecutingActions++;
}
}
}
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime));
}
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "-------- Begin Detailed Action Stats ----------------------------------------------------------");
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "^Action Type^Duration (seconds)^Tool^Task^Using PCH^Description");
double TotalThreadSeconds = 0;
// Check whether any of the tasks failed and log action stats if wanted.
bool bSuccess = true;
foreach (KeyValuePair<Action, ActionThread> ActionProcess in ActionThreadDictionary)
{
Action Action = ActionProcess.Key;
ActionThread ActionThread = ActionProcess.Value;
// Check for pending actions, preemptive failure
if (ActionThread == null)
{
bSuccess = false;
continue;
}
// Check for executed action but general failure
if (ActionThread.ExitCode != 0)
{
bSuccess = false;
}
// Log CPU time, tool and task.
double ThreadSeconds = Action.Duration.TotalSeconds;
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats,
TraceEventType.Information,
"^{0}^{1:0.00}^{2}^{3}^{4}^{5}",
Action.ActionType.ToString(),
ThreadSeconds,
Path.GetFileName(Action.CommandPath),
Action.StatusDescription,
Action.bIsUsingPCH,
Action.StatusDetailedDescription);
// Update statistics
switch (Action.ActionType)
{
case ActionType.BuildProject:
UnrealBuildTool.TotalBuildProjectTime += ThreadSeconds;
break;
case ActionType.Compile:
UnrealBuildTool.TotalCompileTime += ThreadSeconds;
break;
case ActionType.CreateAppBundle:
UnrealBuildTool.TotalCreateAppBundleTime += ThreadSeconds;
break;
case ActionType.GenerateDebugInfo:
UnrealBuildTool.TotalGenerateDebugInfoTime += ThreadSeconds;
break;
case ActionType.Link:
UnrealBuildTool.TotalLinkTime += ThreadSeconds;
break;
default:
UnrealBuildTool.TotalOtherActionsTime += ThreadSeconds;
break;
}
// Keep track of total thread seconds spent on tasks.
TotalThreadSeconds += ThreadSeconds;
}
Log.TraceInformation("-------- End Detailed Actions Stats -----------------------------------------------------------");
// Log total CPU seconds and numbers of processors involved in tasks.
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats || BuildConfiguration.bPrintDebugInfo,
TraceEventType.Information, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds);
return bSuccess;
}
