public static bool ExecuteActions(List <Action> Actions)
{
bool bDistccResult = true;
if (Actions.Count > 0)
{
// Time to sleep after each iteration of the loop in order to not busy wait.
const float LoopSleepTime = 0.1f;
int MaxActionsToExecuteInParallel = 0;
string UserDir = Environment.GetEnvironmentVariable("HOME");
string HostsInfo = UserDir + "/.dmucs/hosts-info";
System.IO.StreamReader File = new System.IO.StreamReader(HostsInfo);
string Line = null;
while ((Line = File.ReadLine()) != null)
{
var HostInfo = Line.Split(' ');
if (HostInfo.Count() == 3)
{
int NumCPUs = 0;
if (System.Int32.TryParse(HostInfo [1], out NumCPUs))
{
MaxActionsToExecuteInParallel += NumCPUs;
}
}
}
File.Close();
if (BuildConfiguration.bAllowDistccLocalFallback == false)
{
Environment.SetEnvironmentVariable("DISTCC_FALLBACK", "0");
}
string DistccExecutable = BuildConfiguration.DistccExecutablesPath + "/distcc";
string GetHostExecutable = BuildConfiguration.DistccExecutablesPath + "/gethost";
Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel, DistccExecutable, GetHostExecutable);
Dictionary <Action, ActionThread> ActionThreadDictionary = new Dictionary <Action, ActionThread>();
int JobNumber = 1;
using (ProgressWriter ProgressWriter = new ProgressWriter("Compiling source code...", false))
{
int ProgressValue = 0;
while (true)
{
// Count the number of pending and still executing actions.
int NumUnexecutedActions = 0;
int NumExecutingActions = 0;
foreach (Action Action in Actions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
NumUnexecutedActions++;
}
else if (ActionThread != null)
{
if (ActionThread.bComplete == false)
{
NumUnexecutedActions++;
NumExecutingActions++;
}
}
}
// Update the current progress
int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions;
if (ProgressValue != NewProgressValue)
{
ProgressWriter.Write(ProgressValue, Actions.Count + 1);
ProgressValue = NewProgressValue;
}
// If there aren't any pending actions left, we're done executing.
if (NumUnexecutedActions == 0)
{
break;
}
// If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
// prerequisites.
foreach (Action Action in Actions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel))
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems)
{
if (PrerequisiteItem.ProducingAction != null && Actions.Contains(PrerequisiteItem.ProducingAction))
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue(PrerequisiteItem.ProducingAction, out PrerequisiteProcess);
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
ActionThreadDictionary.Add(Action, null);
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
if ((Action.ActionType == ActionType.Compile || Action.ActionType == ActionType.Link) && DistccExecutable != null && GetHostExecutable != null)
{
string NewCommandArguments = "--wait -1 \"" + DistccExecutable + "\" " + Action.CommandPath + " " + Action.CommandArguments;
Action.CommandPath = GetHostExecutable;
Action.CommandArguments = NewCommandArguments;
}
ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count);
JobNumber++;
ActionThread.Run();
ActionThreadDictionary.Add(Action, ActionThread);
NumExecutingActions++;
}
}
}
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime));
}
}
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "-------- Begin Detailed Action Stats ----------------------------------------------------------");
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "^Action Type^Duration (seconds)^Tool^Task^Using PCH");
double TotalThreadSeconds = 0;
// Check whether any of the tasks failed and log action stats if wanted.
foreach (KeyValuePair <Action, ActionThread> ActionProcess in ActionThreadDictionary)
{
Action Action = ActionProcess.Key;
ActionThread ActionThread = ActionProcess.Value;
// Check for pending actions, preemptive failure
if (ActionThread == null)
{
bDistccResult = false;
continue;
}
// Check for executed action but general failure
if (ActionThread.ExitCode != 0)
{
bDistccResult = false;
}
// Log CPU time, tool and task.
double ThreadSeconds = Action.Duration.TotalSeconds;
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats,
TraceEventType.Information,
"^{0}^{1:0.00}^{2}^{3}^{4}",
Action.ActionType.ToString(),
ThreadSeconds,
Path.GetFileName(Action.CommandPath),
Action.StatusDescription,
Action.bIsUsingPCH);
// Update statistics
switch (Action.ActionType)
{
case ActionType.BuildProject:
UnrealBuildTool.TotalBuildProjectTime += ThreadSeconds;
break;
case ActionType.Compile:
UnrealBuildTool.TotalCompileTime += ThreadSeconds;
break;
case ActionType.CreateAppBundle:
UnrealBuildTool.TotalCreateAppBundleTime += ThreadSeconds;
break;
case ActionType.GenerateDebugInfo:
UnrealBuildTool.TotalGenerateDebugInfoTime += ThreadSeconds;
break;
case ActionType.Link:
UnrealBuildTool.TotalLinkTime += ThreadSeconds;
break;
default:
UnrealBuildTool.TotalOtherActionsTime += ThreadSeconds;
break;
}
// Keep track of total thread seconds spent on tasks.
TotalThreadSeconds += ThreadSeconds;
}
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats || BuildConfiguration.bPrintDebugInfo,
TraceEventType.Information,
"-------- End Detailed Actions Stats -----------------------------------------------------------");
// Log total CPU seconds and numbers of processors involved in tasks.
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats || BuildConfiguration.bPrintDebugInfo,
TraceEventType.Information, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds);
}
return(bDistccResult);
}
/// <summary>
/// Executes the specified actions locally.
/// </summary>
/// <returns>True if all the tasks successfully executed, or false if any of them failed.</returns>
public override bool ExecuteActions(List <Action> InputActions, bool bLogDetailedActionStats)
{
// Figure out how many processors to use
int MaxProcesses = Math.Min((int)(Environment.ProcessorCount * ProcessorCountMultiplier), MaxProcessorCount);
Log.TraceInformation("Building {0} {1} with {2} {3}...", InputActions.Count, (InputActions.Count == 1) ? "action" : "actions", MaxProcesses, (MaxProcesses == 1)? "process" : "processes");
// Create actions with all our internal metadata
List <BuildAction> Actions = new List <BuildAction>();
for (int Idx = 0; Idx < InputActions.Count; Idx++)
{
BuildAction Action = new BuildAction();
Action.SortIndex = Idx;
Action.Inner = InputActions[Idx];
Actions.Add(Action);
}
// Build a map of items to their producing actions
Dictionary <FileItem, BuildAction> FileToProducingAction = new Dictionary <FileItem, BuildAction>();
foreach (BuildAction Action in Actions)
{
foreach (FileItem ProducedItem in Action.Inner.ProducedItems)
{
FileToProducingAction[ProducedItem] = Action;
}
}
// Update all the actions with all their dependencies
foreach (BuildAction Action in Actions)
{
foreach (FileItem PrerequisiteItem in Action.Inner.PrerequisiteItems)
{
BuildAction Dependency;
if (FileToProducingAction.TryGetValue(PrerequisiteItem, out Dependency))
{
Action.Dependencies.Add(Dependency);
Dependency.Dependants.Add(Action);
}
}
}
// Figure out the recursive dependency count
HashSet <BuildAction> VisitedActions = new HashSet <BuildAction>();
foreach (BuildAction Action in Actions)
{
Action.MissingDependencyCount = Action.Dependencies.Count;
RecursiveIncDependents(Action, VisitedActions);
}
// Create the list of things to process
List <BuildAction> QueuedActions = new List <BuildAction>();
foreach (BuildAction Action in Actions)
{
if (Action.MissingDependencyCount == 0)
{
QueuedActions.Add(Action);
}
}
// Execute the actions
using (ScopedLogIndent Indent = new ScopedLogIndent(" "))
{
// Create a job object for all the child processes
bool bResult = true;
Dictionary <BuildAction, Thread> ExecutingActions = new Dictionary <BuildAction, Thread>();
List <BuildAction> CompletedActions = new List <BuildAction>();
using (ManagedProcessGroup ProcessGroup = new ManagedProcessGroup())
{
using (AutoResetEvent CompletedEvent = new AutoResetEvent(false))
{
int NumCompletedActions = 0;
using (ProgressWriter ProgressWriter = new ProgressWriter("Compiling C++ source code...", false))
{
while (QueuedActions.Count > 0 || ExecutingActions.Count > 0)
{
// Sort the actions by the number of things dependent on them
QueuedActions.Sort((A, B) => (A.TotalDependantCount == B.TotalDependantCount)? (B.SortIndex - A.SortIndex) : (B.TotalDependantCount - A.TotalDependantCount));
// Create threads up to the maximum number of actions
while (ExecutingActions.Count < MaxProcesses && QueuedActions.Count > 0)
{
BuildAction Action = QueuedActions[QueuedActions.Count - 1];
QueuedActions.RemoveAt(QueuedActions.Count - 1);
Thread ExecutingThread = new Thread(() => { ExecuteAction(ProcessGroup, Action, CompletedActions, CompletedEvent); });
ExecutingThread.Name = String.Format("Build:{0}", Action.Inner.StatusDescription);
ExecutingThread.Start();
ExecutingActions.Add(Action, ExecutingThread);
}
// Wait for something to finish
CompletedEvent.WaitOne();
// Wait for something to finish and flush it to the log
lock (CompletedActions)
{
foreach (BuildAction CompletedAction in CompletedActions)
{
// Join the thread
Thread CompletedThread = ExecutingActions[CompletedAction];
CompletedThread.Join();
ExecutingActions.Remove(CompletedAction);
// Write it to the log
if (CompletedAction.LogLines.Count > 0)
{
foreach (string LogLine in CompletedAction.LogLines)
{
Log.TraceInformation(LogLine);
}
}
// Update the progress
NumCompletedActions++;
ProgressWriter.Write(NumCompletedActions, InputActions.Count);
// Check the exit code
if (CompletedAction.ExitCode == 0)
{
// Mark all the dependents as done
foreach (BuildAction DependantAction in CompletedAction.Dependants)
{
if (--DependantAction.MissingDependencyCount == 0)
{
QueuedActions.Add(DependantAction);
}
}
}
else
{
// Update the exit code if it's not already set
if (bResult && CompletedAction.ExitCode != 0)
{
bResult = false;
}
}
}
CompletedActions.Clear();
}
// If we've already got a non-zero exit code, clear out the list of queued actions so nothing else will run
if (!bResult && bStopCompilationAfterErrors)
{
QueuedActions.Clear();
}
}
}
}
}
return(bResult);
}
}
