public BuildActionExecutor(ManagedProcessGroup InProcessGroup, BuildAction InAction, AutoResetEvent InCompletedEvent, List <BuildActionExecutor> InCompletedActions)
{
ProcessGroup = InProcessGroup;
Action = InAction;
CompletedEvent = InCompletedEvent;
CompletedActions = InCompletedActions;
}
/// <summary>
/// Execute an individual action
/// </summary>
/// <param name="ProcessGroup">The process group</param>
/// <param name="Action">The action to execute</param>
/// <param name="CompletedActions">On completion, the list to add the completed action to</param>
/// <param name="CompletedEvent">Event to set once an event is complete</param>
static void ExecuteAction(ManagedProcessGroup ProcessGroup, BuildAction Action, List <BuildAction> CompletedActions, AutoResetEvent CompletedEvent)
{
if (Action.Inner.bShouldOutputStatusDescription && !String.IsNullOrEmpty(Action.Inner.StatusDescription))
{
Action.LogLines.Add(Action.Inner.StatusDescription);
}
try
{
using (ManagedProcess Process = new ManagedProcess(ProcessGroup, Action.Inner.CommandPath.FullName, Action.Inner.CommandArguments, Action.Inner.WorkingDirectory.FullName, null, null, ProcessPriorityClass.BelowNormal))
{
Action.LogLines.AddRange(Process.ReadAllLines());
Action.ExitCode = Process.ExitCode;
}
}
catch (Exception Ex)
{
Log.WriteException(Ex, null);
Action.ExitCode = 1;
}
lock (CompletedActions)
{
CompletedActions.Add(Action);
}
CompletedEvent.Set();
}
public void Dispose()
{
if (ChildProcess != null)
{
ChildProcess.Dispose();
ChildProcess = null;
}
if (ChildProcessGroup != null)
{
ChildProcessGroup.Dispose();
ChildProcessGroup = null;
}
}
public PerforceChildProcess(byte[] InputData, string Format, params object[] Args)
{
string PerforceFileName;
if (Environment.OSVersion.Platform == PlatformID.Win32NT || Environment.OSVersion.Platform == PlatformID.Win32S || Environment.OSVersion.Platform == PlatformID.Win32Windows)
{
PerforceFileName = "p4.exe";
}
else
{
PerforceFileName = File.Exists("/usr/local/bin/p4")? "/usr/local/bin/p4" : "/usr/bin/p4";
}
string FullArgumentList = "-G " + String.Format(Format, Args);
Log.TraceLog("Running {0} {1}", PerforceFileName, FullArgumentList);
ChildProcessGroup = new ManagedProcessGroup();
ChildProcess = new ManagedProcess(ChildProcessGroup, PerforceFileName, FullArgumentList, null, null, InputData, ProcessPriorityClass.Normal);
Buffer = new byte[2048];
}
public static int Execute(string ActionsFileName, int MaxProcesses, bool bStopOnErrors)
{
List <BuildAction> Actions = ReadActions(ActionsFileName);
CommandUtils.LogInformation("Building {0} {1} with {2} {3}...", Actions.Count, (Actions.Count == 1) ? "action" : "actions", MaxProcesses, (MaxProcesses == 1)? "process" : "processes");
using (LogIndentScope Indent = new LogIndentScope(" "))
{
// Create the list of things to process
List <BuildAction> QueuedActions = new List <BuildAction>();
foreach (BuildAction Action in Actions)
{
if (Action.MissingDependencyCount == 0)
{
QueuedActions.Add(Action);
}
}
// Create a job object for all the child processes
int ExitCode = 0;
string CurrentPrefix = "";
Dictionary <BuildActionExecutor, Thread> ExecutingActions = new Dictionary <BuildActionExecutor, Thread>();
List <BuildActionExecutor> CompletedActions = new List <BuildActionExecutor>();
using (ManagedProcessGroup ProcessGroup = new ManagedProcessGroup())
{
using (AutoResetEvent CompletedEvent = new AutoResetEvent(false))
{
while (QueuedActions.Count > 0 || ExecutingActions.Count > 0)
{
// Sort the actions by the number of things dependent on them
QueuedActions.Sort((A, B) => (A.TotalDependants == B.TotalDependants)? (B.SortIndex - A.SortIndex) : (B.TotalDependants - A.TotalDependants));
// Create threads up to the maximum number of actions
while (ExecutingActions.Count < MaxProcesses && QueuedActions.Count > 0)
{
BuildAction Action = QueuedActions[QueuedActions.Count - 1];
QueuedActions.RemoveAt(QueuedActions.Count - 1);
BuildActionExecutor ExecutingAction = new BuildActionExecutor(ProcessGroup, Action, CompletedEvent, CompletedActions);
Thread ExecutingThread = new Thread(() => { ExecutingAction.Run(); });
ExecutingThread.Name = String.Format("Build:{0}", Action.Caption);
ExecutingThread.Start();
ExecutingActions.Add(ExecutingAction, ExecutingThread);
}
// Wait for something to finish
CompletedEvent.WaitOne();
// Wait for something to finish and flush it to the log
lock (CompletedActions)
{
foreach (BuildActionExecutor CompletedAction in CompletedActions)
{
// Join the thread
Thread CompletedThread = ExecutingActions[CompletedAction];
CompletedThread.Join();
ExecutingActions.Remove(CompletedAction);
// Write it to the log
if (CompletedAction.LogLines.Count > 0)
{
if (CurrentPrefix != CompletedAction.Action.GroupPrefix)
{
CurrentPrefix = CompletedAction.Action.GroupPrefix;
CommandUtils.LogInformation(CurrentPrefix);
}
foreach (string LogLine in CompletedAction.LogLines)
{
CommandUtils.LogInformation(LogLine);
}
}
// Check the exit code
if (CompletedAction.ExitCode == 0)
{
// Mark all the dependents as done
foreach (BuildAction DependantAction in CompletedAction.Action.Dependants)
{
if (--DependantAction.MissingDependencyCount == 0)
{
QueuedActions.Add(DependantAction);
}
}
}
else
{
// Update the exit code if it's not already set
if (ExitCode == 0)
{
ExitCode = CompletedAction.ExitCode;
}
}
}
CompletedActions.Clear();
}
// If we've already got a non-zero exit code, clear out the list of queued actions so nothing else will run
if (ExitCode != 0 && bStopOnErrors)
{
QueuedActions.Clear();
}
}
}
}
return(ExitCode);
}
}
/// <summary>
/// Executes the specified actions locally.
/// </summary>
/// <returns>True if all the tasks successfully executed, or false if any of them failed.</returns>
public override bool ExecuteActions(List <Action> InputActions, bool bLogDetailedActionStats)
{
// Figure out how many processors to use
int MaxProcesses = Math.Min((int)(Environment.ProcessorCount * ProcessorCountMultiplier), MaxProcessorCount);
Log.TraceInformation("Building {0} {1} with {2} {3}...", InputActions.Count, (InputActions.Count == 1) ? "action" : "actions", MaxProcesses, (MaxProcesses == 1)? "process" : "processes");
// Create actions with all our internal metadata
List <BuildAction> Actions = new List <BuildAction>();
for (int Idx = 0; Idx < InputActions.Count; Idx++)
{
BuildAction Action = new BuildAction();
Action.SortIndex = Idx;
Action.Inner = InputActions[Idx];
Actions.Add(Action);
}
// Build a map of items to their producing actions
Dictionary <FileItem, BuildAction> FileToProducingAction = new Dictionary <FileItem, BuildAction>();
foreach (BuildAction Action in Actions)
{
foreach (FileItem ProducedItem in Action.Inner.ProducedItems)
{
FileToProducingAction[ProducedItem] = Action;
}
}
// Update all the actions with all their dependencies
foreach (BuildAction Action in Actions)
{
foreach (FileItem PrerequisiteItem in Action.Inner.PrerequisiteItems)
{
BuildAction Dependency;
if (FileToProducingAction.TryGetValue(PrerequisiteItem, out Dependency))
{
Action.Dependencies.Add(Dependency);
Dependency.Dependants.Add(Action);
}
}
}
// Figure out the recursive dependency count
HashSet <BuildAction> VisitedActions = new HashSet <BuildAction>();
foreach (BuildAction Action in Actions)
{
Action.MissingDependencyCount = Action.Dependencies.Count;
RecursiveIncDependents(Action, VisitedActions);
}
// Create the list of things to process
List <BuildAction> QueuedActions = new List <BuildAction>();
foreach (BuildAction Action in Actions)
{
if (Action.MissingDependencyCount == 0)
{
QueuedActions.Add(Action);
}
}
// Execute the actions
using (ScopedLogIndent Indent = new ScopedLogIndent(" "))
{
// Create a job object for all the child processes
bool bResult = true;
Dictionary <BuildAction, Thread> ExecutingActions = new Dictionary <BuildAction, Thread>();
List <BuildAction> CompletedActions = new List <BuildAction>();
using (ManagedProcessGroup ProcessGroup = new ManagedProcessGroup())
{
using (AutoResetEvent CompletedEvent = new AutoResetEvent(false))
{
int NumCompletedActions = 0;
using (ProgressWriter ProgressWriter = new ProgressWriter("Compiling C++ source code...", false))
{
while (QueuedActions.Count > 0 || ExecutingActions.Count > 0)
{
// Sort the actions by the number of things dependent on them
QueuedActions.Sort((A, B) => (A.TotalDependantCount == B.TotalDependantCount)? (B.SortIndex - A.SortIndex) : (B.TotalDependantCount - A.TotalDependantCount));
// Create threads up to the maximum number of actions
while (ExecutingActions.Count < MaxProcesses && QueuedActions.Count > 0)
{
BuildAction Action = QueuedActions[QueuedActions.Count - 1];
QueuedActions.RemoveAt(QueuedActions.Count - 1);
Thread ExecutingThread = new Thread(() => { ExecuteAction(ProcessGroup, Action, CompletedActions, CompletedEvent); });
ExecutingThread.Name = String.Format("Build:{0}", Action.Inner.StatusDescription);
ExecutingThread.Start();
ExecutingActions.Add(Action, ExecutingThread);
}
// Wait for something to finish
CompletedEvent.WaitOne();
// Wait for something to finish and flush it to the log
lock (CompletedActions)
{
foreach (BuildAction CompletedAction in CompletedActions)
{
// Join the thread
Thread CompletedThread = ExecutingActions[CompletedAction];
CompletedThread.Join();
ExecutingActions.Remove(CompletedAction);
// Update the progress
NumCompletedActions++;
ProgressWriter.Write(NumCompletedActions, InputActions.Count);
// Write it to the log
if (CompletedAction.LogLines.Count > 0)
{
Log.TraceInformation("[{0}/{1}] {2}", NumCompletedActions, InputActions.Count, CompletedAction.LogLines[0]);
for (int LineIdx = 1; LineIdx < CompletedAction.LogLines.Count; LineIdx++)
{
Log.TraceInformation("{0}", CompletedAction.LogLines[LineIdx]);
}
}
// Check the exit code
if (CompletedAction.ExitCode == 0)
{
// Mark all the dependents as done
foreach (BuildAction DependantAction in CompletedAction.Dependants)
{
if (--DependantAction.MissingDependencyCount == 0)
{
QueuedActions.Add(DependantAction);
}
}
}
else
{
// Update the exit code if it's not already set
if (bResult && CompletedAction.ExitCode != 0)
{
bResult = false;
}
}
}
CompletedActions.Clear();
}
// If we've already got a non-zero exit code, clear out the list of queued actions so nothing else will run
if (!bResult && bStopCompilationAfterErrors)
{
QueuedActions.Clear();
}
}
}
}
}
return(bResult);
}
}
