public static bool ExecuteActions(List <Action> Actions) { bool bDistccResult = true; if (Actions.Count > 0) { // Time to sleep after each iteration of the loop in order to not busy wait. const float LoopSleepTime = 0.1f; int MaxActionsToExecuteInParallel = 0; string UserDir = Environment.GetEnvironmentVariable("HOME"); string HostsInfo = UserDir + "/.dmucs/hosts-info"; System.IO.StreamReader File = new System.IO.StreamReader(HostsInfo); string Line = null; while ((Line = File.ReadLine()) != null) { var HostInfo = Line.Split(' '); if (HostInfo.Count() == 3) { int NumCPUs = 0; if (System.Int32.TryParse(HostInfo [1], out NumCPUs)) { MaxActionsToExecuteInParallel += NumCPUs; } } } File.Close(); if (BuildConfiguration.bAllowDistccLocalFallback == false) { Environment.SetEnvironmentVariable("DISTCC_FALLBACK", "0"); } string DistccExecutable = BuildConfiguration.DistccExecutablesPath + "/distcc"; string GetHostExecutable = BuildConfiguration.DistccExecutablesPath + "/gethost"; Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel, DistccExecutable, GetHostExecutable); Dictionary <Action, ActionThread> ActionThreadDictionary = new Dictionary <Action, ActionThread>(); int JobNumber = 1; using (ProgressWriter ProgressWriter = new ProgressWriter("Compiling source code...", false)) { int ProgressValue = 0; while (true) { // Count the number of pending and still executing actions. int NumUnexecutedActions = 0; int NumExecutingActions = 0; foreach (Action Action in Actions) { ActionThread ActionThread = null; bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread); if (bFoundActionProcess == false) { NumUnexecutedActions++; } else if (ActionThread != null) { if (ActionThread.bComplete == false) { NumUnexecutedActions++; NumExecutingActions++; } } } // Update the current progress int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions; if (ProgressValue != NewProgressValue) { ProgressWriter.Write(ProgressValue, Actions.Count + 1); ProgressValue = NewProgressValue; } // If there aren't any pending actions left, we're done executing. if (NumUnexecutedActions == 0) { break; } // If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated // prerequisites. foreach (Action Action in Actions) { ActionThread ActionProcess = null; bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess); if (bFoundActionProcess == false) { if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel)) { // Determine whether there are any prerequisites of the action that are outdated. bool bHasOutdatedPrerequisites = false; bool bHasFailedPrerequisites = false; foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems) { if (PrerequisiteItem.ProducingAction != null && Actions.Contains(PrerequisiteItem.ProducingAction)) { ActionThread PrerequisiteProcess = null; bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue(PrerequisiteItem.ProducingAction, out PrerequisiteProcess); if (bFoundPrerequisiteProcess == true) { if (PrerequisiteProcess == null) { bHasFailedPrerequisites = true; } else if (PrerequisiteProcess.bComplete == false) { bHasOutdatedPrerequisites = true; } else if (PrerequisiteProcess.ExitCode != 0) { bHasFailedPrerequisites = true; } } else { bHasOutdatedPrerequisites = true; } } } // If there are any failed prerequisites of this action, don't execute it. if (bHasFailedPrerequisites) { // Add a null entry in the dictionary for this action. ActionThreadDictionary.Add(Action, null); } // If there aren't any outdated prerequisites of this action, execute it. else if (!bHasOutdatedPrerequisites) { if ((Action.ActionType == ActionType.Compile || Action.ActionType == ActionType.Link) && DistccExecutable != null && GetHostExecutable != null) { string NewCommandArguments = "--wait -1 \"" + DistccExecutable + "\" " + Action.CommandPath + " " + Action.CommandArguments; Action.CommandPath = GetHostExecutable; Action.CommandArguments = NewCommandArguments; } ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count); JobNumber++; ActionThread.Run(); ActionThreadDictionary.Add(Action, ActionThread); NumExecutingActions++; } } } } System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime)); } } Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "-------- Begin Detailed Action Stats ----------------------------------------------------------"); Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "^Action Type^Duration (seconds)^Tool^Task^Using PCH"); double TotalThreadSeconds = 0; // Check whether any of the tasks failed and log action stats if wanted. foreach (KeyValuePair <Action, ActionThread> ActionProcess in ActionThreadDictionary) { Action Action = ActionProcess.Key; ActionThread ActionThread = ActionProcess.Value; // Check for pending actions, preemptive failure if (ActionThread == null) { bDistccResult = false; continue; } // Check for executed action but general failure if (ActionThread.ExitCode != 0) { bDistccResult = false; } // Log CPU time, tool and task. double ThreadSeconds = Action.Duration.TotalSeconds; Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "^{0}^{1:0.00}^{2}^{3}^{4}", Action.ActionType.ToString(), ThreadSeconds, Path.GetFileName(Action.CommandPath), Action.StatusDescription, Action.bIsUsingPCH); // Update statistics switch (Action.ActionType) { case ActionType.BuildProject: UnrealBuildTool.TotalBuildProjectTime += ThreadSeconds; break; case ActionType.Compile: UnrealBuildTool.TotalCompileTime += ThreadSeconds; break; case ActionType.CreateAppBundle: UnrealBuildTool.TotalCreateAppBundleTime += ThreadSeconds; break; case ActionType.GenerateDebugInfo: UnrealBuildTool.TotalGenerateDebugInfoTime += ThreadSeconds; break; case ActionType.Link: UnrealBuildTool.TotalLinkTime += ThreadSeconds; break; default: UnrealBuildTool.TotalOtherActionsTime += ThreadSeconds; break; } // Keep track of total thread seconds spent on tasks. TotalThreadSeconds += ThreadSeconds; } Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats || BuildConfiguration.bPrintDebugInfo, TraceEventType.Information, "-------- End Detailed Actions Stats -----------------------------------------------------------"); // Log total CPU seconds and numbers of processors involved in tasks. Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats || BuildConfiguration.bPrintDebugInfo, TraceEventType.Information, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds); } return(bDistccResult); }
/// <summary> /// Executes the specified actions locally. /// </summary> /// <returns>True if all the tasks successfully executed, or false if any of them failed.</returns> public override bool ExecuteActions(List <Action> InputActions, bool bLogDetailedActionStats) { // Figure out how many processors to use int MaxProcesses = Math.Min((int)(Environment.ProcessorCount * ProcessorCountMultiplier), MaxProcessorCount); Log.TraceInformation("Building {0} {1} with {2} {3}...", InputActions.Count, (InputActions.Count == 1) ? "action" : "actions", MaxProcesses, (MaxProcesses == 1)? "process" : "processes"); // Create actions with all our internal metadata List <BuildAction> Actions = new List <BuildAction>(); for (int Idx = 0; Idx < InputActions.Count; Idx++) { BuildAction Action = new BuildAction(); Action.SortIndex = Idx; Action.Inner = InputActions[Idx]; Actions.Add(Action); } // Build a map of items to their producing actions Dictionary <FileItem, BuildAction> FileToProducingAction = new Dictionary <FileItem, BuildAction>(); foreach (BuildAction Action in Actions) { foreach (FileItem ProducedItem in Action.Inner.ProducedItems) { FileToProducingAction[ProducedItem] = Action; } } // Update all the actions with all their dependencies foreach (BuildAction Action in Actions) { foreach (FileItem PrerequisiteItem in Action.Inner.PrerequisiteItems) { BuildAction Dependency; if (FileToProducingAction.TryGetValue(PrerequisiteItem, out Dependency)) { Action.Dependencies.Add(Dependency); Dependency.Dependants.Add(Action); } } } // Figure out the recursive dependency count HashSet <BuildAction> VisitedActions = new HashSet <BuildAction>(); foreach (BuildAction Action in Actions) { Action.MissingDependencyCount = Action.Dependencies.Count; RecursiveIncDependents(Action, VisitedActions); } // Create the list of things to process List <BuildAction> QueuedActions = new List <BuildAction>(); foreach (BuildAction Action in Actions) { if (Action.MissingDependencyCount == 0) { QueuedActions.Add(Action); } } // Execute the actions using (ScopedLogIndent Indent = new ScopedLogIndent(" ")) { // Create a job object for all the child processes bool bResult = true; Dictionary <BuildAction, Thread> ExecutingActions = new Dictionary <BuildAction, Thread>(); List <BuildAction> CompletedActions = new List <BuildAction>(); using (ManagedProcessGroup ProcessGroup = new ManagedProcessGroup()) { using (AutoResetEvent CompletedEvent = new AutoResetEvent(false)) { int NumCompletedActions = 0; using (ProgressWriter ProgressWriter = new ProgressWriter("Compiling C++ source code...", false)) { while (QueuedActions.Count > 0 || ExecutingActions.Count > 0) { // Sort the actions by the number of things dependent on them QueuedActions.Sort((A, B) => (A.TotalDependantCount == B.TotalDependantCount)? (B.SortIndex - A.SortIndex) : (B.TotalDependantCount - A.TotalDependantCount)); // Create threads up to the maximum number of actions while (ExecutingActions.Count < MaxProcesses && QueuedActions.Count > 0) { BuildAction Action = QueuedActions[QueuedActions.Count - 1]; QueuedActions.RemoveAt(QueuedActions.Count - 1); Thread ExecutingThread = new Thread(() => { ExecuteAction(ProcessGroup, Action, CompletedActions, CompletedEvent); }); ExecutingThread.Name = String.Format("Build:{0}", Action.Inner.StatusDescription); ExecutingThread.Start(); ExecutingActions.Add(Action, ExecutingThread); } // Wait for something to finish CompletedEvent.WaitOne(); // Wait for something to finish and flush it to the log lock (CompletedActions) { foreach (BuildAction CompletedAction in CompletedActions) { // Join the thread Thread CompletedThread = ExecutingActions[CompletedAction]; CompletedThread.Join(); ExecutingActions.Remove(CompletedAction); // Write it to the log if (CompletedAction.LogLines.Count > 0) { foreach (string LogLine in CompletedAction.LogLines) { Log.TraceInformation(LogLine); } } // Update the progress NumCompletedActions++; ProgressWriter.Write(NumCompletedActions, InputActions.Count); // Check the exit code if (CompletedAction.ExitCode == 0) { // Mark all the dependents as done foreach (BuildAction DependantAction in CompletedAction.Dependants) { if (--DependantAction.MissingDependencyCount == 0) { QueuedActions.Add(DependantAction); } } } else { // Update the exit code if it's not already set if (bResult && CompletedAction.ExitCode != 0) { bResult = false; } } } CompletedActions.Clear(); } // If we've already got a non-zero exit code, clear out the list of queued actions so nothing else will run if (!bResult && bStopCompilationAfterErrors) { QueuedActions.Clear(); } } } } } return(bResult); } }