public override bool ExecuteActions(List <Action> Actions, bool bLogDetailedActionStats)
{
bool SNDBSResult = true;
if (Actions.Count > 0)
{
// Generate any needed templates. Can only generate one per executable, so just use the first Action for reference
IEnumerable <Action> CommandPaths = Actions.GroupBy(a => a.CommandPath).Select(g => g.FirstOrDefault()).ToList();
foreach (Action CommandPath in CommandPaths)
{
PrepareToolTemplate(CommandPath);
}
// Generate include-rewrite-rules.ini.
GenerateSNDBSIncludeRewriteRules();
// Use WMI to figure out physical cores, excluding hyper threading.
int NumCores = Utils.GetPhysicalProcessorCount();
// If we failed to detect the number of cores, default to the logical processor count
if (NumCores == -1)
{
NumCores = System.Environment.ProcessorCount;
}
// The number of actions to execute in parallel is trying to keep the CPU busy enough in presence of I/O stalls.
MaxActionsToExecuteInParallel = 0;
// The CPU has more logical cores than physical ones, aka uses hyper-threading.
if (NumCores < System.Environment.ProcessorCount)
{
MaxActionsToExecuteInParallel = (int)(NumCores * ProcessorCountMultiplier);
}
// No hyper-threading. Only kicking off a task per CPU to keep machine responsive.
else
{
MaxActionsToExecuteInParallel = NumCores;
}
MaxActionsToExecuteInParallel = Math.Min(MaxActionsToExecuteInParallel, MaxProcessorCount);
JobNumber = 1;
Dictionary <Action, ActionThread> ActionThreadDictionary = new Dictionary <Action, ActionThread>();
while (true)
{
bool bUnexecutedActions = false;
foreach (Action Action in Actions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
bUnexecutedActions = true;
if (!ExecuteActions(Actions, ActionThreadDictionary))
{
return(false);
}
break;
}
}
if (bUnexecutedActions == false)
{
break;
}
}
Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "-------- Begin Detailed Action Stats ----------------------------------------------------------");
Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "^Action Type^Duration (seconds)^Tool^Task^Using PCH");
double TotalThreadSeconds = 0;
// Check whether any of the tasks failed and log action stats if wanted.
foreach (KeyValuePair <Action, ActionThread> ActionProcess in ActionThreadDictionary)
{
Action Action = ActionProcess.Key;
ActionThread ActionThread = ActionProcess.Value;
// Check for pending actions, preemptive failure
if (ActionThread == null)
{
SNDBSResult = false;
continue;
}
// Check for executed action but general failure
if (ActionThread.ExitCode != 0)
{
SNDBSResult = false;
}
// Log CPU time, tool and task.
double ThreadSeconds = Action.Duration.TotalSeconds;
Log.WriteLineIf(bLogDetailedActionStats,
LogEventType.Console,
"^{0}^{1:0.00}^{2}^{3}",
Action.ActionType.ToString(),
ThreadSeconds,
Action.CommandPath.GetFileName(),
Action.StatusDescription);
// Keep track of total thread seconds spent on tasks.
TotalThreadSeconds += ThreadSeconds;
}
Log.TraceInformation("-------- End Detailed Actions Stats -----------------------------------------------------------");
// Log total CPU seconds and numbers of processors involved in tasks.
Log.WriteLineIf(bLogDetailedActionStats,
LogEventType.Console, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds);
}
// Delete the include-rewrite-rules.ini if it was generated.
if (File.Exists(IncludeRewriteRulesFile.FullName))
{
File.Delete(IncludeRewriteRulesFile.FullName);
}
return(SNDBSResult);
}
internal bool ExecuteActions(List <Action> InActions, Dictionary <Action, ActionThread> InActionThreadDictionary)
{
// Build the script file that will be executed by SN-DBS
StreamWriter ScriptFile;
string ScriptFilename = Path.Combine(UnrealBuildTool.EngineDirectory.FullName, "Intermediate", "Build", "SNDBS.bat");
FileStream ScriptFileStream = new FileStream(ScriptFilename, FileMode.Create, FileAccess.ReadWrite, FileShare.Read);
ScriptFile = new StreamWriter(ScriptFileStream);
ScriptFile.AutoFlush = true;
int NumScriptedActions = 0;
List <Action> LocalActions = new List <Action>();
ActionThread DummyActionThread = new ActionThread(null, 1, 1);
foreach (Action Action in InActions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = InActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems)
{
if (PrerequisiteItem.ProducingAction != null && InActions.Contains(PrerequisiteItem.ProducingAction))
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = InActionThreadDictionary.TryGetValue(PrerequisiteItem.ProducingAction, out PrerequisiteProcess);
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
InActionThreadDictionary.Add(Action, null);
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
if (Action.bCanExecuteRemotely == false || Action.bCanExecuteRemotelyWithSNDBS == false)
{
// Execute locally
LocalActions.Add(Action);
}
else
{
// Add to script for execution by SN-DBS
string NewCommandArguments = "\"" + Action.CommandPath + "\"" + " " + Action.CommandArguments;
ScriptFile.WriteLine(ActionThread.ExpandEnvironmentVariables(NewCommandArguments));
InActionThreadDictionary.Add(Action, DummyActionThread);
Action.StartTime = Action.EndTime = DateTimeOffset.Now;
Log.TraceInformation("[{0}/{1}] {2} {3}", JobNumber, InActions.Count, Action.CommandDescription, Action.StatusDescription);
JobNumber++;
NumScriptedActions++;
}
}
}
}
ScriptFile.Flush();
ScriptFile.Close();
ScriptFile.Dispose();
ScriptFile = null;
if (NumScriptedActions > 0)
{
// Create the process
string SCERoot = Environment.GetEnvironmentVariable("SCE_ROOT_DIR");
string SNDBSExecutable = Path.Combine(SCERoot, "Common/SN-DBS/bin/dbsbuild.exe");
ProcessStartInfo PSI = new ProcessStartInfo(SNDBSExecutable, String.Format("-q -p UE4 -s \"{0}\"", FileReference.Combine(UnrealBuildTool.EngineDirectory, "Intermediate", "Build", "sndbs.bat").FullName));
PSI.RedirectStandardOutput = true;
PSI.RedirectStandardError = true;
PSI.UseShellExecute = false;
PSI.CreateNoWindow = true;
PSI.WorkingDirectory = Path.GetFullPath(".");;
Process NewProcess = new Process();
NewProcess.StartInfo = PSI;
NewProcess.OutputDataReceived += new DataReceivedEventHandler(ActionDebugOutput);
NewProcess.ErrorDataReceived += new DataReceivedEventHandler(ActionDebugOutput);
DateTimeOffset StartTime = DateTimeOffset.Now;
NewProcess.Start();
NewProcess.BeginOutputReadLine();
NewProcess.BeginErrorReadLine();
NewProcess.WaitForExit();
TimeSpan Duration;
DateTimeOffset EndTime = DateTimeOffset.Now;
if (EndTime == DateTimeOffset.MinValue)
{
Duration = DateTimeOffset.Now - StartTime;
}
else
{
Duration = EndTime - StartTime;
}
DummyActionThread.bComplete = true;
int ExitCode = NewProcess.ExitCode;
if (ExitCode != 0)
{
return(false);
}
}
// Execute local tasks
if (LocalActions.Count > 0)
{
return(ExecuteLocalActions(LocalActions, InActionThreadDictionary, InActions.Count));
}
return(true);
}
internal bool ExecuteActions(List <Action> InActions, Dictionary <Action, ActionThread> InActionThreadDictionary)
{
// Build the script file that will be executed by SN-DBS
StreamWriter ScriptFile;
string ScriptFilename = Path.Combine(UnrealBuildTool.EngineDirectory.FullName, "Intermediate", "Build", "SNDBS.bat");
FileStream ScriptFileStream = new FileStream(ScriptFilename, FileMode.Create, FileAccess.ReadWrite, FileShare.Read);
ScriptFile = new StreamWriter(ScriptFileStream);
ScriptFile.AutoFlush = true;
int NumScriptedActions = 0;
List <Action> LocalActions = new List <Action>();
ActionThread DummyActionThread = new ActionThread(null, 1, 1);
bool PrintDebugInfo = false;
foreach (Action Action in InActions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = InActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (Action PrerequisiteAction in Action.PrerequisiteActions)
{
if (InActions.Contains(PrerequisiteAction))
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = InActionThreadDictionary.TryGetValue(PrerequisiteAction, out PrerequisiteProcess);
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
InActionThreadDictionary.Add(Action, null);
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
if (Action.bCanExecuteRemotely == false || Action.bCanExecuteRemotelyWithSNDBS == false)
{
// Execute locally
LocalActions.Add(Action);
}
else
{
// Create a dummy force-included file which references PCH files, so that SN-DBS knows they are dependencies.
string AdditionalStubIncludes = "";
if (Action.CommandPath.GetFileName().Equals("cl.exe", StringComparison.OrdinalIgnoreCase))
{
string ResponseFile = Action.CommandArguments.Replace("\"", "").Replace("@", "").Trim();
StringBuilder WrapperContents = new StringBuilder();
using (StringWriter Writer = new StringWriter(WrapperContents))
{
Writer.WriteLine("// PCH dependencies for {0}", ResponseFile);
Writer.WriteLine("#if 0");
foreach (FileItem Preqrequisite in Action.PrerequisiteItems)
{
if (Preqrequisite.AbsolutePath.EndsWith(".pch"))
{
Writer.WriteLine("#include \"{0}\"", Preqrequisite.AbsolutePath.Replace(".pch", ".obj"));
}
}
Writer.WriteLine("#endif");
}
FileItem DummyResponseFileDependency = FileItem.CreateIntermediateTextFile(new FileReference(ResponseFile + ".dummy.h"), WrapperContents.ToString());
AdditionalStubIncludes = string.Format("/FI\"{0}\"", DummyResponseFileDependency);
}
// Add to script for execution by SN-DBS
string NewCommandArguments = "\"" + Action.CommandPath + "\"" + " " + AdditionalStubIncludes + " " + Action.CommandArguments;
ScriptFile.WriteLine(NewCommandArguments);
InActionThreadDictionary.Add(Action, DummyActionThread);
Action.StartTime = Action.EndTime = DateTimeOffset.Now;
Log.TraceInformation("[{0}/{1}] {2} {3}", JobNumber, InActions.Count, Action.CommandDescription, Action.StatusDescription);
JobNumber++;
NumScriptedActions++;
PrintDebugInfo |= Action.bPrintDebugInfo;
if (Action.DependencyListFile != null && File.Exists(Action.DependencyListFile.AbsolutePath))
{
Log.TraceVerbose("Deleting dependency list file {0}", Action.DependencyListFile.AbsolutePath);
File.Delete(Action.DependencyListFile.AbsolutePath);
}
}
}
}
}
ScriptFile.Flush();
ScriptFile.Close();
ScriptFile.Dispose();
ScriptFile = null;
if (NumScriptedActions > 0)
{
// Create the process
string SCERoot = Environment.GetEnvironmentVariable("SCE_ROOT_DIR");
string SNDBSExecutable = Path.Combine(SCERoot, "Common/SN-DBS/bin/dbsbuild.exe");
DirectoryReference TemplatesDir = DirectoryReference.Combine(UnrealBuildTool.EngineDirectory, "Programs", "UnrealBuildTool", "SndbsTemplates");
string IncludeRewriteRulesArg = String.Format("--include-rewrite-rules \"{0}\"", IncludeRewriteRulesFile.FullName);
string VerbosityLevel = PrintDebugInfo ? "-v" : "-q";
ProcessStartInfo PSI = new ProcessStartInfo(SNDBSExecutable, String.Format("{0} -p UE4 -s \"{1}\" -templates \"{2}\" {3}", VerbosityLevel, FileReference.Combine(UnrealBuildTool.EngineDirectory, "Intermediate", "Build", "sndbs.bat").FullName, TemplatesDir.FullName, IncludeRewriteRulesArg));
PSI.RedirectStandardOutput = true;
PSI.RedirectStandardError = true;
PSI.UseShellExecute = false;
PSI.CreateNoWindow = true;
PSI.WorkingDirectory = Path.GetFullPath(".");
Process NewProcess = new Process();
NewProcess.StartInfo = PSI;
NewProcess.OutputDataReceived += new DataReceivedEventHandler(ActionDebugOutput);
NewProcess.ErrorDataReceived += new DataReceivedEventHandler(ActionDebugOutput);
DateTimeOffset StartTime = DateTimeOffset.Now;
NewProcess.Start();
NewProcess.BeginOutputReadLine();
NewProcess.BeginErrorReadLine();
NewProcess.WaitForExit();
TimeSpan Duration;
DateTimeOffset EndTime = DateTimeOffset.Now;
if (EndTime == DateTimeOffset.MinValue)
{
Duration = DateTimeOffset.Now - StartTime;
}
else
{
Duration = EndTime - StartTime;
}
DummyActionThread.bComplete = true;
int ExitCode = NewProcess.ExitCode;
if (ExitCode != 0)
{
return(false);
}
}
// Execute local tasks
if (LocalActions.Count > 0)
{
return(ExecuteLocalActions(LocalActions, InActionThreadDictionary, InActions.Count));
}
return(true);
}
public override FileItem[] LinkAllFiles(LinkEnvironment LinkEnvironment, bool bBuildImportLibraryOnly)
{
List<FileItem> Outputs = new List<FileItem>();
var NDKRoot = Environment.GetEnvironmentVariable("NDKROOT").Replace("\\", "/");
int NDKApiLevelInt = GetNdkApiLevelInt();
string OptionalLinkArguments;
for (int ArchIndex = 0; ArchIndex < Arches.Count; ArchIndex++)
{
string Arch = Arches[ArchIndex];
// 32-bit ABI may need fixup for removed bsd_signal in NDK11 for android-21+
OptionalLinkArguments = "";
if (NDKApiLevelInt >= 21)
{
// this file was added in NDK11 so use existence to detect (RELEASE.TXT no longer present)
if (File.Exists(Path.Combine(NDKRoot, "source.properties")))
{
switch (Arch)
{
case "-armv7":
OptionalLinkArguments = string.Format(" \"{0}\"", Path.Combine(UnrealBuildTool.EngineDirectory.FullName, "Build/Android/Prebuilt/bsdsignal/lib/armeabi-v7a/libbsdsignal.a"));
break;
case "-x86":
OptionalLinkArguments = string.Format(" \"{0}\"", Path.Combine(UnrealBuildTool.EngineDirectory.FullName, "Build/Android/Prebuilt/bsdsignal/lib/x86/libbsdsignal.a"));
break;
}
}
}
for (int GPUArchIndex = 0; GPUArchIndex < GPUArchitectures.Count; GPUArchIndex++)
{
string GPUArchitecture = GPUArchitectures[GPUArchIndex];
int OutputPathIndex = ArchIndex * GPUArchitectures.Count + GPUArchIndex;
// Android will have an array of outputs
if (LinkEnvironment.Config.OutputFilePaths.Count < OutputPathIndex ||
!LinkEnvironment.Config.OutputFilePaths[OutputPathIndex].GetFileNameWithoutExtension().EndsWith(Arch + GPUArchitecture))
{
throw new BuildException("The OutputFilePaths array didn't match the Arches array in AndroidToolChain.LinkAllFiles");
}
// Create an action that invokes the linker.
Action LinkAction = new Action(ActionType.Link);
LinkAction.WorkingDirectory = UnrealBuildTool.EngineSourceDirectory.FullName;
if (LinkEnvironment.Config.bIsBuildingLibrary)
{
switch (Arch)
{
case "-armv7": LinkAction.CommandPath = ArPathArm; break;
case "-arm64": LinkAction.CommandPath = ArPathArm64; break;
case "-x86": LinkAction.CommandPath = ArPathx86; ; break;
case "-x64": LinkAction.CommandPath = ArPathx64; ; break;
default: LinkAction.CommandPath = ArPathArm; ; break;
}
}
else
{
LinkAction.CommandPath = ClangPath;
}
string LinkerPath = LinkAction.WorkingDirectory;
LinkAction.WorkingDirectory = LinkEnvironment.Config.IntermediateDirectory.FullName;
// Get link arguments.
LinkAction.CommandArguments = LinkEnvironment.Config.bIsBuildingLibrary ? GetArArguments(LinkEnvironment) : GetLinkArguments(LinkEnvironment, Arch);
// Add the output file as a production of the link action.
FileItem OutputFile;
OutputFile = FileItem.GetItemByFileReference(LinkEnvironment.Config.OutputFilePaths[OutputPathIndex]);
Outputs.Add(OutputFile);
LinkAction.ProducedItems.Add(OutputFile);
LinkAction.StatusDescription = string.Format("{0}", Path.GetFileName(OutputFile.AbsolutePath));
// LinkAction.bPrintDebugInfo = true;
// Add the output file to the command-line.
if (LinkEnvironment.Config.bIsBuildingLibrary)
{
LinkAction.CommandArguments += string.Format(" \"{0}\"", OutputFile.AbsolutePath);
}
else
{
LinkAction.CommandArguments += string.Format(" -o \"{0}\"", OutputFile.AbsolutePath);
}
// Add the input files to a response file, and pass the response file on the command-line.
List<string> InputFileNames = new List<string>();
foreach (FileItem InputFile in LinkEnvironment.InputFiles)
{
// make sure it's for current Arch
if (Path.GetFileNameWithoutExtension(InputFile.AbsolutePath).EndsWith(Arch + GPUArchitecture))
{
string AbsolutePath = InputFile.AbsolutePath.Replace("\\", "/");
AbsolutePath = AbsolutePath.Replace(LinkEnvironment.Config.IntermediateDirectory.FullName.Replace("\\", "/"), "");
AbsolutePath = AbsolutePath.TrimStart(new char[] { '/' });
InputFileNames.Add(string.Format("\"{0}\"", AbsolutePath));
LinkAction.PrerequisiteItems.Add(InputFile);
}
}
FileReference ResponseFileName = GetResponseFileName(LinkEnvironment, OutputFile);
LinkAction.CommandArguments += string.Format(" @\"{0}\"", ResponseFile.Create(ResponseFileName, InputFileNames));
// libs don't link in other libs
if (!LinkEnvironment.Config.bIsBuildingLibrary)
{
// Add the library paths to the argument list.
foreach (string LibraryPath in LinkEnvironment.Config.LibraryPaths)
{
// LinkerPaths could be relative or absolute
string AbsoluteLibraryPath = ActionThread.ExpandEnvironmentVariables(LibraryPath);
if (IsDirectoryForArch(AbsoluteLibraryPath, Arch))
{
// environment variables aren't expanded when using the $( style
if (Path.IsPathRooted(AbsoluteLibraryPath) == false)
{
AbsoluteLibraryPath = Path.Combine(LinkerPath, AbsoluteLibraryPath);
}
LinkAction.CommandArguments += string.Format(" -L\"{0}\"", AbsoluteLibraryPath);
}
}
// add libraries in a library group
LinkAction.CommandArguments += string.Format(" -Wl,--start-group");
foreach (string AdditionalLibrary in LinkEnvironment.Config.AdditionalLibraries)
{
if (!ShouldSkipLib(AdditionalLibrary, Arch, GPUArchitecture))
{
if (String.IsNullOrEmpty(Path.GetDirectoryName(AdditionalLibrary)))
{
LinkAction.CommandArguments += string.Format(" \"-l{0}\"", AdditionalLibrary);
}
else
{
// full pathed libs are compiled by us, so we depend on linking them
LinkAction.CommandArguments += string.Format(" \"{0}\"", Path.GetFullPath(AdditionalLibrary));
LinkAction.PrerequisiteItems.Add(FileItem.GetItemByPath(AdditionalLibrary));
}
}
}
LinkAction.CommandArguments += OptionalLinkArguments;
LinkAction.CommandArguments += string.Format(" -Wl,--end-group");
}
// Add the additional arguments specified by the environment.
LinkAction.CommandArguments += LinkEnvironment.Config.AdditionalArguments;
LinkAction.CommandArguments = LinkAction.CommandArguments.Replace("\\", "/");
// Only execute linking on the local PC.
LinkAction.bCanExecuteRemotely = false;
}
}
return Outputs.ToArray();
}
public override CPPOutput CompileCPPFiles(UEBuildTarget Target, CPPEnvironment CompileEnvironment, List<FileItem> SourceFiles, string ModuleName)
{
if (Arches.Count == 0)
{
throw new BuildException("At least one architecture (armv7, x86, etc) needs to be selected in the project settings to build");
}
if (!bHasPrintedApiLevel)
{
Console.WriteLine("Compiling Native code with NDK API '{0}'", GetNdkApiLevel());
bHasPrintedApiLevel = true;
}
string BaseArguments = "";
if (CompileEnvironment.Config.PrecompiledHeaderAction != PrecompiledHeaderAction.Create)
{
BaseArguments += " -Werror";
}
// Directly added NDK files for NDK extensions
ConditionallyAddNDKSourceFiles(SourceFiles, ModuleName);
// Deal with dynamic modules removed by architecture
GenerateEmptyLinkFunctionsForRemovedModules(SourceFiles, ModuleName, CompileEnvironment.Config.OutputDirectory);
// Add preprocessor definitions to the argument list.
foreach (string Definition in CompileEnvironment.Config.Definitions)
{
BaseArguments += string.Format(" -D \"{0}\"", Definition);
}
var BuildPlatform = UEBuildPlatform.GetBuildPlatformForCPPTargetPlatform(CompileEnvironment.Config.Target.Platform);
var NDKRoot = Environment.GetEnvironmentVariable("NDKROOT").Replace("\\", "/");
string BasePCHName = "";
var PCHExtension = UEBuildPlatform.GetBuildPlatform(UnrealTargetPlatform.Android).GetBinaryExtension(UEBuildBinaryType.PrecompiledHeader);
if (CompileEnvironment.Config.PrecompiledHeaderAction == PrecompiledHeaderAction.Include)
{
BasePCHName = RemoveArchName(CompileEnvironment.PrecompiledHeaderFile.AbsolutePath).Replace(PCHExtension, "");
}
// Create a compile action for each source file.
CPPOutput Result = new CPPOutput();
foreach (string Arch in Arches)
{
if (ShouldSkipModule(ModuleName, Arch))
{
continue;
}
foreach (string GPUArchitecture in GPUArchitectures)
{
// which toolchain to use
string Arguments = GetCLArguments_Global(CompileEnvironment, Arch) + BaseArguments;
switch (Arch)
{
case "-armv7": Arguments += " -DPLATFORM_64BITS=0 -DPLATFORM_ANDROID_ARM=1"; break;
case "-arm64": Arguments += " -DPLATFORM_64BITS=1 -DPLATFORM_ANDROID_ARM64=1"; break;
case "-x86": Arguments += " -DPLATFORM_64BITS=0 -DPLATFORM_ANDROID_X86=1"; break;
case "-x64": Arguments += " -DPLATFORM_64BITS=1 -DPLATFORM_ANDROID_X64=1"; break;
default: Arguments += " -DPLATFORM_64BITS=0 -DPLATFORM_ANDROID_ARM=1"; break;
}
if (GPUArchitecture == "-esdeferred")
{
Arguments += " -DPLATFORM_ANDROIDESDEFERRED=1";
}
// which PCH file to include
string PCHArguments = "";
if (CompileEnvironment.Config.PrecompiledHeaderAction == PrecompiledHeaderAction.Include)
{
// Add the precompiled header file's path to the include path so Clang can find it.
// This needs to be before the other include paths to ensure Clang uses it instead of the source header file.
PCHArguments += string.Format(" -include \"{0}\"", InlineArchName(BasePCHName, Arch, GPUArchitecture));
}
// Add include paths to the argument list (filtered by architecture)
foreach (string IncludePath in CompileEnvironment.Config.CPPIncludeInfo.SystemIncludePaths)
{
if (IsDirectoryForArch(IncludePath, Arch))
{
Arguments += string.Format(" -I\"{0}\"", IncludePath);
}
}
foreach (string IncludePath in CompileEnvironment.Config.CPPIncludeInfo.IncludePaths)
{
if (IsDirectoryForArch(IncludePath, Arch))
{
Arguments += string.Format(" -I\"{0}\"", IncludePath);
}
}
foreach (FileItem SourceFile in SourceFiles)
{
Action CompileAction = new Action(ActionType.Compile);
string FileArguments = "";
bool bIsPlainCFile = Path.GetExtension(SourceFile.AbsolutePath).ToUpperInvariant() == ".C";
bool bDisableShadowWarning = false;
// should we disable optimizations on this file?
// @todo android - We wouldn't need this if we could disable optimizations per function (via pragma)
bool bDisableOptimizations = false;// SourceFile.AbsolutePath.ToUpperInvariant().IndexOf("\\SLATE\\") != -1;
if (bDisableOptimizations && CompileEnvironment.Config.Target.Configuration != CPPTargetConfiguration.Debug)
{
Log.TraceWarning("Disabling optimizations on {0}", SourceFile.AbsolutePath);
}
bDisableOptimizations = bDisableOptimizations || CompileEnvironment.Config.Target.Configuration == CPPTargetConfiguration.Debug;
// Add C or C++ specific compiler arguments.
if (CompileEnvironment.Config.PrecompiledHeaderAction == PrecompiledHeaderAction.Create)
{
FileArguments += GetCompileArguments_PCH(bDisableOptimizations);
}
else if (bIsPlainCFile)
{
FileArguments += GetCompileArguments_C(bDisableOptimizations);
// remove shadow variable warnings for NDK files
if (SourceFile.AbsolutePath.Replace("\\", "/").StartsWith(NDKRoot))
{
bDisableShadowWarning = true;
}
}
else
{
FileArguments += GetCompileArguments_CPP(bDisableOptimizations);
// only use PCH for .cpp files
FileArguments += PCHArguments;
}
// Add the C++ source file and its included files to the prerequisite item list.
AddPrerequisiteSourceFile(Target, BuildPlatform, CompileEnvironment, SourceFile, CompileAction.PrerequisiteItems);
if (CompileEnvironment.Config.PrecompiledHeaderAction == PrecompiledHeaderAction.Create)
{
// Add the precompiled header file to the produced item list.
FileItem PrecompiledHeaderFile = FileItem.GetItemByFileReference(
FileReference.Combine(
CompileEnvironment.Config.OutputDirectory,
Path.GetFileName(InlineArchName(SourceFile.AbsolutePath, Arch, GPUArchitecture) + PCHExtension)
)
);
CompileAction.ProducedItems.Add(PrecompiledHeaderFile);
Result.PrecompiledHeaderFile = PrecompiledHeaderFile;
// Add the parameters needed to compile the precompiled header file to the command-line.
FileArguments += string.Format(" -o \"{0}\"", PrecompiledHeaderFile.AbsolutePath, false);
}
else
{
if (CompileEnvironment.Config.PrecompiledHeaderAction == PrecompiledHeaderAction.Include)
{
CompileAction.bIsUsingPCH = true;
FileItem ArchPrecompiledHeaderFile = FileItem.GetItemByPath(InlineArchName(BasePCHName, Arch, GPUArchitecture) + PCHExtension);
CompileAction.PrerequisiteItems.Add(ArchPrecompiledHeaderFile);
}
var ObjectFileExtension = UEBuildPlatform.GetBuildPlatform(UnrealTargetPlatform.Android).GetBinaryExtension(UEBuildBinaryType.Object);
// Add the object file to the produced item list.
FileItem ObjectFile = FileItem.GetItemByFileReference(
FileReference.Combine(
CompileEnvironment.Config.OutputDirectory,
InlineArchName(Path.GetFileName(SourceFile.AbsolutePath) + ObjectFileExtension, Arch, GPUArchitecture)
)
);
CompileAction.ProducedItems.Add(ObjectFile);
Result.ObjectFiles.Add(ObjectFile);
FileArguments += string.Format(" -o \"{0}\"", ObjectFile.AbsolutePath, false);
}
// Add the source file path to the command-line.
FileArguments += string.Format(" \"{0}\"", SourceFile.AbsolutePath);
// Build a full argument list
string AllArguments = Arguments + FileArguments + CompileEnvironment.Config.AdditionalArguments;
AllArguments = ActionThread.ExpandEnvironmentVariables(AllArguments);
AllArguments = AllArguments.Replace("\\", "/");
// Remove shadow warning for this file if requested
if (bDisableShadowWarning)
{
int WarningIndex = AllArguments.IndexOf(" -Wshadow");
if (WarningIndex > 0)
{
AllArguments = AllArguments.Remove(WarningIndex, 9);
}
}
// Create the response file
FileReference ResponseFileName = CompileAction.ProducedItems[0].Reference + "_" + AllArguments.GetHashCode().ToString("X") + ".response";
string ResponseArgument = string.Format("@\"{0}\"", ResponseFile.Create(ResponseFileName, new List<string> { AllArguments }).FullName);
CompileAction.WorkingDirectory = UnrealBuildTool.EngineSourceDirectory.FullName;
CompileAction.CommandPath = ClangPath;
CompileAction.CommandArguments = ResponseArgument;
CompileAction.StatusDescription = string.Format("{0} [{1}-{2}]", Path.GetFileName(SourceFile.AbsolutePath), Arch.Replace("-", ""), GPUArchitecture.Replace("-", ""));
// VC++ always outputs the source file name being compiled, so we don't need to emit this ourselves
CompileAction.bShouldOutputStatusDescription = true;
// Don't farm out creation of pre-compiled headers as it is the critical path task.
CompileAction.bCanExecuteRemotely =
CompileEnvironment.Config.PrecompiledHeaderAction != PrecompiledHeaderAction.Create ||
BuildConfiguration.bAllowRemotelyCompiledPCHs;
}
}
}
return Result;
}
internal static ExecutionResult ExecuteActions(List<Action> InActions, Dictionary<Action, ActionThread> InActionThreadDictionary)
{
// Build the script file that will be executed by SN-DBS
StreamWriter ScriptFile;
string ScriptFilename = Path.Combine(BuildConfiguration.BaseIntermediatePath, "SNDBS.bat");
FileStream ScriptFileStream = new FileStream(ScriptFilename, FileMode.Create, FileAccess.ReadWrite, FileShare.Read);
ScriptFile = new StreamWriter(ScriptFileStream);
ScriptFile.AutoFlush = true;
int NumScriptedActions = 0;
List<Action> LocalActions = new List<Action>();
ActionThread DummyActionThread = new ActionThread(null, 1, 1);
foreach (Action Action in InActions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = InActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems)
{
if (PrerequisiteItem.ProducingAction != null && InActions.Contains(PrerequisiteItem.ProducingAction))
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = InActionThreadDictionary.TryGetValue(PrerequisiteItem.ProducingAction, out PrerequisiteProcess);
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
InActionThreadDictionary.Add(Action, null);
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
if (Action.bCanExecuteRemotely == false)
{
// Execute locally
LocalActions.Add(Action);
}
else
{
// Add to script for execution by SN-DBS
string NewCommandArguments = "\"" + Action.CommandPath + "\"" + " " + Action.CommandArguments;
ScriptFile.WriteLine(ActionThread.ExpandEnvironmentVariables(NewCommandArguments));
InActionThreadDictionary.Add(Action, DummyActionThread);
Action.StartTime = Action.EndTime = DateTimeOffset.Now;
Log.TraceInformation("[{0}/{1}] {2} {3}", JobNumber, InActions.Count, Action.CommandDescription, Action.StatusDescription);
JobNumber++;
NumScriptedActions++;
}
}
}
}
ScriptFile.Flush();
ScriptFile.Close();
ScriptFile.Dispose();
ScriptFile = null;
if( NumScriptedActions > 0 )
{
// Create the process
ProcessStartInfo PSI = new ProcessStartInfo("dbsbuild", "-q -p UE4 -s " + BuildConfiguration.BaseIntermediatePath + "/sndbs.bat");
PSI.RedirectStandardOutput = true;
PSI.RedirectStandardError = true;
PSI.UseShellExecute = false;
PSI.CreateNoWindow = true;
PSI.WorkingDirectory = Path.GetFullPath("."); ;
Process NewProcess = new Process();
NewProcess.StartInfo = PSI;
NewProcess.OutputDataReceived += new DataReceivedEventHandler(ActionDebugOutput);
NewProcess.ErrorDataReceived += new DataReceivedEventHandler(ActionDebugOutput);
DateTimeOffset StartTime = DateTimeOffset.Now;
NewProcess.Start();
NewProcess.BeginOutputReadLine();
NewProcess.BeginErrorReadLine();
NewProcess.WaitForExit();
TimeSpan Duration;
DateTimeOffset EndTime = DateTimeOffset.Now;
if (EndTime == DateTimeOffset.MinValue)
{
Duration = DateTimeOffset.Now - StartTime;
}
else
{
Duration = EndTime - StartTime;
}
DummyActionThread.bComplete = true;
int ExitCode = NewProcess.ExitCode;
if (ExitCode != 0)
{
return ExecutionResult.TasksFailed;
}
else
{
UnrealBuildTool.TotalCompileTime += Duration.TotalSeconds;
}
}
// Execute local tasks
if( LocalActions.Count > 0 )
{
return ExecuteLocalActions(LocalActions, InActionThreadDictionary, InActions.Count);
}
return ExecutionResult.TasksSucceeded;
}
/**
* Executes the specified actions locally.
* @return True if all the tasks successfully executed, or false if any of them failed.
*/
public static bool ExecuteActions(List <Action> Actions)
{
// Time to sleep after each iteration of the loop in order to not busy wait.
const float LoopSleepTime = 0.1f;
// Use WMI to figure out physical cores, excluding hyper threading.
int NumCores = 0;
if (!Utils.IsRunningOnMono)
{
try
{
using (var Mos = new System.Management.ManagementObjectSearcher("Select * from Win32_Processor"))
{
var MosCollection = Mos.Get();
foreach (var Item in MosCollection)
{
NumCores += int.Parse(Item["NumberOfCores"].ToString());
}
}
}
catch (Exception Ex)
{
Log.TraceWarning("Unable to get the number of Cores: {0}", Ex.ToString());
Log.TraceWarning("Falling back to processor count.");
}
}
// On some systems this requires a hot fix to work so we fall back to using the (logical) processor count.
if (NumCores == 0)
{
NumCores = System.Environment.ProcessorCount;
}
// The number of actions to execute in parallel is trying to keep the CPU busy enough in presence of I/O stalls.
int MaxActionsToExecuteInParallel = 0;
// The CPU has more logical cores than physical ones, aka uses hyper-threading.
if (NumCores < System.Environment.ProcessorCount)
{
MaxActionsToExecuteInParallel = (int)(NumCores * BuildConfiguration.ProcessorCountMultiplier);
}
// No hyper-threading. Only kicking off a task per CPU to keep machine responsive.
else
{
MaxActionsToExecuteInParallel = NumCores;
}
if (Utils.IsRunningOnMono)
{
// heuristic: give each action at least 1.5GB of RAM (some clang instances will need more, actually)
long MinMemoryPerActionMB = 3 * 1024 / 2;
long PhysicalRAMAvailableMB = (new PerformanceCounter("Mono Memory", "Total Physical Memory").RawValue) / (1024 * 1024);
int MaxActionsAffordedByMemory = (int)(Math.Max(1, (PhysicalRAMAvailableMB) / MinMemoryPerActionMB));
MaxActionsToExecuteInParallel = Math.Min(MaxActionsToExecuteInParallel, MaxActionsAffordedByMemory);
}
MaxActionsToExecuteInParallel = Math.Min(MaxActionsToExecuteInParallel, BuildConfiguration.MaxProcessorCount);
Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel);
Dictionary <Action, ActionThread> ActionThreadDictionary = new Dictionary <Action, ActionThread>();
int JobNumber = 1;
using (ProgressWriter ProgressWriter = new ProgressWriter("Compiling source code...", false))
{
int ProgressValue = 0;
while (true)
{
// Count the number of pending and still executing actions.
int NumUnexecutedActions = 0;
int NumExecutingActions = 0;
foreach (Action Action in Actions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
NumUnexecutedActions++;
}
else if (ActionThread != null)
{
if (ActionThread.bComplete == false)
{
NumUnexecutedActions++;
NumExecutingActions++;
}
}
}
// Update the current progress
int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions;
if (ProgressValue != NewProgressValue)
{
ProgressWriter.Write(ProgressValue, Actions.Count + 1);
ProgressValue = NewProgressValue;
}
// If there aren't any pending actions left, we're done executing.
if (NumUnexecutedActions == 0)
{
break;
}
// If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
// prerequisites.
foreach (Action Action in Actions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel))
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems)
{
if (PrerequisiteItem.ProducingAction != null && Actions.Contains(PrerequisiteItem.ProducingAction))
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue(PrerequisiteItem.ProducingAction, out PrerequisiteProcess);
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
ActionThreadDictionary.Add(Action, null);
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count);
JobNumber++;
ActionThread.Run();
ActionThreadDictionary.Add(Action, ActionThread);
NumExecutingActions++;
}
}
}
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime));
}
}
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "-------- Begin Detailed Action Stats ----------------------------------------------------------");
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "^Action Type^Duration (seconds)^Tool^Task^Using PCH");
double TotalThreadSeconds = 0;
// Check whether any of the tasks failed and log action stats if wanted.
bool bSuccess = true;
foreach (KeyValuePair <Action, ActionThread> ActionProcess in ActionThreadDictionary)
{
Action Action = ActionProcess.Key;
ActionThread ActionThread = ActionProcess.Value;
// Check for pending actions, preemptive failure
if (ActionThread == null)
{
bSuccess = false;
continue;
}
// Check for executed action but general failure
if (ActionThread.ExitCode != 0)
{
bSuccess = false;
}
// Log CPU time, tool and task.
double ThreadSeconds = Action.Duration.TotalSeconds;
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats,
TraceEventType.Information,
"^{0}^{1:0.00}^{2}^{3}^{4}",
Action.ActionType.ToString(),
ThreadSeconds,
Path.GetFileName(Action.CommandPath),
Action.StatusDescription,
Action.bIsUsingPCH);
// Update statistics
switch (Action.ActionType)
{
case ActionType.BuildProject:
UnrealBuildTool.TotalBuildProjectTime += ThreadSeconds;
break;
case ActionType.Compile:
UnrealBuildTool.TotalCompileTime += ThreadSeconds;
break;
case ActionType.CreateAppBundle:
UnrealBuildTool.TotalCreateAppBundleTime += ThreadSeconds;
break;
case ActionType.GenerateDebugInfo:
UnrealBuildTool.TotalGenerateDebugInfoTime += ThreadSeconds;
break;
case ActionType.Link:
UnrealBuildTool.TotalLinkTime += ThreadSeconds;
break;
default:
UnrealBuildTool.TotalOtherActionsTime += ThreadSeconds;
break;
}
// Keep track of total thread seconds spent on tasks.
TotalThreadSeconds += ThreadSeconds;
}
Log.TraceInformation("-------- End Detailed Actions Stats -----------------------------------------------------------");
// Log total CPU seconds and numbers of processors involved in tasks.
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats || BuildConfiguration.bPrintDebugInfo,
TraceEventType.Information, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds);
return(bSuccess);
}
/**
* Executes the specified actions locally.
* @return True if all the tasks successfully executed, or false if any of them failed.
*/
public static bool ExecuteActions(List<Action> Actions)
{
// Time to sleep after each iteration of the loop in order to not busy wait.
const float LoopSleepTime = 0.1f;
// Use WMI to figure out physical cores, excluding hyper threading.
int NumCores = 0;
if (!Utils.IsRunningOnMono)
{
try
{
using (var Mos = new System.Management.ManagementObjectSearcher("Select * from Win32_Processor"))
{
var MosCollection = Mos.Get();
foreach (var Item in MosCollection)
{
NumCores += int.Parse(Item["NumberOfCores"].ToString());
}
}
}
catch (Exception Ex)
{
Log.TraceWarning("Unable to get the number of Cores: {0}", Ex.ToString());
Log.TraceWarning("Falling back to processor count.");
}
}
// On some systems this requires a hot fix to work so we fall back to using the (logical) processor count.
if( NumCores == 0 )
{
NumCores = System.Environment.ProcessorCount;
}
// The number of actions to execute in parallel is trying to keep the CPU busy enough in presence of I/O stalls.
int MaxActionsToExecuteInParallel = 0;
if (NumCores < System.Environment.ProcessorCount && BuildConfiguration.ProcessorCountMultiplier != 1.0)
{
// The CPU has more logical cores than physical ones, aka uses hyper-threading.
// Use multiplier if provided
MaxActionsToExecuteInParallel = (int)(NumCores * BuildConfiguration.ProcessorCountMultiplier);
}
else if (NumCores < System.Environment.ProcessorCount && NumCores > 4)
{
// The CPU has more logical cores than physical ones, aka uses hyper-threading.
// Use average of logical and physical if we have "lots of cores"
MaxActionsToExecuteInParallel = (int)(NumCores + System.Environment.ProcessorCount) / 2;
}
// No hyper-threading. Only kicking off a task per CPU to keep machine responsive.
else
{
MaxActionsToExecuteInParallel = NumCores;
}
if (Utils.IsRunningOnMono)
{
// heuristic: give each action at least 1.5GB of RAM (some clang instances will need more, actually)
long MinMemoryPerActionMB = 3 * 1024 / 2;
long PhysicalRAMAvailableMB = (new PerformanceCounter ("Mono Memory", "Total Physical Memory").RawValue) / (1024 * 1024);
int MaxActionsAffordedByMemory = (int)(Math.Max(1, (PhysicalRAMAvailableMB) / MinMemoryPerActionMB));
MaxActionsToExecuteInParallel = Math.Min(MaxActionsToExecuteInParallel, MaxActionsAffordedByMemory);
}
MaxActionsToExecuteInParallel = Math.Max( 1, Math.Min(MaxActionsToExecuteInParallel, BuildConfiguration.MaxProcessorCount) );
Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel);
Dictionary<Action, ActionThread> ActionThreadDictionary = new Dictionary<Action, ActionThread>();
int JobNumber = 1;
using(ProgressWriter ProgressWriter = new ProgressWriter("Compiling C++ source code...", false))
{
int ProgressValue = 0;
while(true)
{
// Count the number of pending and still executing actions.
int NumUnexecutedActions = 0;
int NumExecutingActions = 0;
foreach (Action Action in Actions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
NumUnexecutedActions++;
}
else if (ActionThread != null)
{
if (ActionThread.bComplete == false)
{
NumUnexecutedActions++;
NumExecutingActions++;
}
}
}
// Update the current progress
int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions;
if (ProgressValue != NewProgressValue)
{
ProgressWriter.Write(ProgressValue, Actions.Count + 1);
ProgressValue = NewProgressValue;
}
// If there aren't any pending actions left, we're done executing.
if (NumUnexecutedActions == 0)
{
break;
}
// If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
// prerequisites.
foreach (Action Action in Actions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
if (NumExecutingActions < Math.Max(1,MaxActionsToExecuteInParallel))
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems)
{
if (PrerequisiteItem.ProducingAction != null && Actions.Contains(PrerequisiteItem.ProducingAction))
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue( PrerequisiteItem.ProducingAction, out PrerequisiteProcess );
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
ActionThreadDictionary.Add( Action, null );
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count);
JobNumber++;
ActionThread.Run();
ActionThreadDictionary.Add(Action, ActionThread);
NumExecutingActions++;
}
}
}
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime));
}
}
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, LogEventType.Console, "-------- Begin Detailed Action Stats ----------------------------------------------------------");
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, LogEventType.Console, "^Action Type^Duration (seconds)^Tool^Task^Using PCH");
double TotalThreadSeconds = 0;
// Check whether any of the tasks failed and log action stats if wanted.
bool bSuccess = true;
foreach (KeyValuePair<Action, ActionThread> ActionProcess in ActionThreadDictionary)
{
Action Action = ActionProcess.Key;
ActionThread ActionThread = ActionProcess.Value;
// Check for pending actions, preemptive failure
if (ActionThread == null)
{
bSuccess = false;
continue;
}
// Check for executed action but general failure
if (ActionThread.ExitCode != 0)
{
bSuccess = false;
}
// Log CPU time, tool and task.
double ThreadSeconds = Action.Duration.TotalSeconds;
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats,
LogEventType.Console,
"^{0}^{1:0.00}^{2}^{3}^{4}",
Action.ActionType.ToString(),
ThreadSeconds,
Path.GetFileName(Action.CommandPath),
Action.StatusDescription,
Action.bIsUsingPCH);
// Update statistics
switch (Action.ActionType)
{
case ActionType.BuildProject:
UnrealBuildTool.TotalBuildProjectTime += ThreadSeconds;
break;
case ActionType.Compile:
UnrealBuildTool.TotalCompileTime += ThreadSeconds;
break;
case ActionType.CreateAppBundle:
UnrealBuildTool.TotalCreateAppBundleTime += ThreadSeconds;
break;
case ActionType.GenerateDebugInfo:
UnrealBuildTool.TotalGenerateDebugInfoTime += ThreadSeconds;
break;
case ActionType.Link:
UnrealBuildTool.TotalLinkTime += ThreadSeconds;
break;
default:
UnrealBuildTool.TotalOtherActionsTime += ThreadSeconds;
break;
}
// Keep track of total thread seconds spent on tasks.
TotalThreadSeconds += ThreadSeconds;
}
Log.TraceInformation("-------- End Detailed Actions Stats -----------------------------------------------------------");
// Log total CPU seconds and numbers of processors involved in tasks.
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats || BuildConfiguration.bPrintDebugInfo,
LogEventType.Console, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds);
return bSuccess;
}
public override bool ExecuteActions(List <Action> Actions, bool bLogDetailedActionStats)
{
bool bDistccResult = true;
if (Actions.Count > 0)
{
// Time to sleep after each iteration of the loop in order to not busy wait.
const float LoopSleepTime = 0.1f;
int MaxActionsToExecuteInParallel = 0;
string UserDir = Environment.GetEnvironmentVariable("HOME");
string HostsInfo = UserDir + "/.dmucs/hosts-info";
string NumUBTBuildTasks = Environment.GetEnvironmentVariable("NumUBTBuildTasks");
Int32 MaxUBTBuildTasks = MaxActionsToExecuteInParallel;
if (Int32.TryParse(NumUBTBuildTasks, out MaxUBTBuildTasks))
{
MaxActionsToExecuteInParallel = MaxUBTBuildTasks;
}
else if (BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Mac)
{
using (System.Diagnostics.Process DefaultsProcess = new System.Diagnostics.Process())
{
try
{
DefaultsProcess.StartInfo.FileName = "/usr/bin/defaults";
DefaultsProcess.StartInfo.CreateNoWindow = true;
DefaultsProcess.StartInfo.UseShellExecute = false;
DefaultsProcess.StartInfo.RedirectStandardOutput = true;
DefaultsProcess.StartInfo.Arguments = "read com.apple.dt.Xcode IDEBuildOperationMaxNumberOfConcurrentCompileTasks";
DefaultsProcess.Start();
string Output = DefaultsProcess.StandardOutput.ReadToEnd();
DefaultsProcess.WaitForExit();
if (DefaultsProcess.ExitCode == 0 && Int32.TryParse(Output, out MaxUBTBuildTasks))
{
MaxActionsToExecuteInParallel = MaxUBTBuildTasks;
}
}
catch (Exception)
{
}
}
}
else if (System.IO.File.Exists(HostsInfo))
{
System.IO.StreamReader File = new System.IO.StreamReader(HostsInfo);
string Line = null;
while ((Line = File.ReadLine()) != null)
{
string[] HostInfo = Line.Split(' ');
if (HostInfo.Count() == 3)
{
int NumCPUs = 0;
if (System.Int32.TryParse(HostInfo[1], out NumCPUs))
{
MaxActionsToExecuteInParallel += NumCPUs;
}
}
}
File.Close();
}
else
{
MaxActionsToExecuteInParallel = System.Environment.ProcessorCount;
}
if (bAllowDistccLocalFallback == false)
{
Environment.SetEnvironmentVariable("DISTCC_FALLBACK", "0");
}
if (bVerboseDistccOutput == true)
{
Environment.SetEnvironmentVariable("DISTCC_VERBOSE", "1");
}
FileReference DistccExecutable = new FileReference(DistccExecutablesPath + "/distcc");
FileReference GetHostExecutable = new FileReference(DistccExecutablesPath + "/gethost");
Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel, DistccExecutable, GetHostExecutable);
Dictionary <Action, ActionThread> ActionThreadDictionary = new Dictionary <Action, ActionThread>();
int JobNumber = 1;
using (ProgressWriter ProgressWriter = new ProgressWriter("Compiling source code...", false))
{
int ProgressValue = 0;
while (true)
{
// Count the number of pending and still executing actions.
int NumUnexecutedActions = 0;
int NumExecutingActions = 0;
foreach (Action Action in Actions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
NumUnexecutedActions++;
}
else if (ActionThread != null)
{
if (ActionThread.bComplete == false)
{
NumUnexecutedActions++;
NumExecutingActions++;
}
}
}
// Update the current progress
int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions;
if (ProgressValue != NewProgressValue)
{
ProgressWriter.Write(ProgressValue, Actions.Count + 1);
ProgressValue = NewProgressValue;
}
// If there aren't any pending actions left, we're done executing.
if (NumUnexecutedActions == 0)
{
break;
}
// If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
// prerequisites.
foreach (Action Action in Actions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel))
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (Action PrerequisiteAction in Action.PrerequisiteActions)
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue(PrerequisiteAction, out PrerequisiteProcess);
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
ActionThreadDictionary.Add(Action, null);
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
if (Action.ActionType == ActionType.Compile || Action.ActionType == ActionType.Link)
{
string TypeCommand = String.IsNullOrEmpty(DMUCSDistProp) ? "" : (" --type " + DMUCSDistProp);
string NewCommandArguments = "--server " + DMUCSCoordinator + TypeCommand + " --wait -1 \"" + DistccExecutable + "\" " + Action.CommandPath + " " + Action.CommandArguments;
if (Action.ActionType == ActionType.Compile)
{
// Distcc separates preprocessing and compilation which means we must silence these warnings
NewCommandArguments += " -Wno-constant-logical-operand";
NewCommandArguments += " -Wno-parentheses-equality";
}
Action.CommandPath = GetHostExecutable;
Action.CommandArguments = NewCommandArguments;
}
ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count);
JobNumber++;
ActionThread.Run();
ActionThreadDictionary.Add(Action, ActionThread);
NumExecutingActions++;
}
}
}
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime));
}
}
Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "-------- Begin Detailed Action Stats ----------------------------------------------------------");
Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "^Action Type^Duration (seconds)^Tool^Task^Using PCH");
double TotalThreadSeconds = 0;
// Check whether any of the tasks failed and log action stats if wanted.
foreach (KeyValuePair <Action, ActionThread> ActionProcess in ActionThreadDictionary)
{
Action Action = ActionProcess.Key;
ActionThread ActionThread = ActionProcess.Value;
// Check for pending actions, preemptive failure
if (ActionThread == null)
{
bDistccResult = false;
continue;
}
// Check for executed action but general failure
if (ActionThread.ExitCode != 0)
{
bDistccResult = false;
}
// Log CPU time, tool and task.
double ThreadSeconds = Action.Duration.TotalSeconds;
Log.WriteLineIf(bLogDetailedActionStats,
LogEventType.Console,
"^{0}^{1:0.00}^{2}^{3}^{4}",
Action.ActionType.ToString(),
ThreadSeconds,
Action.CommandPath.GetFileName(),
Action.StatusDescription,
Action.bIsUsingPCH);
// Keep track of total thread seconds spent on tasks.
TotalThreadSeconds += ThreadSeconds;
}
Log.WriteLineIf(bLogDetailedActionStats,
LogEventType.Console,
"-------- End Detailed Actions Stats -----------------------------------------------------------");
// Log total CPU seconds and numbers of processors involved in tasks.
Log.WriteLineIf(bLogDetailedActionStats,
LogEventType.Console, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds);
}
return(bDistccResult);
}
public override FileItem[] LinkAllFiles(LinkEnvironment LinkEnvironment, bool bBuildImportLibraryOnly)
{
List <FileItem> Outputs = new List <FileItem>();
for (int ArchIndex = 0; ArchIndex < Arches.Length; ArchIndex++)
{
string Arch = Arches[ArchIndex];
for (int GPUArchIndex = 0; GPUArchIndex < GPUArchitectures.Length; GPUArchIndex++)
{
string GPUArchitecture = GPUArchitectures[GPUArchIndex];
int OutputPathIndex = ArchIndex * GPUArchitectures.Length + GPUArchIndex;
// Android will have an array of outputs
if (LinkEnvironment.Config.OutputFilePaths.Length < OutputPathIndex ||
!Path.GetFileNameWithoutExtension(LinkEnvironment.Config.OutputFilePaths[OutputPathIndex]).EndsWith(Arch + GPUArchitecture))
{
throw new BuildException("The OutputFilePaths array didn't match the Arches array in AndroidToolChain.LinkAllFiles");
}
// Create an action that invokes the linker.
Action LinkAction = new Action(ActionType.Link);
LinkAction.WorkingDirectory = Path.GetFullPath(".");
if (LinkEnvironment.Config.bIsBuildingLibrary)
{
LinkAction.CommandPath = Arch == "-armv7" ? ArPathArm : ArPathx86;
}
else
{
LinkAction.CommandPath = ClangPath;
}
string LinkerPath = LinkAction.WorkingDirectory;
LinkAction.WorkingDirectory = LinkEnvironment.Config.IntermediateDirectory;
// Get link arguments.
LinkAction.CommandArguments = LinkEnvironment.Config.bIsBuildingLibrary ? GetArArguments(LinkEnvironment) : GetLinkArguments(LinkEnvironment, Arch);
// Add the output file as a production of the link action.
FileItem OutputFile;
OutputFile = FileItem.GetItemByPath(LinkEnvironment.Config.OutputFilePaths[OutputPathIndex]);
Outputs.Add(OutputFile);
LinkAction.ProducedItems.Add(OutputFile);
LinkAction.StatusDescription = string.Format("{0}", Path.GetFileName(OutputFile.AbsolutePath));
// LinkAction.bPrintDebugInfo = true;
// Add the output file to the command-line.
if (LinkEnvironment.Config.bIsBuildingLibrary)
{
LinkAction.CommandArguments += string.Format(" \"{0}\"", OutputFile.AbsolutePath);
}
else
{
LinkAction.CommandArguments += string.Format(" -o \"{0}\"", OutputFile.AbsolutePath);
}
// Add the input files to a response file, and pass the response file on the command-line.
List <string> InputFileNames = new List <string>();
foreach (FileItem InputFile in LinkEnvironment.InputFiles)
{
// make sure it's for current Arch
if (Path.GetFileNameWithoutExtension(InputFile.AbsolutePath).EndsWith(Arch + GPUArchitecture))
{
string AbsolutePath = InputFile.AbsolutePath.Replace("\\", "/");
AbsolutePath = AbsolutePath.Replace(LinkEnvironment.Config.IntermediateDirectory.Replace("\\", "/"), "");
AbsolutePath = AbsolutePath.TrimStart(new char[] { '/' });
InputFileNames.Add(string.Format("\"{0}\"", AbsolutePath));
LinkAction.PrerequisiteItems.Add(InputFile);
}
}
string ResponseFileName = GetResponseFileName(LinkEnvironment, OutputFile);
LinkAction.CommandArguments += string.Format(" @\"{0}\"", ResponseFile.Create(ResponseFileName, InputFileNames));
// libs don't link in other libs
if (!LinkEnvironment.Config.bIsBuildingLibrary)
{
// Add the library paths to the argument list.
foreach (string LibraryPath in LinkEnvironment.Config.LibraryPaths)
{
// LinkerPaths could be relative or absolute
string AbsoluteLibraryPath = ActionThread.ExpandEnvironmentVariables(LibraryPath);
if (IsDirectoryForArch(AbsoluteLibraryPath, Arch))
{
// environment variables aren't expanded when using the $( style
if (Path.IsPathRooted(AbsoluteLibraryPath) == false)
{
AbsoluteLibraryPath = Path.Combine(LinkerPath, AbsoluteLibraryPath);
}
LinkAction.CommandArguments += string.Format(" -L\"{0}\"", AbsoluteLibraryPath);
}
}
// add libraries in a library group
LinkAction.CommandArguments += string.Format(" -Wl,--start-group");
foreach (string AdditionalLibrary in LinkEnvironment.Config.AdditionalLibraries)
{
if (!ShouldSkipLib(AdditionalLibrary, Arch, GPUArchitecture))
{
if (String.IsNullOrEmpty(Path.GetDirectoryName(AdditionalLibrary)))
{
LinkAction.CommandArguments += string.Format(" \"-l{0}\"", AdditionalLibrary);
}
else
{
// full pathed libs are compiled by us, so we depend on linking them
LinkAction.CommandArguments += string.Format(" \"{0}\"", AdditionalLibrary);
LinkAction.PrerequisiteItems.Add(FileItem.GetItemByPath(AdditionalLibrary));
}
}
}
LinkAction.CommandArguments += string.Format(" -Wl,--end-group");
}
// Add the additional arguments specified by the environment.
LinkAction.CommandArguments += LinkEnvironment.Config.AdditionalArguments;
LinkAction.CommandArguments = LinkAction.CommandArguments.Replace("\\", "/");
// Only execute linking on the local PC.
LinkAction.bCanExecuteRemotely = false;
}
}
return(Outputs.ToArray());
}
public static bool ExecuteActions(List <Action> Actions)
{
bool bDistccResult = true;
if (Actions.Count > 0)
{
// Time to sleep after each iteration of the loop in order to not busy wait.
const float LoopSleepTime = 0.1f;
int MaxActionsToExecuteInParallel = 0;
string UserDir = Environment.GetEnvironmentVariable("HOME");
string HostsInfo = UserDir + "/.dmucs/hosts-info";
System.IO.StreamReader File = new System.IO.StreamReader(HostsInfo);
string Line = null;
while ((Line = File.ReadLine()) != null)
{
var HostInfo = Line.Split(' ');
if (HostInfo.Count() == 3)
{
int NumCPUs = 0;
if (System.Int32.TryParse(HostInfo [1], out NumCPUs))
{
MaxActionsToExecuteInParallel += NumCPUs;
}
}
}
File.Close();
if (BuildConfiguration.bAllowDistccLocalFallback == false)
{
Environment.SetEnvironmentVariable("DISTCC_FALLBACK", "0");
}
string DistccExecutable = BuildConfiguration.DistccExecutablesPath + "/distcc";
string GetHostExecutable = BuildConfiguration.DistccExecutablesPath + "/gethost";
Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel, DistccExecutable, GetHostExecutable);
Dictionary <Action, ActionThread> ActionThreadDictionary = new Dictionary <Action, ActionThread>();
int JobNumber = 1;
using (ProgressWriter ProgressWriter = new ProgressWriter("Compiling source code...", false))
{
int ProgressValue = 0;
while (true)
{
// Count the number of pending and still executing actions.
int NumUnexecutedActions = 0;
int NumExecutingActions = 0;
foreach (Action Action in Actions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
NumUnexecutedActions++;
}
else if (ActionThread != null)
{
if (ActionThread.bComplete == false)
{
NumUnexecutedActions++;
NumExecutingActions++;
}
}
}
// Update the current progress
int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions;
if (ProgressValue != NewProgressValue)
{
ProgressWriter.Write(ProgressValue, Actions.Count + 1);
ProgressValue = NewProgressValue;
}
// If there aren't any pending actions left, we're done executing.
if (NumUnexecutedActions == 0)
{
break;
}
// If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
// prerequisites.
foreach (Action Action in Actions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel))
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems)
{
if (PrerequisiteItem.ProducingAction != null && Actions.Contains(PrerequisiteItem.ProducingAction))
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue(PrerequisiteItem.ProducingAction, out PrerequisiteProcess);
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
ActionThreadDictionary.Add(Action, null);
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
if ((Action.ActionType == ActionType.Compile || Action.ActionType == ActionType.Link) && DistccExecutable != null && GetHostExecutable != null)
{
string NewCommandArguments = "--wait -1 \"" + DistccExecutable + "\" " + Action.CommandPath + " " + Action.CommandArguments;
Action.CommandPath = GetHostExecutable;
Action.CommandArguments = NewCommandArguments;
}
ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count);
JobNumber++;
ActionThread.Run();
ActionThreadDictionary.Add(Action, ActionThread);
NumExecutingActions++;
}
}
}
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime));
}
}
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "-------- Begin Detailed Action Stats ----------------------------------------------------------");
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "^Action Type^Duration (seconds)^Tool^Task^Using PCH");
double TotalThreadSeconds = 0;
// Check whether any of the tasks failed and log action stats if wanted.
foreach (KeyValuePair <Action, ActionThread> ActionProcess in ActionThreadDictionary)
{
Action Action = ActionProcess.Key;
ActionThread ActionThread = ActionProcess.Value;
// Check for pending actions, preemptive failure
if (ActionThread == null)
{
bDistccResult = false;
continue;
}
// Check for executed action but general failure
if (ActionThread.ExitCode != 0)
{
bDistccResult = false;
}
// Log CPU time, tool and task.
double ThreadSeconds = Action.Duration.TotalSeconds;
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats,
TraceEventType.Information,
"^{0}^{1:0.00}^{2}^{3}^{4}",
Action.ActionType.ToString(),
ThreadSeconds,
Path.GetFileName(Action.CommandPath),
Action.StatusDescription,
Action.bIsUsingPCH);
// Update statistics
switch (Action.ActionType)
{
case ActionType.BuildProject:
UnrealBuildTool.TotalBuildProjectTime += ThreadSeconds;
break;
case ActionType.Compile:
UnrealBuildTool.TotalCompileTime += ThreadSeconds;
break;
case ActionType.CreateAppBundle:
UnrealBuildTool.TotalCreateAppBundleTime += ThreadSeconds;
break;
case ActionType.GenerateDebugInfo:
UnrealBuildTool.TotalGenerateDebugInfoTime += ThreadSeconds;
break;
case ActionType.Link:
UnrealBuildTool.TotalLinkTime += ThreadSeconds;
break;
default:
UnrealBuildTool.TotalOtherActionsTime += ThreadSeconds;
break;
}
// Keep track of total thread seconds spent on tasks.
TotalThreadSeconds += ThreadSeconds;
}
Log.TraceInformation("-------- End Detailed Actions Stats -----------------------------------------------------------");
// Log total CPU seconds and numbers of processors involved in tasks.
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats || BuildConfiguration.bPrintDebugInfo,
TraceEventType.Information, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds);
}
return(bDistccResult);
}
public override CPPOutput CompileCPPFiles(UEBuildTarget Target, CPPEnvironment CompileEnvironment, List <FileItem> SourceFiles, string ModuleName)
{
if (Arches.Length == 0)
{
throw new BuildException("At least one architecture (armv7, x86, etc) needs to be selected in the project settings to build");
}
if (!bHasPrintedApiLevel)
{
Console.WriteLine("Compiling with NDK API '{0}'", GetNdkApiLevel());
bHasPrintedApiLevel = true;
}
string BaseArguments = "";
if (CompileEnvironment.Config.PrecompiledHeaderAction != PrecompiledHeaderAction.Create)
{
BaseArguments += " -Werror";
}
// Directly added NDK files for NDK extensions
if (!UnrealBuildTool.RunningRocket())
{
ConditionallyAddNDKSourceFiles(SourceFiles);
}
// Add preprocessor definitions to the argument list.
foreach (string Definition in CompileEnvironment.Config.Definitions)
{
BaseArguments += string.Format(" -D \"{0}\"", Definition);
}
var BuildPlatform = UEBuildPlatform.GetBuildPlatformForCPPTargetPlatform(CompileEnvironment.Config.Target.Platform);
string BasePCHName = "";
var PCHExtension = UEBuildPlatform.BuildPlatformDictionary[UnrealTargetPlatform.Android].GetBinaryExtension(UEBuildBinaryType.PrecompiledHeader);
if (CompileEnvironment.Config.PrecompiledHeaderAction == PrecompiledHeaderAction.Include)
{
BasePCHName = RemoveArchName(CompileEnvironment.PrecompiledHeaderFile.AbsolutePath).Replace(PCHExtension, "");
}
// Create a compile action for each source file.
CPPOutput Result = new CPPOutput();
foreach (string Arch in Arches)
{
foreach (string GPUArchitecture in GPUArchitectures)
{
// which toolchain to use
string Arguments = GetCLArguments_Global(CompileEnvironment, Arch) + BaseArguments;
if (GPUArchitecture == "-gl4")
{
Arguments += " -DPLATFORM_ANDROIDGL4=1";
}
else if (GPUArchitecture == "-es31")
{
Arguments += " -DPLATFORM_ANDROIDES31=1";
}
// which PCH file to include
string PCHArguments = "";
if (CompileEnvironment.Config.PrecompiledHeaderAction == PrecompiledHeaderAction.Include)
{
// Add the precompiled header file's path to the include path so Clang can find it.
// This needs to be before the other include paths to ensure Clang uses it instead of the source header file.
PCHArguments += string.Format(" -include \"{0}\"", InlineArchName(BasePCHName, Arch, GPUArchitecture));
}
// Add include paths to the argument list (filtered by architecture)
foreach (string IncludePath in CompileEnvironment.Config.CPPIncludeInfo.SystemIncludePaths)
{
if (IsDirectoryForArch(IncludePath, Arch))
{
Arguments += string.Format(" -I\"{0}\"", IncludePath);
}
}
foreach (string IncludePath in CompileEnvironment.Config.CPPIncludeInfo.IncludePaths)
{
if (IsDirectoryForArch(IncludePath, Arch))
{
Arguments += string.Format(" -I\"{0}\"", IncludePath);
}
}
foreach (FileItem SourceFile in SourceFiles)
{
Action CompileAction = new Action(ActionType.Compile);
string FileArguments = "";
bool bIsPlainCFile = Path.GetExtension(SourceFile.AbsolutePath).ToUpperInvariant() == ".C";
// should we disable optimizations on this file?
// @todo android - We wouldn't need this if we could disable optimizations per function (via pragma)
bool bDisableOptimizations = false; // SourceFile.AbsolutePath.ToUpperInvariant().IndexOf("\\SLATE\\") != -1;
if (bDisableOptimizations && CompileEnvironment.Config.Target.Configuration != CPPTargetConfiguration.Debug)
{
Log.TraceWarning("Disabling optimizations on {0}", SourceFile.AbsolutePath);
}
bDisableOptimizations = bDisableOptimizations || CompileEnvironment.Config.Target.Configuration == CPPTargetConfiguration.Debug;
// Add C or C++ specific compiler arguments.
if (CompileEnvironment.Config.PrecompiledHeaderAction == PrecompiledHeaderAction.Create)
{
FileArguments += GetCompileArguments_PCH(bDisableOptimizations);
}
else if (bIsPlainCFile)
{
FileArguments += GetCompileArguments_C(bDisableOptimizations);
}
else
{
FileArguments += GetCompileArguments_CPP(bDisableOptimizations);
// only use PCH for .cpp files
FileArguments += PCHArguments;
}
// Add the C++ source file and its included files to the prerequisite item list.
AddPrerequisiteSourceFile(Target, BuildPlatform, CompileEnvironment, SourceFile, CompileAction.PrerequisiteItems);
if (CompileEnvironment.Config.PrecompiledHeaderAction == PrecompiledHeaderAction.Create)
{
// Add the precompiled header file to the produced item list.
FileItem PrecompiledHeaderFile = FileItem.GetItemByPath(
Path.Combine(
CompileEnvironment.Config.OutputDirectory,
Path.GetFileName(InlineArchName(SourceFile.AbsolutePath, Arch, GPUArchitecture) + PCHExtension)
)
);
CompileAction.ProducedItems.Add(PrecompiledHeaderFile);
Result.PrecompiledHeaderFile = PrecompiledHeaderFile;
// Add the parameters needed to compile the precompiled header file to the command-line.
FileArguments += string.Format(" -o \"{0}\"", PrecompiledHeaderFile.AbsolutePath, false);
}
else
{
if (CompileEnvironment.Config.PrecompiledHeaderAction == PrecompiledHeaderAction.Include)
{
CompileAction.bIsUsingPCH = true;
FileItem ArchPrecompiledHeaderFile = FileItem.GetItemByPath(InlineArchName(BasePCHName, Arch, GPUArchitecture) + PCHExtension);
CompileAction.PrerequisiteItems.Add(ArchPrecompiledHeaderFile);
}
var ObjectFileExtension = UEBuildPlatform.BuildPlatformDictionary[UnrealTargetPlatform.Android].GetBinaryExtension(UEBuildBinaryType.Object);
// Add the object file to the produced item list.
FileItem ObjectFile = FileItem.GetItemByPath(
InlineArchName(Path.Combine(
CompileEnvironment.Config.OutputDirectory,
Path.GetFileName(SourceFile.AbsolutePath) + ObjectFileExtension), Arch, GPUArchitecture
)
);
CompileAction.ProducedItems.Add(ObjectFile);
Result.ObjectFiles.Add(ObjectFile);
FileArguments += string.Format(" -o \"{0}\"", ObjectFile.AbsolutePath, false);
}
// Add the source file path to the command-line.
FileArguments += string.Format(" \"{0}\"", SourceFile.AbsolutePath);
// Build a full argument list
string AllArguments = Arguments + FileArguments + CompileEnvironment.Config.AdditionalArguments;
AllArguments = ActionThread.ExpandEnvironmentVariables(AllArguments);
AllArguments = AllArguments.Replace("\\", "/");
// Create the response file
string ResponseFileName = CompileAction.ProducedItems[0].AbsolutePath + ".response";
string ResponseArgument = string.Format("@\"{0}\"", ResponseFile.Create(ResponseFileName, new List <string> {
AllArguments
}));
CompileAction.WorkingDirectory = Path.GetFullPath(".");
CompileAction.CommandPath = ClangPath;
CompileAction.CommandArguments = ResponseArgument;
CompileAction.StatusDescription = string.Format("{0} [{1}-{2}]", Path.GetFileName(SourceFile.AbsolutePath), Arch.Replace("-", ""), GPUArchitecture.Replace("-", ""));
CompileAction.OutputEventHandler = new DataReceivedEventHandler(CompileOutputReceivedDataEventHandler);
// VC++ always outputs the source file name being compiled, so we don't need to emit this ourselves
CompileAction.bShouldOutputStatusDescription = true;
// Don't farm out creation of pre-compiled headers as it is the critical path task.
CompileAction.bCanExecuteRemotely =
CompileEnvironment.Config.PrecompiledHeaderAction != PrecompiledHeaderAction.Create ||
BuildConfiguration.bAllowRemotelyCompiledPCHs;
}
}
}
return(Result);
}
public static ExecutionResult ExecuteActions(List <Action> Actions)
{
ExecutionResult SNDBSResult = ExecutionResult.TasksSucceeded;
if (Actions.Count > 0)
{
string SCERoot = Environment.GetEnvironmentVariable("SCE_ROOT_DIR");
bool bSNDBSExists = false;
if (SCERoot != null)
{
string SNDBSExecutable = Path.Combine(SCERoot, "Common/SN-DBS/bin/dbsbuild.exe");
// Check that SN-DBS is available
bSNDBSExists = File.Exists(SNDBSExecutable);
}
if (bSNDBSExists == false)
{
return(ExecutionResult.Unavailable);
}
// Use WMI to figure out physical cores, excluding hyper threading.
int NumCores = 0;
if (!Utils.IsRunningOnMono)
{
try
{
using (ManagementObjectSearcher Mos = new System.Management.ManagementObjectSearcher("Select * from Win32_Processor"))
{
ManagementObjectCollection MosCollection = Mos.Get();
foreach (ManagementBaseObject Item in MosCollection)
{
NumCores += int.Parse(Item["NumberOfCores"].ToString());
}
}
}
catch (Exception Ex)
{
Log.TraceWarning("Unable to get the number of Cores: {0}", Ex.ToString());
Log.TraceWarning("Falling back to processor count.");
}
}
// On some systems this requires a hot fix to work so we fall back to using the (logical) processor count.
if (NumCores == 0)
{
NumCores = System.Environment.ProcessorCount;
}
// The number of actions to execute in parallel is trying to keep the CPU busy enough in presence of I/O stalls.
MaxActionsToExecuteInParallel = 0;
// The CPU has more logical cores than physical ones, aka uses hyper-threading.
if (NumCores < System.Environment.ProcessorCount)
{
MaxActionsToExecuteInParallel = (int)(NumCores * BuildConfiguration.ProcessorCountMultiplier);
}
// No hyper-threading. Only kicking off a task per CPU to keep machine responsive.
else
{
MaxActionsToExecuteInParallel = NumCores;
}
MaxActionsToExecuteInParallel = Math.Min(MaxActionsToExecuteInParallel, BuildConfiguration.MaxProcessorCount);
JobNumber = 1;
Dictionary <Action, ActionThread> ActionThreadDictionary = new Dictionary <Action, ActionThread>();
while (true)
{
bool bUnexecutedActions = false;
foreach (Action Action in Actions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
bUnexecutedActions = true;
ExecutionResult CompileResult = ExecuteActions(Actions, ActionThreadDictionary);
if (CompileResult != ExecutionResult.TasksSucceeded)
{
return(ExecutionResult.TasksFailed);
}
break;
}
}
if (bUnexecutedActions == false)
{
break;
}
}
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, LogEventType.Console, "-------- Begin Detailed Action Stats ----------------------------------------------------------");
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, LogEventType.Console, "^Action Type^Duration (seconds)^Tool^Task^Using PCH");
double TotalThreadSeconds = 0;
// Check whether any of the tasks failed and log action stats if wanted.
foreach (KeyValuePair <Action, ActionThread> ActionProcess in ActionThreadDictionary)
{
Action Action = ActionProcess.Key;
ActionThread ActionThread = ActionProcess.Value;
// Check for pending actions, preemptive failure
if (ActionThread == null)
{
SNDBSResult = ExecutionResult.TasksFailed;
continue;
}
// Check for executed action but general failure
if (ActionThread.ExitCode != 0)
{
SNDBSResult = ExecutionResult.TasksFailed;
}
// Log CPU time, tool and task.
double ThreadSeconds = Action.Duration.TotalSeconds;
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats,
LogEventType.Console,
"^{0}^{1:0.00}^{2}^{3}^{4}",
Action.ActionType.ToString(),
ThreadSeconds,
Path.GetFileName(Action.CommandPath),
Action.StatusDescription,
Action.bIsUsingPCH);
// Update statistics
switch (Action.ActionType)
{
case ActionType.BuildProject:
UnrealBuildTool.TotalBuildProjectTime += ThreadSeconds;
break;
case ActionType.Compile:
UnrealBuildTool.TotalCompileTime += ThreadSeconds;
break;
case ActionType.CreateAppBundle:
UnrealBuildTool.TotalCreateAppBundleTime += ThreadSeconds;
break;
case ActionType.GenerateDebugInfo:
UnrealBuildTool.TotalGenerateDebugInfoTime += ThreadSeconds;
break;
case ActionType.Link:
UnrealBuildTool.TotalLinkTime += ThreadSeconds;
break;
default:
UnrealBuildTool.TotalOtherActionsTime += ThreadSeconds;
break;
}
// Keep track of total thread seconds spent on tasks.
TotalThreadSeconds += ThreadSeconds;
}
Log.TraceInformation("-------- End Detailed Actions Stats -----------------------------------------------------------");
// Log total CPU seconds and numbers of processors involved in tasks.
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats || BuildConfiguration.bPrintDebugInfo,
LogEventType.Console, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds);
}
return(SNDBSResult);
}
public static bool ExecuteActions(List<Action> Actions)
{
bool bDistccResult = true;
if (Actions.Count > 0)
{
// Time to sleep after each iteration of the loop in order to not busy wait.
const float LoopSleepTime = 0.1f;
int MaxActionsToExecuteInParallel = 0;
string UserDir = Environment.GetEnvironmentVariable("HOME");
string HostsInfo = UserDir + "/.dmucs/hosts-info";
string NumUBTBuildTasks = Environment.GetEnvironmentVariable("NumUBTBuildTasks");
Int32 MaxUBTBuildTasks = MaxActionsToExecuteInParallel;
if(Int32.TryParse(NumUBTBuildTasks, out MaxUBTBuildTasks))
{
MaxActionsToExecuteInParallel = MaxUBTBuildTasks;
}
else if (BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Mac)
{
using (System.Diagnostics.Process DefaultsProcess = new System.Diagnostics.Process())
{
try
{
DefaultsProcess.StartInfo.FileName = "/usr/bin/defaults";
DefaultsProcess.StartInfo.CreateNoWindow = true;
DefaultsProcess.StartInfo.UseShellExecute = false;
DefaultsProcess.StartInfo.RedirectStandardOutput = true;
DefaultsProcess.StartInfo.Arguments = "com.apple.dt.Xcode IDEBuildOperationMaxNumberOfConcurrentCompileTasks";
DefaultsProcess.Start();
string Output = DefaultsProcess.StandardOutput.ReadToEnd();
DefaultsProcess.WaitForExit();
if(DefaultsProcess.ExitCode == 0 && Int32.TryParse(Output, out MaxUBTBuildTasks))
{
MaxActionsToExecuteInParallel = MaxUBTBuildTasks;
}
}
catch (Exception)
{
}
}
}
else if (System.IO.File.Exists (HostsInfo))
{
System.IO.StreamReader File = new System.IO.StreamReader(HostsInfo);
string Line = null;
while((Line = File.ReadLine()) != null)
{
var HostInfo = Line.Split(' ');
if (HostInfo.Count () == 3)
{
int NumCPUs = 0;
if (System.Int32.TryParse (HostInfo [1], out NumCPUs))
{
MaxActionsToExecuteInParallel += NumCPUs;
}
}
}
File.Close();
}
else
{
MaxActionsToExecuteInParallel = System.Environment.ProcessorCount;
}
if (BuildConfiguration.bAllowDistccLocalFallback == false)
{
Environment.SetEnvironmentVariable("DISTCC_FALLBACK", "0");
}
if (BuildConfiguration.bVerboseDistccOutput == true)
{
Environment.SetEnvironmentVariable("DISTCC_VERBOSE", "1");
}
string DistccExecutable = BuildConfiguration.DistccExecutablesPath + "/distcc";
string GetHostExecutable = BuildConfiguration.DistccExecutablesPath + "/gethost";
Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel, DistccExecutable, GetHostExecutable);
Dictionary<Action, ActionThread> ActionThreadDictionary = new Dictionary<Action, ActionThread>();
int JobNumber = 1;
using(ProgressWriter ProgressWriter = new ProgressWriter("Compiling source code...", false))
{
int ProgressValue = 0;
while(true)
{
// Count the number of pending and still executing actions.
int NumUnexecutedActions = 0;
int NumExecutingActions = 0;
foreach (Action Action in Actions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
NumUnexecutedActions++;
}
else if (ActionThread != null)
{
if (ActionThread.bComplete == false)
{
NumUnexecutedActions++;
NumExecutingActions++;
}
}
}
// Update the current progress
int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions;
if (ProgressValue != NewProgressValue)
{
ProgressWriter.Write(ProgressValue, Actions.Count + 1);
ProgressValue = NewProgressValue;
}
// If there aren't any pending actions left, we're done executing.
if (NumUnexecutedActions == 0)
{
break;
}
// If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
// prerequisites.
foreach (Action Action in Actions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
if (NumExecutingActions < Math.Max(1,MaxActionsToExecuteInParallel))
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems)
{
if (PrerequisiteItem.ProducingAction != null && Actions.Contains(PrerequisiteItem.ProducingAction))
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue( PrerequisiteItem.ProducingAction, out PrerequisiteProcess );
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
ActionThreadDictionary.Add( Action, null );
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
if ((Action.ActionType == ActionType.Compile || Action.ActionType == ActionType.Link) && DistccExecutable != null && GetHostExecutable != null)
{
string TypeCommand = String.IsNullOrEmpty(BuildConfiguration.DMUCSDistProp) ? "" : (" --type " + BuildConfiguration.DMUCSDistProp);
string NewCommandArguments = "--server " + BuildConfiguration.DMUCSCoordinator + TypeCommand + " --wait -1 \"" + DistccExecutable + "\" " + Action.CommandPath + " " + Action.CommandArguments;
Action.CommandPath = GetHostExecutable;
Action.CommandArguments = NewCommandArguments;
}
ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count);
JobNumber++;
ActionThread.Run();
ActionThreadDictionary.Add(Action, ActionThread);
NumExecutingActions++;
}
}
}
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime));
}
}
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, LogEventType.Console, "-------- Begin Detailed Action Stats ----------------------------------------------------------");
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, LogEventType.Console, "^Action Type^Duration (seconds)^Tool^Task^Using PCH");
double TotalThreadSeconds = 0;
// Check whether any of the tasks failed and log action stats if wanted.
foreach (KeyValuePair<Action, ActionThread> ActionProcess in ActionThreadDictionary)
{
Action Action = ActionProcess.Key;
ActionThread ActionThread = ActionProcess.Value;
// Check for pending actions, preemptive failure
if (ActionThread == null)
{
bDistccResult = false;
continue;
}
// Check for executed action but general failure
if (ActionThread.ExitCode != 0)
{
bDistccResult = false;
}
// Log CPU time, tool and task.
double ThreadSeconds = Action.Duration.TotalSeconds;
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats,
LogEventType.Console,
"^{0}^{1:0.00}^{2}^{3}^{4}",
Action.ActionType.ToString(),
ThreadSeconds,
Path.GetFileName(Action.CommandPath),
Action.StatusDescription,
Action.bIsUsingPCH);
// Update statistics
switch (Action.ActionType)
{
case ActionType.BuildProject:
UnrealBuildTool.TotalBuildProjectTime += ThreadSeconds;
break;
case ActionType.Compile:
UnrealBuildTool.TotalCompileTime += ThreadSeconds;
break;
case ActionType.CreateAppBundle:
UnrealBuildTool.TotalCreateAppBundleTime += ThreadSeconds;
break;
case ActionType.GenerateDebugInfo:
UnrealBuildTool.TotalGenerateDebugInfoTime += ThreadSeconds;
break;
case ActionType.Link:
UnrealBuildTool.TotalLinkTime += ThreadSeconds;
break;
default:
UnrealBuildTool.TotalOtherActionsTime += ThreadSeconds;
break;
}
// Keep track of total thread seconds spent on tasks.
TotalThreadSeconds += ThreadSeconds;
}
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats || BuildConfiguration.bPrintDebugInfo,
LogEventType.Console,
"-------- End Detailed Actions Stats -----------------------------------------------------------");
// Log total CPU seconds and numbers of processors involved in tasks.
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats || BuildConfiguration.bPrintDebugInfo,
LogEventType.Console, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds);
}
return bDistccResult;
}
internal bool ExecuteLocalActions(List <Action> InLocalActions, Dictionary <Action, ActionThread> InActionThreadDictionary, int TotalNumJobs)
{
// Time to sleep after each iteration of the loop in order to not busy wait.
const float LoopSleepTime = 0.1f;
bool LocalActionsResult = true;
while (true)
{
// Count the number of pending and still executing actions.
int NumUnexecutedActions = 0;
int NumExecutingActions = 0;
foreach (Action Action in InLocalActions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = InActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
NumUnexecutedActions++;
}
else if (ActionThread != null)
{
if (ActionThread.bComplete == false)
{
NumUnexecutedActions++;
NumExecutingActions++;
}
}
}
// If there aren't any pending actions left, we're done executing.
if (NumUnexecutedActions == 0)
{
break;
}
// If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
// prerequisites.
foreach (Action Action in InLocalActions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = InActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel))
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (Action PrerequisiteAction in Action.PrerequisiteActions)
{
if (InLocalActions.Contains(PrerequisiteAction))
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = InActionThreadDictionary.TryGetValue(PrerequisiteAction, out PrerequisiteProcess);
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
InActionThreadDictionary.Add(Action, null);
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
ActionThread ActionThread = new ActionThread(Action, JobNumber, TotalNumJobs);
ActionThread.Run();
InActionThreadDictionary.Add(Action, ActionThread);
NumExecutingActions++;
JobNumber++;
}
}
}
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime));
}
return(LocalActionsResult);
}
/// <summary>
/// Executes the specified actions locally.
/// </summary>
/// <returns>True if all the tasks successfully executed, or false if any of them failed.</returns>
public override bool ExecuteActions(List <Action> Actions, bool bLogDetailedActionStats)
{
// Time to sleep after each iteration of the loop in order to not busy wait.
const float LoopSleepTime = 0.1f;
// The number of actions to execute in parallel is trying to keep the CPU busy enough in presence of I/O stalls.
int MaxActionsToExecuteInParallel = GetMaxActionsToExecuteInParallel();
Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel);
Dictionary <Action, ActionThread> ActionThreadDictionary = new Dictionary <Action, ActionThread>();
int JobNumber = 1;
using (ProgressWriter ProgressWriter = new ProgressWriter("Compiling C++ source code...", false))
{
int ProgressValue = 0;
while (true)
{
// Count the number of pending and still executing actions.
int NumUnexecutedActions = 0;
int NumExecutingActions = 0;
foreach (Action Action in Actions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
NumUnexecutedActions++;
}
else if (ActionThread != null)
{
if (ActionThread.bComplete == false)
{
NumUnexecutedActions++;
NumExecutingActions++;
}
}
}
// Update the current progress
int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions;
if (ProgressValue != NewProgressValue)
{
ProgressWriter.Write(ProgressValue, Actions.Count + 1);
ProgressValue = NewProgressValue;
}
// If there aren't any pending actions left, we're done executing.
if (NumUnexecutedActions == 0)
{
break;
}
// If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
// prerequisites.
foreach (Action Action in Actions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel))
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (Action PrerequisiteAction in Action.PrerequisiteActions)
{
if (Actions.Contains(PrerequisiteAction))
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue(PrerequisiteAction, out PrerequisiteProcess);
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
ActionThreadDictionary.Add(Action, null);
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count);
JobNumber++;
try
{
ActionThread.Run();
}
catch (Exception ex)
{
throw new BuildException(ex, "Failed to start thread for action: {0} {1}\r\n{2}", Action.CommandPath, Action.CommandArguments, ex.ToString());
}
ActionThreadDictionary.Add(Action, ActionThread);
NumExecutingActions++;
}
}
}
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime));
}
}
Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "-------- Begin Detailed Action Stats ----------------------------------------------------------");
Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "^Action Type^Duration (seconds)^Tool^Task^Using PCH");
double TotalThreadSeconds = 0;
// Check whether any of the tasks failed and log action stats if wanted.
bool bSuccess = true;
double TotalBuildProjectTime = 0;
double TotalCompileTime = 0;
double TotalCreateAppBundleTime = 0;
double TotalGenerateDebugInfoTime = 0;
double TotalLinkTime = 0;
double TotalOtherActionsTime = 0;
foreach (KeyValuePair <Action, ActionThread> ActionProcess in ActionThreadDictionary)
{
Action Action = ActionProcess.Key;
ActionThread ActionThread = ActionProcess.Value;
// Check for pending actions, preemptive failure
if (ActionThread == null)
{
bSuccess = false;
continue;
}
// Check for executed action but general failure
if (ActionThread.ExitCode != 0)
{
bSuccess = false;
}
// Log CPU time, tool and task.
double ThreadSeconds = Action.Duration.TotalSeconds;
Log.WriteLineIf(bLogDetailedActionStats,
LogEventType.Console,
"^{0}^{1:0.00}^{2}^{3}^{4}",
Action.ActionType.ToString(),
ThreadSeconds,
Action.CommandPath.GetFileName(),
Action.StatusDescription,
Action.bIsUsingPCH);
// Update statistics
switch (Action.ActionType)
{
case ActionType.BuildProject:
TotalBuildProjectTime += ThreadSeconds;
break;
case ActionType.Compile:
TotalCompileTime += ThreadSeconds;
break;
case ActionType.CreateAppBundle:
TotalCreateAppBundleTime += ThreadSeconds;
break;
case ActionType.GenerateDebugInfo:
TotalGenerateDebugInfoTime += ThreadSeconds;
break;
case ActionType.Link:
TotalLinkTime += ThreadSeconds;
break;
default:
TotalOtherActionsTime += ThreadSeconds;
break;
}
// Keep track of total thread seconds spent on tasks.
TotalThreadSeconds += ThreadSeconds;
}
Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "-------- End Detailed Actions Stats -----------------------------------------------------------");
// Log total CPU seconds and numbers of processors involved in tasks.
Log.WriteLineIf(bLogDetailedActionStats,
LogEventType.Console, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds);
// Log detailed stats
Log.WriteLineIf(bLogDetailedActionStats,
LogEventType.Console,
"Cumulative action seconds ({0} processors): {1:0.00} building projects, {2:0.00} compiling, {3:0.00} creating app bundles, {4:0.00} generating debug info, {5:0.00} linking, {6:0.00} other",
System.Environment.ProcessorCount,
TotalBuildProjectTime,
TotalCompileTime,
TotalCreateAppBundleTime,
TotalGenerateDebugInfoTime,
TotalLinkTime,
TotalOtherActionsTime
);
return(bSuccess);
}
static string GetFullIncludePath(string IncludePath)
{
return(Path.GetFullPath(ActionThread.ExpandEnvironmentVariables(IncludePath)));
}
/// <summary>
/// Executes the specified actions locally.
/// </summary>
/// <returns>True if all the tasks successfully executed, or false if any of them failed.</returns>
public override bool ExecuteActions(List <Action> Actions, bool bLogDetailedActionStats)
{
// Time to sleep after each iteration of the loop in order to not busy wait.
const float LoopSleepTime = 0.1f;
// Use WMI to figure out physical cores, excluding hyper threading.
int NumCores = Utils.GetPhysicalProcessorCount();
if (NumCores == -1)
{
NumCores = System.Environment.ProcessorCount;
}
// The number of actions to execute in parallel is trying to keep the CPU busy enough in presence of I/O stalls.
int MaxActionsToExecuteInParallel = 0;
if (NumCores < System.Environment.ProcessorCount && ProcessorCountMultiplier != 1.0)
{
// The CPU has more logical cores than physical ones, aka uses hyper-threading.
// Use multiplier if provided
MaxActionsToExecuteInParallel = (int)(NumCores * ProcessorCountMultiplier);
}
else if (NumCores < System.Environment.ProcessorCount && NumCores > 4)
{
// The CPU has more logical cores than physical ones, aka uses hyper-threading.
// Use average of logical and physical if we have "lots of cores"
MaxActionsToExecuteInParallel = (int)(NumCores + System.Environment.ProcessorCount) / 2;
}
// No hyper-threading. Only kicking off a task per CPU to keep machine responsive.
else
{
MaxActionsToExecuteInParallel = NumCores;
}
if (Utils.IsRunningOnMono)
{
// heuristic: give each action at least 1.5GB of RAM (some clang instances will need more, actually)
long MinMemoryPerActionMB = 3 * 1024 / 2;
long PhysicalRAMAvailableMB = (new PerformanceCounter("Mono Memory", "Total Physical Memory").RawValue) / (1024 * 1024);
int MaxActionsAffordedByMemory = (int)(Math.Max(1, (PhysicalRAMAvailableMB) / MinMemoryPerActionMB));
MaxActionsToExecuteInParallel = Math.Min(MaxActionsToExecuteInParallel, MaxActionsAffordedByMemory);
}
MaxActionsToExecuteInParallel = Math.Max(1, Math.Min(MaxActionsToExecuteInParallel, MaxProcessorCount));
Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel);
Dictionary <Action, ActionThread> ActionThreadDictionary = new Dictionary <Action, ActionThread>();
int JobNumber = 1;
using (ProgressWriter ProgressWriter = new ProgressWriter("Compiling C++ source code...", false))
{
int ProgressValue = 0;
while (true)
{
// Count the number of pending and still executing actions.
int NumUnexecutedActions = 0;
int NumExecutingActions = 0;
foreach (Action Action in Actions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
NumUnexecutedActions++;
}
else if (ActionThread != null)
{
if (ActionThread.bComplete == false)
{
NumUnexecutedActions++;
NumExecutingActions++;
}
}
}
// Update the current progress
int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions;
if (ProgressValue != NewProgressValue)
{
ProgressWriter.Write(ProgressValue, Actions.Count + 1);
ProgressValue = NewProgressValue;
}
// If there aren't any pending actions left, we're done executing.
if (NumUnexecutedActions == 0)
{
break;
}
// If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
// prerequisites.
foreach (Action Action in Actions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel))
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems)
{
if (PrerequisiteItem.ProducingAction != null && Actions.Contains(PrerequisiteItem.ProducingAction))
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue(PrerequisiteItem.ProducingAction, out PrerequisiteProcess);
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
ActionThreadDictionary.Add(Action, null);
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count);
JobNumber++;
try
{
ActionThread.Run();
}
catch (Exception ex)
{
throw new BuildException(ex, "Failed to start thread for action: {0} {1}\r\n{2}", Action.CommandPath, Action.CommandArguments, ex.ToString());
}
ActionThreadDictionary.Add(Action, ActionThread);
NumExecutingActions++;
}
}
}
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime));
}
}
Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "-------- Begin Detailed Action Stats ----------------------------------------------------------");
Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "^Action Type^Duration (seconds)^Tool^Task^Using PCH");
double TotalThreadSeconds = 0;
// Check whether any of the tasks failed and log action stats if wanted.
bool bSuccess = true;
double TotalBuildProjectTime = 0;
double TotalCompileTime = 0;
double TotalCreateAppBundleTime = 0;
double TotalGenerateDebugInfoTime = 0;
double TotalLinkTime = 0;
double TotalOtherActionsTime = 0;
foreach (KeyValuePair <Action, ActionThread> ActionProcess in ActionThreadDictionary)
{
Action Action = ActionProcess.Key;
ActionThread ActionThread = ActionProcess.Value;
// Check for pending actions, preemptive failure
if (ActionThread == null)
{
bSuccess = false;
continue;
}
// Check for executed action but general failure
if (ActionThread.ExitCode != 0)
{
bSuccess = false;
}
// Log CPU time, tool and task.
double ThreadSeconds = Action.Duration.TotalSeconds;
Log.WriteLineIf(bLogDetailedActionStats,
LogEventType.Console,
"^{0}^{1:0.00}^{2}^{3}^{4}",
Action.ActionType.ToString(),
ThreadSeconds,
Path.GetFileName(Action.CommandPath),
Action.StatusDescription,
Action.bIsUsingPCH);
// Update statistics
switch (Action.ActionType)
{
case ActionType.BuildProject:
TotalBuildProjectTime += ThreadSeconds;
break;
case ActionType.Compile:
TotalCompileTime += ThreadSeconds;
break;
case ActionType.CreateAppBundle:
TotalCreateAppBundleTime += ThreadSeconds;
break;
case ActionType.GenerateDebugInfo:
TotalGenerateDebugInfoTime += ThreadSeconds;
break;
case ActionType.Link:
TotalLinkTime += ThreadSeconds;
break;
default:
TotalOtherActionsTime += ThreadSeconds;
break;
}
// Keep track of total thread seconds spent on tasks.
TotalThreadSeconds += ThreadSeconds;
}
Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "-------- End Detailed Actions Stats -----------------------------------------------------------");
// Log total CPU seconds and numbers of processors involved in tasks.
Log.WriteLineIf(bLogDetailedActionStats || UnrealBuildTool.bPrintDebugInfo,
LogEventType.Console, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds);
// Log detailed stats
Log.WriteLineIf(bLogDetailedActionStats || UnrealBuildTool.bPrintDebugInfo,
LogEventType.Console,
"Cumulative action seconds ({0} processors): {1:0.00} building projects, {2:0.00} compiling, {3:0.00} creating app bundles, {4:0.00} generating debug info, {5:0.00} linking, {6:0.00} other",
System.Environment.ProcessorCount,
TotalBuildProjectTime,
TotalCompileTime,
TotalCreateAppBundleTime,
TotalGenerateDebugInfoTime,
TotalLinkTime,
TotalOtherActionsTime
);
return(bSuccess);
}
internal static ExecutionResult ExecuteLocalActions(List<Action> InLocalActions, Dictionary<Action, ActionThread> InActionThreadDictionary, int TotalNumJobs)
{
// Time to sleep after each iteration of the loop in order to not busy wait.
const float LoopSleepTime = 0.1f;
ExecutionResult LocalActionsResult = ExecutionResult.TasksSucceeded;
while (true)
{
// Count the number of pending and still executing actions.
int NumUnexecutedActions = 0;
int NumExecutingActions = 0;
foreach (Action Action in InLocalActions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = InActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
NumUnexecutedActions++;
}
else if (ActionThread != null)
{
if (ActionThread.bComplete == false)
{
NumUnexecutedActions++;
NumExecutingActions++;
}
}
}
// If there aren't any pending actions left, we're done executing.
if (NumUnexecutedActions == 0)
{
break;
}
// If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
// prerequisites.
foreach (Action Action in InLocalActions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = InActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel))
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems)
{
if (PrerequisiteItem.ProducingAction != null && InLocalActions.Contains(PrerequisiteItem.ProducingAction))
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = InActionThreadDictionary.TryGetValue(PrerequisiteItem.ProducingAction, out PrerequisiteProcess);
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
InActionThreadDictionary.Add(Action, null);
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
ActionThread ActionThread = new ActionThread(Action, JobNumber, TotalNumJobs);
ActionThread.Run();
InActionThreadDictionary.Add(Action, ActionThread);
NumExecutingActions++;
JobNumber++;
}
}
}
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime));
}
return LocalActionsResult;
}
/**
* Executes the specified actions locally.
* @return True if all the tasks successfully executed, or false if any of them failed.
*/
public static bool ExecuteActions(List<Action> Actions)
{
// Time to sleep after each iteration of the loop in order to not busy wait.
const float LoopSleepTime = 0.1f;
// Use WMI to figure out physical cores, excluding hyper threading.
int NumCores = 0;
if (!Utils.IsRunningOnMono)
{
foreach(var Item in new System.Management.ManagementObjectSearcher("Select * from Win32_Processor").Get())
{
NumCores += int.Parse(Item["NumberOfCores"].ToString());
}
}
// On some systems this requires a hot fix to work so we fall back to using the (logical) processor count.
if( NumCores == 0 )
{
NumCores = System.Environment.ProcessorCount;
}
// The number of actions to execute in parallel is trying to keep the CPU busy enough in presence of I/O stalls.
int MaxActionsToExecuteInParallel = 0;
// The CPU has more logical cores than physical ones, aka uses hyper-threading.
if( NumCores < System.Environment.ProcessorCount )
{
MaxActionsToExecuteInParallel = (int) (NumCores * BuildConfiguration.ProcessorCountMultiplier);
}
// No hyper-threading. Only kicking off a task per CPU to keep machine responsive.
else
{
MaxActionsToExecuteInParallel = NumCores;
}
if( Utils.IsRunningOnMono )
{
// @todo iosmerge: this should be looking at available memory as well since some of our
// Macs have a poor memory/CPU ratio
MaxActionsToExecuteInParallel /= 2;
}
Dictionary<Action, ActionThread> ActionThreadDictionary = new Dictionary<Action, ActionThread>();
while(true)
{
// Count the number of pending and still executing actions.
int NumUnexecutedActions = 0;
int NumExecutingActions = 0;
foreach (Action Action in Actions)
{
ActionThread ActionThread = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread);
if (bFoundActionProcess == false)
{
NumUnexecutedActions++;
}
else if (ActionThread != null)
{
if (ActionThread.bComplete == false)
{
NumUnexecutedActions++;
NumExecutingActions++;
}
}
}
// If there aren't any pending actions left, we're done executing.
if (NumUnexecutedActions == 0)
{
break;
}
// If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
// prerequisites.
foreach (Action Action in Actions)
{
ActionThread ActionProcess = null;
bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess);
if (bFoundActionProcess == false)
{
if (NumExecutingActions < Math.Max(1,MaxActionsToExecuteInParallel))
{
// Determine whether there are any prerequisites of the action that are outdated.
bool bHasOutdatedPrerequisites = false;
bool bHasFailedPrerequisites = false;
foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems)
{
if (PrerequisiteItem.ProducingAction != null && Actions.Contains(PrerequisiteItem.ProducingAction))
{
ActionThread PrerequisiteProcess = null;
bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue( PrerequisiteItem.ProducingAction, out PrerequisiteProcess );
if (bFoundPrerequisiteProcess == true)
{
if (PrerequisiteProcess == null)
{
bHasFailedPrerequisites = true;
}
else if (PrerequisiteProcess.bComplete == false)
{
bHasOutdatedPrerequisites = true;
}
else if (PrerequisiteProcess.ExitCode != 0)
{
bHasFailedPrerequisites = true;
}
}
else
{
bHasOutdatedPrerequisites = true;
}
}
}
// If there are any failed prerequisites of this action, don't execute it.
if (bHasFailedPrerequisites)
{
// Add a null entry in the dictionary for this action.
ActionThreadDictionary.Add( Action, null );
}
// If there aren't any outdated prerequisites of this action, execute it.
else if (!bHasOutdatedPrerequisites)
{
ActionThread ActionThread = new ActionThread(Action);
ActionThread.Run();
ActionThreadDictionary.Add(Action, ActionThread);
NumExecutingActions++;
}
}
}
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime));
}
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "-------- Begin Detailed Action Stats ----------------------------------------------------------");
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "^Action Type^Duration (seconds)^Tool^Task^Using PCH^Description");
double TotalThreadSeconds = 0;
// Check whether any of the tasks failed and log action stats if wanted.
bool bSuccess = true;
foreach (KeyValuePair<Action, ActionThread> ActionProcess in ActionThreadDictionary)
{
Action Action = ActionProcess.Key;
ActionThread ActionThread = ActionProcess.Value;
// Check for pending actions, preemptive failure
if (ActionThread == null)
{
bSuccess = false;
continue;
}
// Check for executed action but general failure
if (ActionThread.ExitCode != 0)
{
bSuccess = false;
}
// Log CPU time, tool and task.
double ThreadSeconds = Action.Duration.TotalSeconds;
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats,
TraceEventType.Information,
"^{0}^{1:0.00}^{2}^{3}^{4}^{5}",
Action.ActionType.ToString(),
ThreadSeconds,
Path.GetFileName(Action.CommandPath),
Action.StatusDescription,
Action.bIsUsingPCH,
Action.StatusDetailedDescription);
// Update statistics
switch (Action.ActionType)
{
case ActionType.BuildProject:
UnrealBuildTool.TotalBuildProjectTime += ThreadSeconds;
break;
case ActionType.Compile:
UnrealBuildTool.TotalCompileTime += ThreadSeconds;
break;
case ActionType.CreateAppBundle:
UnrealBuildTool.TotalCreateAppBundleTime += ThreadSeconds;
break;
case ActionType.GenerateDebugInfo:
UnrealBuildTool.TotalGenerateDebugInfoTime += ThreadSeconds;
break;
case ActionType.Link:
UnrealBuildTool.TotalLinkTime += ThreadSeconds;
break;
default:
UnrealBuildTool.TotalOtherActionsTime += ThreadSeconds;
break;
}
// Keep track of total thread seconds spent on tasks.
TotalThreadSeconds += ThreadSeconds;
}
Log.TraceInformation("-------- End Detailed Actions Stats -----------------------------------------------------------");
// Log total CPU seconds and numbers of processors involved in tasks.
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats || BuildConfiguration.bPrintDebugInfo,
TraceEventType.Information, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds);
return bSuccess;
}
