public override bool ExecuteActions(List <Action> Actions, bool bLogDetailedActionStats) { bool SNDBSResult = true; if (Actions.Count > 0) { // Generate any needed templates. Can only generate one per executable, so just use the first Action for reference IEnumerable <Action> CommandPaths = Actions.GroupBy(a => a.CommandPath).Select(g => g.FirstOrDefault()).ToList(); foreach (Action CommandPath in CommandPaths) { PrepareToolTemplate(CommandPath); } // Generate include-rewrite-rules.ini. GenerateSNDBSIncludeRewriteRules(); // Use WMI to figure out physical cores, excluding hyper threading. int NumCores = Utils.GetPhysicalProcessorCount(); // If we failed to detect the number of cores, default to the logical processor count if (NumCores == -1) { NumCores = System.Environment.ProcessorCount; } // The number of actions to execute in parallel is trying to keep the CPU busy enough in presence of I/O stalls. MaxActionsToExecuteInParallel = 0; // The CPU has more logical cores than physical ones, aka uses hyper-threading. if (NumCores < System.Environment.ProcessorCount) { MaxActionsToExecuteInParallel = (int)(NumCores * ProcessorCountMultiplier); } // No hyper-threading. Only kicking off a task per CPU to keep machine responsive. else { MaxActionsToExecuteInParallel = NumCores; } MaxActionsToExecuteInParallel = Math.Min(MaxActionsToExecuteInParallel, MaxProcessorCount); JobNumber = 1; Dictionary <Action, ActionThread> ActionThreadDictionary = new Dictionary <Action, ActionThread>(); while (true) { bool bUnexecutedActions = false; foreach (Action Action in Actions) { ActionThread ActionThread = null; bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread); if (bFoundActionProcess == false) { bUnexecutedActions = true; if (!ExecuteActions(Actions, ActionThreadDictionary)) { return(false); } break; } } if (bUnexecutedActions == false) { break; } } Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "-------- Begin Detailed Action Stats ----------------------------------------------------------"); Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "^Action Type^Duration (seconds)^Tool^Task^Using PCH"); double TotalThreadSeconds = 0; // Check whether any of the tasks failed and log action stats if wanted. foreach (KeyValuePair <Action, ActionThread> ActionProcess in ActionThreadDictionary) { Action Action = ActionProcess.Key; ActionThread ActionThread = ActionProcess.Value; // Check for pending actions, preemptive failure if (ActionThread == null) { SNDBSResult = false; continue; } // Check for executed action but general failure if (ActionThread.ExitCode != 0) { SNDBSResult = false; } // Log CPU time, tool and task. double ThreadSeconds = Action.Duration.TotalSeconds; Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "^{0}^{1:0.00}^{2}^{3}", Action.ActionType.ToString(), ThreadSeconds, Action.CommandPath.GetFileName(), Action.StatusDescription); // Keep track of total thread seconds spent on tasks. TotalThreadSeconds += ThreadSeconds; } Log.TraceInformation("-------- End Detailed Actions Stats -----------------------------------------------------------"); // Log total CPU seconds and numbers of processors involved in tasks. Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds); } // Delete the include-rewrite-rules.ini if it was generated. if (File.Exists(IncludeRewriteRulesFile.FullName)) { File.Delete(IncludeRewriteRulesFile.FullName); } return(SNDBSResult); }
internal bool ExecuteActions(List <Action> InActions, Dictionary <Action, ActionThread> InActionThreadDictionary) { // Build the script file that will be executed by SN-DBS StreamWriter ScriptFile; string ScriptFilename = Path.Combine(UnrealBuildTool.EngineDirectory.FullName, "Intermediate", "Build", "SNDBS.bat"); FileStream ScriptFileStream = new FileStream(ScriptFilename, FileMode.Create, FileAccess.ReadWrite, FileShare.Read); ScriptFile = new StreamWriter(ScriptFileStream); ScriptFile.AutoFlush = true; int NumScriptedActions = 0; List <Action> LocalActions = new List <Action>(); ActionThread DummyActionThread = new ActionThread(null, 1, 1); foreach (Action Action in InActions) { ActionThread ActionProcess = null; bool bFoundActionProcess = InActionThreadDictionary.TryGetValue(Action, out ActionProcess); if (bFoundActionProcess == false) { // Determine whether there are any prerequisites of the action that are outdated. bool bHasOutdatedPrerequisites = false; bool bHasFailedPrerequisites = false; foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems) { if (PrerequisiteItem.ProducingAction != null && InActions.Contains(PrerequisiteItem.ProducingAction)) { ActionThread PrerequisiteProcess = null; bool bFoundPrerequisiteProcess = InActionThreadDictionary.TryGetValue(PrerequisiteItem.ProducingAction, out PrerequisiteProcess); if (bFoundPrerequisiteProcess == true) { if (PrerequisiteProcess == null) { bHasFailedPrerequisites = true; } else if (PrerequisiteProcess.bComplete == false) { bHasOutdatedPrerequisites = true; } else if (PrerequisiteProcess.ExitCode != 0) { bHasFailedPrerequisites = true; } } else { bHasOutdatedPrerequisites = true; } } } // If there are any failed prerequisites of this action, don't execute it. if (bHasFailedPrerequisites) { // Add a null entry in the dictionary for this action. InActionThreadDictionary.Add(Action, null); } // If there aren't any outdated prerequisites of this action, execute it. else if (!bHasOutdatedPrerequisites) { if (Action.bCanExecuteRemotely == false || Action.bCanExecuteRemotelyWithSNDBS == false) { // Execute locally LocalActions.Add(Action); } else { // Add to script for execution by SN-DBS string NewCommandArguments = "\"" + Action.CommandPath + "\"" + " " + Action.CommandArguments; ScriptFile.WriteLine(ActionThread.ExpandEnvironmentVariables(NewCommandArguments)); InActionThreadDictionary.Add(Action, DummyActionThread); Action.StartTime = Action.EndTime = DateTimeOffset.Now; Log.TraceInformation("[{0}/{1}] {2} {3}", JobNumber, InActions.Count, Action.CommandDescription, Action.StatusDescription); JobNumber++; NumScriptedActions++; } } } } ScriptFile.Flush(); ScriptFile.Close(); ScriptFile.Dispose(); ScriptFile = null; if (NumScriptedActions > 0) { // Create the process string SCERoot = Environment.GetEnvironmentVariable("SCE_ROOT_DIR"); string SNDBSExecutable = Path.Combine(SCERoot, "Common/SN-DBS/bin/dbsbuild.exe"); ProcessStartInfo PSI = new ProcessStartInfo(SNDBSExecutable, String.Format("-q -p UE4 -s \"{0}\"", FileReference.Combine(UnrealBuildTool.EngineDirectory, "Intermediate", "Build", "sndbs.bat").FullName)); PSI.RedirectStandardOutput = true; PSI.RedirectStandardError = true; PSI.UseShellExecute = false; PSI.CreateNoWindow = true; PSI.WorkingDirectory = Path.GetFullPath(".");; Process NewProcess = new Process(); NewProcess.StartInfo = PSI; NewProcess.OutputDataReceived += new DataReceivedEventHandler(ActionDebugOutput); NewProcess.ErrorDataReceived += new DataReceivedEventHandler(ActionDebugOutput); DateTimeOffset StartTime = DateTimeOffset.Now; NewProcess.Start(); NewProcess.BeginOutputReadLine(); NewProcess.BeginErrorReadLine(); NewProcess.WaitForExit(); TimeSpan Duration; DateTimeOffset EndTime = DateTimeOffset.Now; if (EndTime == DateTimeOffset.MinValue) { Duration = DateTimeOffset.Now - StartTime; } else { Duration = EndTime - StartTime; } DummyActionThread.bComplete = true; int ExitCode = NewProcess.ExitCode; if (ExitCode != 0) { return(false); } } // Execute local tasks if (LocalActions.Count > 0) { return(ExecuteLocalActions(LocalActions, InActionThreadDictionary, InActions.Count)); } return(true); }
internal bool ExecuteActions(List <Action> InActions, Dictionary <Action, ActionThread> InActionThreadDictionary) { // Build the script file that will be executed by SN-DBS StreamWriter ScriptFile; string ScriptFilename = Path.Combine(UnrealBuildTool.EngineDirectory.FullName, "Intermediate", "Build", "SNDBS.bat"); FileStream ScriptFileStream = new FileStream(ScriptFilename, FileMode.Create, FileAccess.ReadWrite, FileShare.Read); ScriptFile = new StreamWriter(ScriptFileStream); ScriptFile.AutoFlush = true; int NumScriptedActions = 0; List <Action> LocalActions = new List <Action>(); ActionThread DummyActionThread = new ActionThread(null, 1, 1); bool PrintDebugInfo = false; foreach (Action Action in InActions) { ActionThread ActionProcess = null; bool bFoundActionProcess = InActionThreadDictionary.TryGetValue(Action, out ActionProcess); if (bFoundActionProcess == false) { // Determine whether there are any prerequisites of the action that are outdated. bool bHasOutdatedPrerequisites = false; bool bHasFailedPrerequisites = false; foreach (Action PrerequisiteAction in Action.PrerequisiteActions) { if (InActions.Contains(PrerequisiteAction)) { ActionThread PrerequisiteProcess = null; bool bFoundPrerequisiteProcess = InActionThreadDictionary.TryGetValue(PrerequisiteAction, out PrerequisiteProcess); if (bFoundPrerequisiteProcess == true) { if (PrerequisiteProcess == null) { bHasFailedPrerequisites = true; } else if (PrerequisiteProcess.bComplete == false) { bHasOutdatedPrerequisites = true; } else if (PrerequisiteProcess.ExitCode != 0) { bHasFailedPrerequisites = true; } } else { bHasOutdatedPrerequisites = true; } } } // If there are any failed prerequisites of this action, don't execute it. if (bHasFailedPrerequisites) { // Add a null entry in the dictionary for this action. InActionThreadDictionary.Add(Action, null); } // If there aren't any outdated prerequisites of this action, execute it. else if (!bHasOutdatedPrerequisites) { if (Action.bCanExecuteRemotely == false || Action.bCanExecuteRemotelyWithSNDBS == false) { // Execute locally LocalActions.Add(Action); } else { // Create a dummy force-included file which references PCH files, so that SN-DBS knows they are dependencies. string AdditionalStubIncludes = ""; if (Action.CommandPath.GetFileName().Equals("cl.exe", StringComparison.OrdinalIgnoreCase)) { string ResponseFile = Action.CommandArguments.Replace("\"", "").Replace("@", "").Trim(); StringBuilder WrapperContents = new StringBuilder(); using (StringWriter Writer = new StringWriter(WrapperContents)) { Writer.WriteLine("// PCH dependencies for {0}", ResponseFile); Writer.WriteLine("#if 0"); foreach (FileItem Preqrequisite in Action.PrerequisiteItems) { if (Preqrequisite.AbsolutePath.EndsWith(".pch")) { Writer.WriteLine("#include \"{0}\"", Preqrequisite.AbsolutePath.Replace(".pch", ".obj")); } } Writer.WriteLine("#endif"); } FileItem DummyResponseFileDependency = FileItem.CreateIntermediateTextFile(new FileReference(ResponseFile + ".dummy.h"), WrapperContents.ToString()); AdditionalStubIncludes = string.Format("/FI\"{0}\"", DummyResponseFileDependency); } // Add to script for execution by SN-DBS string NewCommandArguments = "\"" + Action.CommandPath + "\"" + " " + AdditionalStubIncludes + " " + Action.CommandArguments; ScriptFile.WriteLine(NewCommandArguments); InActionThreadDictionary.Add(Action, DummyActionThread); Action.StartTime = Action.EndTime = DateTimeOffset.Now; Log.TraceInformation("[{0}/{1}] {2} {3}", JobNumber, InActions.Count, Action.CommandDescription, Action.StatusDescription); JobNumber++; NumScriptedActions++; PrintDebugInfo |= Action.bPrintDebugInfo; if (Action.DependencyListFile != null && File.Exists(Action.DependencyListFile.AbsolutePath)) { Log.TraceVerbose("Deleting dependency list file {0}", Action.DependencyListFile.AbsolutePath); File.Delete(Action.DependencyListFile.AbsolutePath); } } } } } ScriptFile.Flush(); ScriptFile.Close(); ScriptFile.Dispose(); ScriptFile = null; if (NumScriptedActions > 0) { // Create the process string SCERoot = Environment.GetEnvironmentVariable("SCE_ROOT_DIR"); string SNDBSExecutable = Path.Combine(SCERoot, "Common/SN-DBS/bin/dbsbuild.exe"); DirectoryReference TemplatesDir = DirectoryReference.Combine(UnrealBuildTool.EngineDirectory, "Programs", "UnrealBuildTool", "SndbsTemplates"); string IncludeRewriteRulesArg = String.Format("--include-rewrite-rules \"{0}\"", IncludeRewriteRulesFile.FullName); string VerbosityLevel = PrintDebugInfo ? "-v" : "-q"; ProcessStartInfo PSI = new ProcessStartInfo(SNDBSExecutable, String.Format("{0} -p UE4 -s \"{1}\" -templates \"{2}\" {3}", VerbosityLevel, FileReference.Combine(UnrealBuildTool.EngineDirectory, "Intermediate", "Build", "sndbs.bat").FullName, TemplatesDir.FullName, IncludeRewriteRulesArg)); PSI.RedirectStandardOutput = true; PSI.RedirectStandardError = true; PSI.UseShellExecute = false; PSI.CreateNoWindow = true; PSI.WorkingDirectory = Path.GetFullPath("."); Process NewProcess = new Process(); NewProcess.StartInfo = PSI; NewProcess.OutputDataReceived += new DataReceivedEventHandler(ActionDebugOutput); NewProcess.ErrorDataReceived += new DataReceivedEventHandler(ActionDebugOutput); DateTimeOffset StartTime = DateTimeOffset.Now; NewProcess.Start(); NewProcess.BeginOutputReadLine(); NewProcess.BeginErrorReadLine(); NewProcess.WaitForExit(); TimeSpan Duration; DateTimeOffset EndTime = DateTimeOffset.Now; if (EndTime == DateTimeOffset.MinValue) { Duration = DateTimeOffset.Now - StartTime; } else { Duration = EndTime - StartTime; } DummyActionThread.bComplete = true; int ExitCode = NewProcess.ExitCode; if (ExitCode != 0) { return(false); } } // Execute local tasks if (LocalActions.Count > 0) { return(ExecuteLocalActions(LocalActions, InActionThreadDictionary, InActions.Count)); } return(true); }
public override FileItem[] LinkAllFiles(LinkEnvironment LinkEnvironment, bool bBuildImportLibraryOnly) { List<FileItem> Outputs = new List<FileItem>(); var NDKRoot = Environment.GetEnvironmentVariable("NDKROOT").Replace("\\", "/"); int NDKApiLevelInt = GetNdkApiLevelInt(); string OptionalLinkArguments; for (int ArchIndex = 0; ArchIndex < Arches.Count; ArchIndex++) { string Arch = Arches[ArchIndex]; // 32-bit ABI may need fixup for removed bsd_signal in NDK11 for android-21+ OptionalLinkArguments = ""; if (NDKApiLevelInt >= 21) { // this file was added in NDK11 so use existence to detect (RELEASE.TXT no longer present) if (File.Exists(Path.Combine(NDKRoot, "source.properties"))) { switch (Arch) { case "-armv7": OptionalLinkArguments = string.Format(" \"{0}\"", Path.Combine(UnrealBuildTool.EngineDirectory.FullName, "Build/Android/Prebuilt/bsdsignal/lib/armeabi-v7a/libbsdsignal.a")); break; case "-x86": OptionalLinkArguments = string.Format(" \"{0}\"", Path.Combine(UnrealBuildTool.EngineDirectory.FullName, "Build/Android/Prebuilt/bsdsignal/lib/x86/libbsdsignal.a")); break; } } } for (int GPUArchIndex = 0; GPUArchIndex < GPUArchitectures.Count; GPUArchIndex++) { string GPUArchitecture = GPUArchitectures[GPUArchIndex]; int OutputPathIndex = ArchIndex * GPUArchitectures.Count + GPUArchIndex; // Android will have an array of outputs if (LinkEnvironment.Config.OutputFilePaths.Count < OutputPathIndex || !LinkEnvironment.Config.OutputFilePaths[OutputPathIndex].GetFileNameWithoutExtension().EndsWith(Arch + GPUArchitecture)) { throw new BuildException("The OutputFilePaths array didn't match the Arches array in AndroidToolChain.LinkAllFiles"); } // Create an action that invokes the linker. Action LinkAction = new Action(ActionType.Link); LinkAction.WorkingDirectory = UnrealBuildTool.EngineSourceDirectory.FullName; if (LinkEnvironment.Config.bIsBuildingLibrary) { switch (Arch) { case "-armv7": LinkAction.CommandPath = ArPathArm; break; case "-arm64": LinkAction.CommandPath = ArPathArm64; break; case "-x86": LinkAction.CommandPath = ArPathx86; ; break; case "-x64": LinkAction.CommandPath = ArPathx64; ; break; default: LinkAction.CommandPath = ArPathArm; ; break; } } else { LinkAction.CommandPath = ClangPath; } string LinkerPath = LinkAction.WorkingDirectory; LinkAction.WorkingDirectory = LinkEnvironment.Config.IntermediateDirectory.FullName; // Get link arguments. LinkAction.CommandArguments = LinkEnvironment.Config.bIsBuildingLibrary ? GetArArguments(LinkEnvironment) : GetLinkArguments(LinkEnvironment, Arch); // Add the output file as a production of the link action. FileItem OutputFile; OutputFile = FileItem.GetItemByFileReference(LinkEnvironment.Config.OutputFilePaths[OutputPathIndex]); Outputs.Add(OutputFile); LinkAction.ProducedItems.Add(OutputFile); LinkAction.StatusDescription = string.Format("{0}", Path.GetFileName(OutputFile.AbsolutePath)); // LinkAction.bPrintDebugInfo = true; // Add the output file to the command-line. if (LinkEnvironment.Config.bIsBuildingLibrary) { LinkAction.CommandArguments += string.Format(" \"{0}\"", OutputFile.AbsolutePath); } else { LinkAction.CommandArguments += string.Format(" -o \"{0}\"", OutputFile.AbsolutePath); } // Add the input files to a response file, and pass the response file on the command-line. List<string> InputFileNames = new List<string>(); foreach (FileItem InputFile in LinkEnvironment.InputFiles) { // make sure it's for current Arch if (Path.GetFileNameWithoutExtension(InputFile.AbsolutePath).EndsWith(Arch + GPUArchitecture)) { string AbsolutePath = InputFile.AbsolutePath.Replace("\\", "/"); AbsolutePath = AbsolutePath.Replace(LinkEnvironment.Config.IntermediateDirectory.FullName.Replace("\\", "/"), ""); AbsolutePath = AbsolutePath.TrimStart(new char[] { '/' }); InputFileNames.Add(string.Format("\"{0}\"", AbsolutePath)); LinkAction.PrerequisiteItems.Add(InputFile); } } FileReference ResponseFileName = GetResponseFileName(LinkEnvironment, OutputFile); LinkAction.CommandArguments += string.Format(" @\"{0}\"", ResponseFile.Create(ResponseFileName, InputFileNames)); // libs don't link in other libs if (!LinkEnvironment.Config.bIsBuildingLibrary) { // Add the library paths to the argument list. foreach (string LibraryPath in LinkEnvironment.Config.LibraryPaths) { // LinkerPaths could be relative or absolute string AbsoluteLibraryPath = ActionThread.ExpandEnvironmentVariables(LibraryPath); if (IsDirectoryForArch(AbsoluteLibraryPath, Arch)) { // environment variables aren't expanded when using the $( style if (Path.IsPathRooted(AbsoluteLibraryPath) == false) { AbsoluteLibraryPath = Path.Combine(LinkerPath, AbsoluteLibraryPath); } LinkAction.CommandArguments += string.Format(" -L\"{0}\"", AbsoluteLibraryPath); } } // add libraries in a library group LinkAction.CommandArguments += string.Format(" -Wl,--start-group"); foreach (string AdditionalLibrary in LinkEnvironment.Config.AdditionalLibraries) { if (!ShouldSkipLib(AdditionalLibrary, Arch, GPUArchitecture)) { if (String.IsNullOrEmpty(Path.GetDirectoryName(AdditionalLibrary))) { LinkAction.CommandArguments += string.Format(" \"-l{0}\"", AdditionalLibrary); } else { // full pathed libs are compiled by us, so we depend on linking them LinkAction.CommandArguments += string.Format(" \"{0}\"", Path.GetFullPath(AdditionalLibrary)); LinkAction.PrerequisiteItems.Add(FileItem.GetItemByPath(AdditionalLibrary)); } } } LinkAction.CommandArguments += OptionalLinkArguments; LinkAction.CommandArguments += string.Format(" -Wl,--end-group"); } // Add the additional arguments specified by the environment. LinkAction.CommandArguments += LinkEnvironment.Config.AdditionalArguments; LinkAction.CommandArguments = LinkAction.CommandArguments.Replace("\\", "/"); // Only execute linking on the local PC. LinkAction.bCanExecuteRemotely = false; } } return Outputs.ToArray(); }
public override CPPOutput CompileCPPFiles(UEBuildTarget Target, CPPEnvironment CompileEnvironment, List<FileItem> SourceFiles, string ModuleName) { if (Arches.Count == 0) { throw new BuildException("At least one architecture (armv7, x86, etc) needs to be selected in the project settings to build"); } if (!bHasPrintedApiLevel) { Console.WriteLine("Compiling Native code with NDK API '{0}'", GetNdkApiLevel()); bHasPrintedApiLevel = true; } string BaseArguments = ""; if (CompileEnvironment.Config.PrecompiledHeaderAction != PrecompiledHeaderAction.Create) { BaseArguments += " -Werror"; } // Directly added NDK files for NDK extensions ConditionallyAddNDKSourceFiles(SourceFiles, ModuleName); // Deal with dynamic modules removed by architecture GenerateEmptyLinkFunctionsForRemovedModules(SourceFiles, ModuleName, CompileEnvironment.Config.OutputDirectory); // Add preprocessor definitions to the argument list. foreach (string Definition in CompileEnvironment.Config.Definitions) { BaseArguments += string.Format(" -D \"{0}\"", Definition); } var BuildPlatform = UEBuildPlatform.GetBuildPlatformForCPPTargetPlatform(CompileEnvironment.Config.Target.Platform); var NDKRoot = Environment.GetEnvironmentVariable("NDKROOT").Replace("\\", "/"); string BasePCHName = ""; var PCHExtension = UEBuildPlatform.GetBuildPlatform(UnrealTargetPlatform.Android).GetBinaryExtension(UEBuildBinaryType.PrecompiledHeader); if (CompileEnvironment.Config.PrecompiledHeaderAction == PrecompiledHeaderAction.Include) { BasePCHName = RemoveArchName(CompileEnvironment.PrecompiledHeaderFile.AbsolutePath).Replace(PCHExtension, ""); } // Create a compile action for each source file. CPPOutput Result = new CPPOutput(); foreach (string Arch in Arches) { if (ShouldSkipModule(ModuleName, Arch)) { continue; } foreach (string GPUArchitecture in GPUArchitectures) { // which toolchain to use string Arguments = GetCLArguments_Global(CompileEnvironment, Arch) + BaseArguments; switch (Arch) { case "-armv7": Arguments += " -DPLATFORM_64BITS=0 -DPLATFORM_ANDROID_ARM=1"; break; case "-arm64": Arguments += " -DPLATFORM_64BITS=1 -DPLATFORM_ANDROID_ARM64=1"; break; case "-x86": Arguments += " -DPLATFORM_64BITS=0 -DPLATFORM_ANDROID_X86=1"; break; case "-x64": Arguments += " -DPLATFORM_64BITS=1 -DPLATFORM_ANDROID_X64=1"; break; default: Arguments += " -DPLATFORM_64BITS=0 -DPLATFORM_ANDROID_ARM=1"; break; } if (GPUArchitecture == "-esdeferred") { Arguments += " -DPLATFORM_ANDROIDESDEFERRED=1"; } // which PCH file to include string PCHArguments = ""; if (CompileEnvironment.Config.PrecompiledHeaderAction == PrecompiledHeaderAction.Include) { // Add the precompiled header file's path to the include path so Clang can find it. // This needs to be before the other include paths to ensure Clang uses it instead of the source header file. PCHArguments += string.Format(" -include \"{0}\"", InlineArchName(BasePCHName, Arch, GPUArchitecture)); } // Add include paths to the argument list (filtered by architecture) foreach (string IncludePath in CompileEnvironment.Config.CPPIncludeInfo.SystemIncludePaths) { if (IsDirectoryForArch(IncludePath, Arch)) { Arguments += string.Format(" -I\"{0}\"", IncludePath); } } foreach (string IncludePath in CompileEnvironment.Config.CPPIncludeInfo.IncludePaths) { if (IsDirectoryForArch(IncludePath, Arch)) { Arguments += string.Format(" -I\"{0}\"", IncludePath); } } foreach (FileItem SourceFile in SourceFiles) { Action CompileAction = new Action(ActionType.Compile); string FileArguments = ""; bool bIsPlainCFile = Path.GetExtension(SourceFile.AbsolutePath).ToUpperInvariant() == ".C"; bool bDisableShadowWarning = false; // should we disable optimizations on this file? // @todo android - We wouldn't need this if we could disable optimizations per function (via pragma) bool bDisableOptimizations = false;// SourceFile.AbsolutePath.ToUpperInvariant().IndexOf("\\SLATE\\") != -1; if (bDisableOptimizations && CompileEnvironment.Config.Target.Configuration != CPPTargetConfiguration.Debug) { Log.TraceWarning("Disabling optimizations on {0}", SourceFile.AbsolutePath); } bDisableOptimizations = bDisableOptimizations || CompileEnvironment.Config.Target.Configuration == CPPTargetConfiguration.Debug; // Add C or C++ specific compiler arguments. if (CompileEnvironment.Config.PrecompiledHeaderAction == PrecompiledHeaderAction.Create) { FileArguments += GetCompileArguments_PCH(bDisableOptimizations); } else if (bIsPlainCFile) { FileArguments += GetCompileArguments_C(bDisableOptimizations); // remove shadow variable warnings for NDK files if (SourceFile.AbsolutePath.Replace("\\", "/").StartsWith(NDKRoot)) { bDisableShadowWarning = true; } } else { FileArguments += GetCompileArguments_CPP(bDisableOptimizations); // only use PCH for .cpp files FileArguments += PCHArguments; } // Add the C++ source file and its included files to the prerequisite item list. AddPrerequisiteSourceFile(Target, BuildPlatform, CompileEnvironment, SourceFile, CompileAction.PrerequisiteItems); if (CompileEnvironment.Config.PrecompiledHeaderAction == PrecompiledHeaderAction.Create) { // Add the precompiled header file to the produced item list. FileItem PrecompiledHeaderFile = FileItem.GetItemByFileReference( FileReference.Combine( CompileEnvironment.Config.OutputDirectory, Path.GetFileName(InlineArchName(SourceFile.AbsolutePath, Arch, GPUArchitecture) + PCHExtension) ) ); CompileAction.ProducedItems.Add(PrecompiledHeaderFile); Result.PrecompiledHeaderFile = PrecompiledHeaderFile; // Add the parameters needed to compile the precompiled header file to the command-line. FileArguments += string.Format(" -o \"{0}\"", PrecompiledHeaderFile.AbsolutePath, false); } else { if (CompileEnvironment.Config.PrecompiledHeaderAction == PrecompiledHeaderAction.Include) { CompileAction.bIsUsingPCH = true; FileItem ArchPrecompiledHeaderFile = FileItem.GetItemByPath(InlineArchName(BasePCHName, Arch, GPUArchitecture) + PCHExtension); CompileAction.PrerequisiteItems.Add(ArchPrecompiledHeaderFile); } var ObjectFileExtension = UEBuildPlatform.GetBuildPlatform(UnrealTargetPlatform.Android).GetBinaryExtension(UEBuildBinaryType.Object); // Add the object file to the produced item list. FileItem ObjectFile = FileItem.GetItemByFileReference( FileReference.Combine( CompileEnvironment.Config.OutputDirectory, InlineArchName(Path.GetFileName(SourceFile.AbsolutePath) + ObjectFileExtension, Arch, GPUArchitecture) ) ); CompileAction.ProducedItems.Add(ObjectFile); Result.ObjectFiles.Add(ObjectFile); FileArguments += string.Format(" -o \"{0}\"", ObjectFile.AbsolutePath, false); } // Add the source file path to the command-line. FileArguments += string.Format(" \"{0}\"", SourceFile.AbsolutePath); // Build a full argument list string AllArguments = Arguments + FileArguments + CompileEnvironment.Config.AdditionalArguments; AllArguments = ActionThread.ExpandEnvironmentVariables(AllArguments); AllArguments = AllArguments.Replace("\\", "/"); // Remove shadow warning for this file if requested if (bDisableShadowWarning) { int WarningIndex = AllArguments.IndexOf(" -Wshadow"); if (WarningIndex > 0) { AllArguments = AllArguments.Remove(WarningIndex, 9); } } // Create the response file FileReference ResponseFileName = CompileAction.ProducedItems[0].Reference + "_" + AllArguments.GetHashCode().ToString("X") + ".response"; string ResponseArgument = string.Format("@\"{0}\"", ResponseFile.Create(ResponseFileName, new List<string> { AllArguments }).FullName); CompileAction.WorkingDirectory = UnrealBuildTool.EngineSourceDirectory.FullName; CompileAction.CommandPath = ClangPath; CompileAction.CommandArguments = ResponseArgument; CompileAction.StatusDescription = string.Format("{0} [{1}-{2}]", Path.GetFileName(SourceFile.AbsolutePath), Arch.Replace("-", ""), GPUArchitecture.Replace("-", "")); // VC++ always outputs the source file name being compiled, so we don't need to emit this ourselves CompileAction.bShouldOutputStatusDescription = true; // Don't farm out creation of pre-compiled headers as it is the critical path task. CompileAction.bCanExecuteRemotely = CompileEnvironment.Config.PrecompiledHeaderAction != PrecompiledHeaderAction.Create || BuildConfiguration.bAllowRemotelyCompiledPCHs; } } } return Result; }
internal static ExecutionResult ExecuteActions(List<Action> InActions, Dictionary<Action, ActionThread> InActionThreadDictionary) { // Build the script file that will be executed by SN-DBS StreamWriter ScriptFile; string ScriptFilename = Path.Combine(BuildConfiguration.BaseIntermediatePath, "SNDBS.bat"); FileStream ScriptFileStream = new FileStream(ScriptFilename, FileMode.Create, FileAccess.ReadWrite, FileShare.Read); ScriptFile = new StreamWriter(ScriptFileStream); ScriptFile.AutoFlush = true; int NumScriptedActions = 0; List<Action> LocalActions = new List<Action>(); ActionThread DummyActionThread = new ActionThread(null, 1, 1); foreach (Action Action in InActions) { ActionThread ActionProcess = null; bool bFoundActionProcess = InActionThreadDictionary.TryGetValue(Action, out ActionProcess); if (bFoundActionProcess == false) { // Determine whether there are any prerequisites of the action that are outdated. bool bHasOutdatedPrerequisites = false; bool bHasFailedPrerequisites = false; foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems) { if (PrerequisiteItem.ProducingAction != null && InActions.Contains(PrerequisiteItem.ProducingAction)) { ActionThread PrerequisiteProcess = null; bool bFoundPrerequisiteProcess = InActionThreadDictionary.TryGetValue(PrerequisiteItem.ProducingAction, out PrerequisiteProcess); if (bFoundPrerequisiteProcess == true) { if (PrerequisiteProcess == null) { bHasFailedPrerequisites = true; } else if (PrerequisiteProcess.bComplete == false) { bHasOutdatedPrerequisites = true; } else if (PrerequisiteProcess.ExitCode != 0) { bHasFailedPrerequisites = true; } } else { bHasOutdatedPrerequisites = true; } } } // If there are any failed prerequisites of this action, don't execute it. if (bHasFailedPrerequisites) { // Add a null entry in the dictionary for this action. InActionThreadDictionary.Add(Action, null); } // If there aren't any outdated prerequisites of this action, execute it. else if (!bHasOutdatedPrerequisites) { if (Action.bCanExecuteRemotely == false) { // Execute locally LocalActions.Add(Action); } else { // Add to script for execution by SN-DBS string NewCommandArguments = "\"" + Action.CommandPath + "\"" + " " + Action.CommandArguments; ScriptFile.WriteLine(ActionThread.ExpandEnvironmentVariables(NewCommandArguments)); InActionThreadDictionary.Add(Action, DummyActionThread); Action.StartTime = Action.EndTime = DateTimeOffset.Now; Log.TraceInformation("[{0}/{1}] {2} {3}", JobNumber, InActions.Count, Action.CommandDescription, Action.StatusDescription); JobNumber++; NumScriptedActions++; } } } } ScriptFile.Flush(); ScriptFile.Close(); ScriptFile.Dispose(); ScriptFile = null; if( NumScriptedActions > 0 ) { // Create the process ProcessStartInfo PSI = new ProcessStartInfo("dbsbuild", "-q -p UE4 -s " + BuildConfiguration.BaseIntermediatePath + "/sndbs.bat"); PSI.RedirectStandardOutput = true; PSI.RedirectStandardError = true; PSI.UseShellExecute = false; PSI.CreateNoWindow = true; PSI.WorkingDirectory = Path.GetFullPath("."); ; Process NewProcess = new Process(); NewProcess.StartInfo = PSI; NewProcess.OutputDataReceived += new DataReceivedEventHandler(ActionDebugOutput); NewProcess.ErrorDataReceived += new DataReceivedEventHandler(ActionDebugOutput); DateTimeOffset StartTime = DateTimeOffset.Now; NewProcess.Start(); NewProcess.BeginOutputReadLine(); NewProcess.BeginErrorReadLine(); NewProcess.WaitForExit(); TimeSpan Duration; DateTimeOffset EndTime = DateTimeOffset.Now; if (EndTime == DateTimeOffset.MinValue) { Duration = DateTimeOffset.Now - StartTime; } else { Duration = EndTime - StartTime; } DummyActionThread.bComplete = true; int ExitCode = NewProcess.ExitCode; if (ExitCode != 0) { return ExecutionResult.TasksFailed; } else { UnrealBuildTool.TotalCompileTime += Duration.TotalSeconds; } } // Execute local tasks if( LocalActions.Count > 0 ) { return ExecuteLocalActions(LocalActions, InActionThreadDictionary, InActions.Count); } return ExecutionResult.TasksSucceeded; }
/** * Executes the specified actions locally. * @return True if all the tasks successfully executed, or false if any of them failed. */ public static bool ExecuteActions(List <Action> Actions) { // Time to sleep after each iteration of the loop in order to not busy wait. const float LoopSleepTime = 0.1f; // Use WMI to figure out physical cores, excluding hyper threading. int NumCores = 0; if (!Utils.IsRunningOnMono) { try { using (var Mos = new System.Management.ManagementObjectSearcher("Select * from Win32_Processor")) { var MosCollection = Mos.Get(); foreach (var Item in MosCollection) { NumCores += int.Parse(Item["NumberOfCores"].ToString()); } } } catch (Exception Ex) { Log.TraceWarning("Unable to get the number of Cores: {0}", Ex.ToString()); Log.TraceWarning("Falling back to processor count."); } } // On some systems this requires a hot fix to work so we fall back to using the (logical) processor count. if (NumCores == 0) { NumCores = System.Environment.ProcessorCount; } // The number of actions to execute in parallel is trying to keep the CPU busy enough in presence of I/O stalls. int MaxActionsToExecuteInParallel = 0; // The CPU has more logical cores than physical ones, aka uses hyper-threading. if (NumCores < System.Environment.ProcessorCount) { MaxActionsToExecuteInParallel = (int)(NumCores * BuildConfiguration.ProcessorCountMultiplier); } // No hyper-threading. Only kicking off a task per CPU to keep machine responsive. else { MaxActionsToExecuteInParallel = NumCores; } if (Utils.IsRunningOnMono) { // heuristic: give each action at least 1.5GB of RAM (some clang instances will need more, actually) long MinMemoryPerActionMB = 3 * 1024 / 2; long PhysicalRAMAvailableMB = (new PerformanceCounter("Mono Memory", "Total Physical Memory").RawValue) / (1024 * 1024); int MaxActionsAffordedByMemory = (int)(Math.Max(1, (PhysicalRAMAvailableMB) / MinMemoryPerActionMB)); MaxActionsToExecuteInParallel = Math.Min(MaxActionsToExecuteInParallel, MaxActionsAffordedByMemory); } MaxActionsToExecuteInParallel = Math.Min(MaxActionsToExecuteInParallel, BuildConfiguration.MaxProcessorCount); Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel); Dictionary <Action, ActionThread> ActionThreadDictionary = new Dictionary <Action, ActionThread>(); int JobNumber = 1; using (ProgressWriter ProgressWriter = new ProgressWriter("Compiling source code...", false)) { int ProgressValue = 0; while (true) { // Count the number of pending and still executing actions. int NumUnexecutedActions = 0; int NumExecutingActions = 0; foreach (Action Action in Actions) { ActionThread ActionThread = null; bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread); if (bFoundActionProcess == false) { NumUnexecutedActions++; } else if (ActionThread != null) { if (ActionThread.bComplete == false) { NumUnexecutedActions++; NumExecutingActions++; } } } // Update the current progress int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions; if (ProgressValue != NewProgressValue) { ProgressWriter.Write(ProgressValue, Actions.Count + 1); ProgressValue = NewProgressValue; } // If there aren't any pending actions left, we're done executing. if (NumUnexecutedActions == 0) { break; } // If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated // prerequisites. foreach (Action Action in Actions) { ActionThread ActionProcess = null; bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess); if (bFoundActionProcess == false) { if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel)) { // Determine whether there are any prerequisites of the action that are outdated. bool bHasOutdatedPrerequisites = false; bool bHasFailedPrerequisites = false; foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems) { if (PrerequisiteItem.ProducingAction != null && Actions.Contains(PrerequisiteItem.ProducingAction)) { ActionThread PrerequisiteProcess = null; bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue(PrerequisiteItem.ProducingAction, out PrerequisiteProcess); if (bFoundPrerequisiteProcess == true) { if (PrerequisiteProcess == null) { bHasFailedPrerequisites = true; } else if (PrerequisiteProcess.bComplete == false) { bHasOutdatedPrerequisites = true; } else if (PrerequisiteProcess.ExitCode != 0) { bHasFailedPrerequisites = true; } } else { bHasOutdatedPrerequisites = true; } } } // If there are any failed prerequisites of this action, don't execute it. if (bHasFailedPrerequisites) { // Add a null entry in the dictionary for this action. ActionThreadDictionary.Add(Action, null); } // If there aren't any outdated prerequisites of this action, execute it. else if (!bHasOutdatedPrerequisites) { ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count); JobNumber++; ActionThread.Run(); ActionThreadDictionary.Add(Action, ActionThread); NumExecutingActions++; } } } } System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime)); } } Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "-------- Begin Detailed Action Stats ----------------------------------------------------------"); Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "^Action Type^Duration (seconds)^Tool^Task^Using PCH"); double TotalThreadSeconds = 0; // Check whether any of the tasks failed and log action stats if wanted. bool bSuccess = true; foreach (KeyValuePair <Action, ActionThread> ActionProcess in ActionThreadDictionary) { Action Action = ActionProcess.Key; ActionThread ActionThread = ActionProcess.Value; // Check for pending actions, preemptive failure if (ActionThread == null) { bSuccess = false; continue; } // Check for executed action but general failure if (ActionThread.ExitCode != 0) { bSuccess = false; } // Log CPU time, tool and task. double ThreadSeconds = Action.Duration.TotalSeconds; Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "^{0}^{1:0.00}^{2}^{3}^{4}", Action.ActionType.ToString(), ThreadSeconds, Path.GetFileName(Action.CommandPath), Action.StatusDescription, Action.bIsUsingPCH); // Update statistics switch (Action.ActionType) { case ActionType.BuildProject: UnrealBuildTool.TotalBuildProjectTime += ThreadSeconds; break; case ActionType.Compile: UnrealBuildTool.TotalCompileTime += ThreadSeconds; break; case ActionType.CreateAppBundle: UnrealBuildTool.TotalCreateAppBundleTime += ThreadSeconds; break; case ActionType.GenerateDebugInfo: UnrealBuildTool.TotalGenerateDebugInfoTime += ThreadSeconds; break; case ActionType.Link: UnrealBuildTool.TotalLinkTime += ThreadSeconds; break; default: UnrealBuildTool.TotalOtherActionsTime += ThreadSeconds; break; } // Keep track of total thread seconds spent on tasks. TotalThreadSeconds += ThreadSeconds; } Log.TraceInformation("-------- End Detailed Actions Stats -----------------------------------------------------------"); // Log total CPU seconds and numbers of processors involved in tasks. Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats || BuildConfiguration.bPrintDebugInfo, TraceEventType.Information, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds); return(bSuccess); }
/** * Executes the specified actions locally. * @return True if all the tasks successfully executed, or false if any of them failed. */ public static bool ExecuteActions(List<Action> Actions) { // Time to sleep after each iteration of the loop in order to not busy wait. const float LoopSleepTime = 0.1f; // Use WMI to figure out physical cores, excluding hyper threading. int NumCores = 0; if (!Utils.IsRunningOnMono) { try { using (var Mos = new System.Management.ManagementObjectSearcher("Select * from Win32_Processor")) { var MosCollection = Mos.Get(); foreach (var Item in MosCollection) { NumCores += int.Parse(Item["NumberOfCores"].ToString()); } } } catch (Exception Ex) { Log.TraceWarning("Unable to get the number of Cores: {0}", Ex.ToString()); Log.TraceWarning("Falling back to processor count."); } } // On some systems this requires a hot fix to work so we fall back to using the (logical) processor count. if( NumCores == 0 ) { NumCores = System.Environment.ProcessorCount; } // The number of actions to execute in parallel is trying to keep the CPU busy enough in presence of I/O stalls. int MaxActionsToExecuteInParallel = 0; if (NumCores < System.Environment.ProcessorCount && BuildConfiguration.ProcessorCountMultiplier != 1.0) { // The CPU has more logical cores than physical ones, aka uses hyper-threading. // Use multiplier if provided MaxActionsToExecuteInParallel = (int)(NumCores * BuildConfiguration.ProcessorCountMultiplier); } else if (NumCores < System.Environment.ProcessorCount && NumCores > 4) { // The CPU has more logical cores than physical ones, aka uses hyper-threading. // Use average of logical and physical if we have "lots of cores" MaxActionsToExecuteInParallel = (int)(NumCores + System.Environment.ProcessorCount) / 2; } // No hyper-threading. Only kicking off a task per CPU to keep machine responsive. else { MaxActionsToExecuteInParallel = NumCores; } if (Utils.IsRunningOnMono) { // heuristic: give each action at least 1.5GB of RAM (some clang instances will need more, actually) long MinMemoryPerActionMB = 3 * 1024 / 2; long PhysicalRAMAvailableMB = (new PerformanceCounter ("Mono Memory", "Total Physical Memory").RawValue) / (1024 * 1024); int MaxActionsAffordedByMemory = (int)(Math.Max(1, (PhysicalRAMAvailableMB) / MinMemoryPerActionMB)); MaxActionsToExecuteInParallel = Math.Min(MaxActionsToExecuteInParallel, MaxActionsAffordedByMemory); } MaxActionsToExecuteInParallel = Math.Max( 1, Math.Min(MaxActionsToExecuteInParallel, BuildConfiguration.MaxProcessorCount) ); Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel); Dictionary<Action, ActionThread> ActionThreadDictionary = new Dictionary<Action, ActionThread>(); int JobNumber = 1; using(ProgressWriter ProgressWriter = new ProgressWriter("Compiling C++ source code...", false)) { int ProgressValue = 0; while(true) { // Count the number of pending and still executing actions. int NumUnexecutedActions = 0; int NumExecutingActions = 0; foreach (Action Action in Actions) { ActionThread ActionThread = null; bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread); if (bFoundActionProcess == false) { NumUnexecutedActions++; } else if (ActionThread != null) { if (ActionThread.bComplete == false) { NumUnexecutedActions++; NumExecutingActions++; } } } // Update the current progress int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions; if (ProgressValue != NewProgressValue) { ProgressWriter.Write(ProgressValue, Actions.Count + 1); ProgressValue = NewProgressValue; } // If there aren't any pending actions left, we're done executing. if (NumUnexecutedActions == 0) { break; } // If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated // prerequisites. foreach (Action Action in Actions) { ActionThread ActionProcess = null; bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess); if (bFoundActionProcess == false) { if (NumExecutingActions < Math.Max(1,MaxActionsToExecuteInParallel)) { // Determine whether there are any prerequisites of the action that are outdated. bool bHasOutdatedPrerequisites = false; bool bHasFailedPrerequisites = false; foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems) { if (PrerequisiteItem.ProducingAction != null && Actions.Contains(PrerequisiteItem.ProducingAction)) { ActionThread PrerequisiteProcess = null; bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue( PrerequisiteItem.ProducingAction, out PrerequisiteProcess ); if (bFoundPrerequisiteProcess == true) { if (PrerequisiteProcess == null) { bHasFailedPrerequisites = true; } else if (PrerequisiteProcess.bComplete == false) { bHasOutdatedPrerequisites = true; } else if (PrerequisiteProcess.ExitCode != 0) { bHasFailedPrerequisites = true; } } else { bHasOutdatedPrerequisites = true; } } } // If there are any failed prerequisites of this action, don't execute it. if (bHasFailedPrerequisites) { // Add a null entry in the dictionary for this action. ActionThreadDictionary.Add( Action, null ); } // If there aren't any outdated prerequisites of this action, execute it. else if (!bHasOutdatedPrerequisites) { ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count); JobNumber++; ActionThread.Run(); ActionThreadDictionary.Add(Action, ActionThread); NumExecutingActions++; } } } } System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime)); } } Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, LogEventType.Console, "-------- Begin Detailed Action Stats ----------------------------------------------------------"); Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, LogEventType.Console, "^Action Type^Duration (seconds)^Tool^Task^Using PCH"); double TotalThreadSeconds = 0; // Check whether any of the tasks failed and log action stats if wanted. bool bSuccess = true; foreach (KeyValuePair<Action, ActionThread> ActionProcess in ActionThreadDictionary) { Action Action = ActionProcess.Key; ActionThread ActionThread = ActionProcess.Value; // Check for pending actions, preemptive failure if (ActionThread == null) { bSuccess = false; continue; } // Check for executed action but general failure if (ActionThread.ExitCode != 0) { bSuccess = false; } // Log CPU time, tool and task. double ThreadSeconds = Action.Duration.TotalSeconds; Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, LogEventType.Console, "^{0}^{1:0.00}^{2}^{3}^{4}", Action.ActionType.ToString(), ThreadSeconds, Path.GetFileName(Action.CommandPath), Action.StatusDescription, Action.bIsUsingPCH); // Update statistics switch (Action.ActionType) { case ActionType.BuildProject: UnrealBuildTool.TotalBuildProjectTime += ThreadSeconds; break; case ActionType.Compile: UnrealBuildTool.TotalCompileTime += ThreadSeconds; break; case ActionType.CreateAppBundle: UnrealBuildTool.TotalCreateAppBundleTime += ThreadSeconds; break; case ActionType.GenerateDebugInfo: UnrealBuildTool.TotalGenerateDebugInfoTime += ThreadSeconds; break; case ActionType.Link: UnrealBuildTool.TotalLinkTime += ThreadSeconds; break; default: UnrealBuildTool.TotalOtherActionsTime += ThreadSeconds; break; } // Keep track of total thread seconds spent on tasks. TotalThreadSeconds += ThreadSeconds; } Log.TraceInformation("-------- End Detailed Actions Stats -----------------------------------------------------------"); // Log total CPU seconds and numbers of processors involved in tasks. Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats || BuildConfiguration.bPrintDebugInfo, LogEventType.Console, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds); return bSuccess; }
public override bool ExecuteActions(List <Action> Actions, bool bLogDetailedActionStats) { bool bDistccResult = true; if (Actions.Count > 0) { // Time to sleep after each iteration of the loop in order to not busy wait. const float LoopSleepTime = 0.1f; int MaxActionsToExecuteInParallel = 0; string UserDir = Environment.GetEnvironmentVariable("HOME"); string HostsInfo = UserDir + "/.dmucs/hosts-info"; string NumUBTBuildTasks = Environment.GetEnvironmentVariable("NumUBTBuildTasks"); Int32 MaxUBTBuildTasks = MaxActionsToExecuteInParallel; if (Int32.TryParse(NumUBTBuildTasks, out MaxUBTBuildTasks)) { MaxActionsToExecuteInParallel = MaxUBTBuildTasks; } else if (BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Mac) { using (System.Diagnostics.Process DefaultsProcess = new System.Diagnostics.Process()) { try { DefaultsProcess.StartInfo.FileName = "/usr/bin/defaults"; DefaultsProcess.StartInfo.CreateNoWindow = true; DefaultsProcess.StartInfo.UseShellExecute = false; DefaultsProcess.StartInfo.RedirectStandardOutput = true; DefaultsProcess.StartInfo.Arguments = "read com.apple.dt.Xcode IDEBuildOperationMaxNumberOfConcurrentCompileTasks"; DefaultsProcess.Start(); string Output = DefaultsProcess.StandardOutput.ReadToEnd(); DefaultsProcess.WaitForExit(); if (DefaultsProcess.ExitCode == 0 && Int32.TryParse(Output, out MaxUBTBuildTasks)) { MaxActionsToExecuteInParallel = MaxUBTBuildTasks; } } catch (Exception) { } } } else if (System.IO.File.Exists(HostsInfo)) { System.IO.StreamReader File = new System.IO.StreamReader(HostsInfo); string Line = null; while ((Line = File.ReadLine()) != null) { string[] HostInfo = Line.Split(' '); if (HostInfo.Count() == 3) { int NumCPUs = 0; if (System.Int32.TryParse(HostInfo[1], out NumCPUs)) { MaxActionsToExecuteInParallel += NumCPUs; } } } File.Close(); } else { MaxActionsToExecuteInParallel = System.Environment.ProcessorCount; } if (bAllowDistccLocalFallback == false) { Environment.SetEnvironmentVariable("DISTCC_FALLBACK", "0"); } if (bVerboseDistccOutput == true) { Environment.SetEnvironmentVariable("DISTCC_VERBOSE", "1"); } FileReference DistccExecutable = new FileReference(DistccExecutablesPath + "/distcc"); FileReference GetHostExecutable = new FileReference(DistccExecutablesPath + "/gethost"); Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel, DistccExecutable, GetHostExecutable); Dictionary <Action, ActionThread> ActionThreadDictionary = new Dictionary <Action, ActionThread>(); int JobNumber = 1; using (ProgressWriter ProgressWriter = new ProgressWriter("Compiling source code...", false)) { int ProgressValue = 0; while (true) { // Count the number of pending and still executing actions. int NumUnexecutedActions = 0; int NumExecutingActions = 0; foreach (Action Action in Actions) { ActionThread ActionThread = null; bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread); if (bFoundActionProcess == false) { NumUnexecutedActions++; } else if (ActionThread != null) { if (ActionThread.bComplete == false) { NumUnexecutedActions++; NumExecutingActions++; } } } // Update the current progress int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions; if (ProgressValue != NewProgressValue) { ProgressWriter.Write(ProgressValue, Actions.Count + 1); ProgressValue = NewProgressValue; } // If there aren't any pending actions left, we're done executing. if (NumUnexecutedActions == 0) { break; } // If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated // prerequisites. foreach (Action Action in Actions) { ActionThread ActionProcess = null; bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess); if (bFoundActionProcess == false) { if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel)) { // Determine whether there are any prerequisites of the action that are outdated. bool bHasOutdatedPrerequisites = false; bool bHasFailedPrerequisites = false; foreach (Action PrerequisiteAction in Action.PrerequisiteActions) { ActionThread PrerequisiteProcess = null; bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue(PrerequisiteAction, out PrerequisiteProcess); if (bFoundPrerequisiteProcess == true) { if (PrerequisiteProcess == null) { bHasFailedPrerequisites = true; } else if (PrerequisiteProcess.bComplete == false) { bHasOutdatedPrerequisites = true; } else if (PrerequisiteProcess.ExitCode != 0) { bHasFailedPrerequisites = true; } } else { bHasOutdatedPrerequisites = true; } } // If there are any failed prerequisites of this action, don't execute it. if (bHasFailedPrerequisites) { // Add a null entry in the dictionary for this action. ActionThreadDictionary.Add(Action, null); } // If there aren't any outdated prerequisites of this action, execute it. else if (!bHasOutdatedPrerequisites) { if (Action.ActionType == ActionType.Compile || Action.ActionType == ActionType.Link) { string TypeCommand = String.IsNullOrEmpty(DMUCSDistProp) ? "" : (" --type " + DMUCSDistProp); string NewCommandArguments = "--server " + DMUCSCoordinator + TypeCommand + " --wait -1 \"" + DistccExecutable + "\" " + Action.CommandPath + " " + Action.CommandArguments; if (Action.ActionType == ActionType.Compile) { // Distcc separates preprocessing and compilation which means we must silence these warnings NewCommandArguments += " -Wno-constant-logical-operand"; NewCommandArguments += " -Wno-parentheses-equality"; } Action.CommandPath = GetHostExecutable; Action.CommandArguments = NewCommandArguments; } ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count); JobNumber++; ActionThread.Run(); ActionThreadDictionary.Add(Action, ActionThread); NumExecutingActions++; } } } } System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime)); } } Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "-------- Begin Detailed Action Stats ----------------------------------------------------------"); Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "^Action Type^Duration (seconds)^Tool^Task^Using PCH"); double TotalThreadSeconds = 0; // Check whether any of the tasks failed and log action stats if wanted. foreach (KeyValuePair <Action, ActionThread> ActionProcess in ActionThreadDictionary) { Action Action = ActionProcess.Key; ActionThread ActionThread = ActionProcess.Value; // Check for pending actions, preemptive failure if (ActionThread == null) { bDistccResult = false; continue; } // Check for executed action but general failure if (ActionThread.ExitCode != 0) { bDistccResult = false; } // Log CPU time, tool and task. double ThreadSeconds = Action.Duration.TotalSeconds; Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "^{0}^{1:0.00}^{2}^{3}^{4}", Action.ActionType.ToString(), ThreadSeconds, Action.CommandPath.GetFileName(), Action.StatusDescription, Action.bIsUsingPCH); // Keep track of total thread seconds spent on tasks. TotalThreadSeconds += ThreadSeconds; } Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "-------- End Detailed Actions Stats -----------------------------------------------------------"); // Log total CPU seconds and numbers of processors involved in tasks. Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds); } return(bDistccResult); }
public override FileItem[] LinkAllFiles(LinkEnvironment LinkEnvironment, bool bBuildImportLibraryOnly) { List <FileItem> Outputs = new List <FileItem>(); for (int ArchIndex = 0; ArchIndex < Arches.Length; ArchIndex++) { string Arch = Arches[ArchIndex]; for (int GPUArchIndex = 0; GPUArchIndex < GPUArchitectures.Length; GPUArchIndex++) { string GPUArchitecture = GPUArchitectures[GPUArchIndex]; int OutputPathIndex = ArchIndex * GPUArchitectures.Length + GPUArchIndex; // Android will have an array of outputs if (LinkEnvironment.Config.OutputFilePaths.Length < OutputPathIndex || !Path.GetFileNameWithoutExtension(LinkEnvironment.Config.OutputFilePaths[OutputPathIndex]).EndsWith(Arch + GPUArchitecture)) { throw new BuildException("The OutputFilePaths array didn't match the Arches array in AndroidToolChain.LinkAllFiles"); } // Create an action that invokes the linker. Action LinkAction = new Action(ActionType.Link); LinkAction.WorkingDirectory = Path.GetFullPath("."); if (LinkEnvironment.Config.bIsBuildingLibrary) { LinkAction.CommandPath = Arch == "-armv7" ? ArPathArm : ArPathx86; } else { LinkAction.CommandPath = ClangPath; } string LinkerPath = LinkAction.WorkingDirectory; LinkAction.WorkingDirectory = LinkEnvironment.Config.IntermediateDirectory; // Get link arguments. LinkAction.CommandArguments = LinkEnvironment.Config.bIsBuildingLibrary ? GetArArguments(LinkEnvironment) : GetLinkArguments(LinkEnvironment, Arch); // Add the output file as a production of the link action. FileItem OutputFile; OutputFile = FileItem.GetItemByPath(LinkEnvironment.Config.OutputFilePaths[OutputPathIndex]); Outputs.Add(OutputFile); LinkAction.ProducedItems.Add(OutputFile); LinkAction.StatusDescription = string.Format("{0}", Path.GetFileName(OutputFile.AbsolutePath)); // LinkAction.bPrintDebugInfo = true; // Add the output file to the command-line. if (LinkEnvironment.Config.bIsBuildingLibrary) { LinkAction.CommandArguments += string.Format(" \"{0}\"", OutputFile.AbsolutePath); } else { LinkAction.CommandArguments += string.Format(" -o \"{0}\"", OutputFile.AbsolutePath); } // Add the input files to a response file, and pass the response file on the command-line. List <string> InputFileNames = new List <string>(); foreach (FileItem InputFile in LinkEnvironment.InputFiles) { // make sure it's for current Arch if (Path.GetFileNameWithoutExtension(InputFile.AbsolutePath).EndsWith(Arch + GPUArchitecture)) { string AbsolutePath = InputFile.AbsolutePath.Replace("\\", "/"); AbsolutePath = AbsolutePath.Replace(LinkEnvironment.Config.IntermediateDirectory.Replace("\\", "/"), ""); AbsolutePath = AbsolutePath.TrimStart(new char[] { '/' }); InputFileNames.Add(string.Format("\"{0}\"", AbsolutePath)); LinkAction.PrerequisiteItems.Add(InputFile); } } string ResponseFileName = GetResponseFileName(LinkEnvironment, OutputFile); LinkAction.CommandArguments += string.Format(" @\"{0}\"", ResponseFile.Create(ResponseFileName, InputFileNames)); // libs don't link in other libs if (!LinkEnvironment.Config.bIsBuildingLibrary) { // Add the library paths to the argument list. foreach (string LibraryPath in LinkEnvironment.Config.LibraryPaths) { // LinkerPaths could be relative or absolute string AbsoluteLibraryPath = ActionThread.ExpandEnvironmentVariables(LibraryPath); if (IsDirectoryForArch(AbsoluteLibraryPath, Arch)) { // environment variables aren't expanded when using the $( style if (Path.IsPathRooted(AbsoluteLibraryPath) == false) { AbsoluteLibraryPath = Path.Combine(LinkerPath, AbsoluteLibraryPath); } LinkAction.CommandArguments += string.Format(" -L\"{0}\"", AbsoluteLibraryPath); } } // add libraries in a library group LinkAction.CommandArguments += string.Format(" -Wl,--start-group"); foreach (string AdditionalLibrary in LinkEnvironment.Config.AdditionalLibraries) { if (!ShouldSkipLib(AdditionalLibrary, Arch, GPUArchitecture)) { if (String.IsNullOrEmpty(Path.GetDirectoryName(AdditionalLibrary))) { LinkAction.CommandArguments += string.Format(" \"-l{0}\"", AdditionalLibrary); } else { // full pathed libs are compiled by us, so we depend on linking them LinkAction.CommandArguments += string.Format(" \"{0}\"", AdditionalLibrary); LinkAction.PrerequisiteItems.Add(FileItem.GetItemByPath(AdditionalLibrary)); } } } LinkAction.CommandArguments += string.Format(" -Wl,--end-group"); } // Add the additional arguments specified by the environment. LinkAction.CommandArguments += LinkEnvironment.Config.AdditionalArguments; LinkAction.CommandArguments = LinkAction.CommandArguments.Replace("\\", "/"); // Only execute linking on the local PC. LinkAction.bCanExecuteRemotely = false; } } return(Outputs.ToArray()); }
public static bool ExecuteActions(List <Action> Actions) { bool bDistccResult = true; if (Actions.Count > 0) { // Time to sleep after each iteration of the loop in order to not busy wait. const float LoopSleepTime = 0.1f; int MaxActionsToExecuteInParallel = 0; string UserDir = Environment.GetEnvironmentVariable("HOME"); string HostsInfo = UserDir + "/.dmucs/hosts-info"; System.IO.StreamReader File = new System.IO.StreamReader(HostsInfo); string Line = null; while ((Line = File.ReadLine()) != null) { var HostInfo = Line.Split(' '); if (HostInfo.Count() == 3) { int NumCPUs = 0; if (System.Int32.TryParse(HostInfo [1], out NumCPUs)) { MaxActionsToExecuteInParallel += NumCPUs; } } } File.Close(); if (BuildConfiguration.bAllowDistccLocalFallback == false) { Environment.SetEnvironmentVariable("DISTCC_FALLBACK", "0"); } string DistccExecutable = BuildConfiguration.DistccExecutablesPath + "/distcc"; string GetHostExecutable = BuildConfiguration.DistccExecutablesPath + "/gethost"; Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel, DistccExecutable, GetHostExecutable); Dictionary <Action, ActionThread> ActionThreadDictionary = new Dictionary <Action, ActionThread>(); int JobNumber = 1; using (ProgressWriter ProgressWriter = new ProgressWriter("Compiling source code...", false)) { int ProgressValue = 0; while (true) { // Count the number of pending and still executing actions. int NumUnexecutedActions = 0; int NumExecutingActions = 0; foreach (Action Action in Actions) { ActionThread ActionThread = null; bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread); if (bFoundActionProcess == false) { NumUnexecutedActions++; } else if (ActionThread != null) { if (ActionThread.bComplete == false) { NumUnexecutedActions++; NumExecutingActions++; } } } // Update the current progress int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions; if (ProgressValue != NewProgressValue) { ProgressWriter.Write(ProgressValue, Actions.Count + 1); ProgressValue = NewProgressValue; } // If there aren't any pending actions left, we're done executing. if (NumUnexecutedActions == 0) { break; } // If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated // prerequisites. foreach (Action Action in Actions) { ActionThread ActionProcess = null; bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess); if (bFoundActionProcess == false) { if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel)) { // Determine whether there are any prerequisites of the action that are outdated. bool bHasOutdatedPrerequisites = false; bool bHasFailedPrerequisites = false; foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems) { if (PrerequisiteItem.ProducingAction != null && Actions.Contains(PrerequisiteItem.ProducingAction)) { ActionThread PrerequisiteProcess = null; bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue(PrerequisiteItem.ProducingAction, out PrerequisiteProcess); if (bFoundPrerequisiteProcess == true) { if (PrerequisiteProcess == null) { bHasFailedPrerequisites = true; } else if (PrerequisiteProcess.bComplete == false) { bHasOutdatedPrerequisites = true; } else if (PrerequisiteProcess.ExitCode != 0) { bHasFailedPrerequisites = true; } } else { bHasOutdatedPrerequisites = true; } } } // If there are any failed prerequisites of this action, don't execute it. if (bHasFailedPrerequisites) { // Add a null entry in the dictionary for this action. ActionThreadDictionary.Add(Action, null); } // If there aren't any outdated prerequisites of this action, execute it. else if (!bHasOutdatedPrerequisites) { if ((Action.ActionType == ActionType.Compile || Action.ActionType == ActionType.Link) && DistccExecutable != null && GetHostExecutable != null) { string NewCommandArguments = "--wait -1 \"" + DistccExecutable + "\" " + Action.CommandPath + " " + Action.CommandArguments; Action.CommandPath = GetHostExecutable; Action.CommandArguments = NewCommandArguments; } ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count); JobNumber++; ActionThread.Run(); ActionThreadDictionary.Add(Action, ActionThread); NumExecutingActions++; } } } } System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime)); } } Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "-------- Begin Detailed Action Stats ----------------------------------------------------------"); Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "^Action Type^Duration (seconds)^Tool^Task^Using PCH"); double TotalThreadSeconds = 0; // Check whether any of the tasks failed and log action stats if wanted. foreach (KeyValuePair <Action, ActionThread> ActionProcess in ActionThreadDictionary) { Action Action = ActionProcess.Key; ActionThread ActionThread = ActionProcess.Value; // Check for pending actions, preemptive failure if (ActionThread == null) { bDistccResult = false; continue; } // Check for executed action but general failure if (ActionThread.ExitCode != 0) { bDistccResult = false; } // Log CPU time, tool and task. double ThreadSeconds = Action.Duration.TotalSeconds; Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "^{0}^{1:0.00}^{2}^{3}^{4}", Action.ActionType.ToString(), ThreadSeconds, Path.GetFileName(Action.CommandPath), Action.StatusDescription, Action.bIsUsingPCH); // Update statistics switch (Action.ActionType) { case ActionType.BuildProject: UnrealBuildTool.TotalBuildProjectTime += ThreadSeconds; break; case ActionType.Compile: UnrealBuildTool.TotalCompileTime += ThreadSeconds; break; case ActionType.CreateAppBundle: UnrealBuildTool.TotalCreateAppBundleTime += ThreadSeconds; break; case ActionType.GenerateDebugInfo: UnrealBuildTool.TotalGenerateDebugInfoTime += ThreadSeconds; break; case ActionType.Link: UnrealBuildTool.TotalLinkTime += ThreadSeconds; break; default: UnrealBuildTool.TotalOtherActionsTime += ThreadSeconds; break; } // Keep track of total thread seconds spent on tasks. TotalThreadSeconds += ThreadSeconds; } Log.TraceInformation("-------- End Detailed Actions Stats -----------------------------------------------------------"); // Log total CPU seconds and numbers of processors involved in tasks. Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats || BuildConfiguration.bPrintDebugInfo, TraceEventType.Information, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds); } return(bDistccResult); }
public override CPPOutput CompileCPPFiles(UEBuildTarget Target, CPPEnvironment CompileEnvironment, List <FileItem> SourceFiles, string ModuleName) { if (Arches.Length == 0) { throw new BuildException("At least one architecture (armv7, x86, etc) needs to be selected in the project settings to build"); } if (!bHasPrintedApiLevel) { Console.WriteLine("Compiling with NDK API '{0}'", GetNdkApiLevel()); bHasPrintedApiLevel = true; } string BaseArguments = ""; if (CompileEnvironment.Config.PrecompiledHeaderAction != PrecompiledHeaderAction.Create) { BaseArguments += " -Werror"; } // Directly added NDK files for NDK extensions if (!UnrealBuildTool.RunningRocket()) { ConditionallyAddNDKSourceFiles(SourceFiles); } // Add preprocessor definitions to the argument list. foreach (string Definition in CompileEnvironment.Config.Definitions) { BaseArguments += string.Format(" -D \"{0}\"", Definition); } var BuildPlatform = UEBuildPlatform.GetBuildPlatformForCPPTargetPlatform(CompileEnvironment.Config.Target.Platform); string BasePCHName = ""; var PCHExtension = UEBuildPlatform.BuildPlatformDictionary[UnrealTargetPlatform.Android].GetBinaryExtension(UEBuildBinaryType.PrecompiledHeader); if (CompileEnvironment.Config.PrecompiledHeaderAction == PrecompiledHeaderAction.Include) { BasePCHName = RemoveArchName(CompileEnvironment.PrecompiledHeaderFile.AbsolutePath).Replace(PCHExtension, ""); } // Create a compile action for each source file. CPPOutput Result = new CPPOutput(); foreach (string Arch in Arches) { foreach (string GPUArchitecture in GPUArchitectures) { // which toolchain to use string Arguments = GetCLArguments_Global(CompileEnvironment, Arch) + BaseArguments; if (GPUArchitecture == "-gl4") { Arguments += " -DPLATFORM_ANDROIDGL4=1"; } else if (GPUArchitecture == "-es31") { Arguments += " -DPLATFORM_ANDROIDES31=1"; } // which PCH file to include string PCHArguments = ""; if (CompileEnvironment.Config.PrecompiledHeaderAction == PrecompiledHeaderAction.Include) { // Add the precompiled header file's path to the include path so Clang can find it. // This needs to be before the other include paths to ensure Clang uses it instead of the source header file. PCHArguments += string.Format(" -include \"{0}\"", InlineArchName(BasePCHName, Arch, GPUArchitecture)); } // Add include paths to the argument list (filtered by architecture) foreach (string IncludePath in CompileEnvironment.Config.CPPIncludeInfo.SystemIncludePaths) { if (IsDirectoryForArch(IncludePath, Arch)) { Arguments += string.Format(" -I\"{0}\"", IncludePath); } } foreach (string IncludePath in CompileEnvironment.Config.CPPIncludeInfo.IncludePaths) { if (IsDirectoryForArch(IncludePath, Arch)) { Arguments += string.Format(" -I\"{0}\"", IncludePath); } } foreach (FileItem SourceFile in SourceFiles) { Action CompileAction = new Action(ActionType.Compile); string FileArguments = ""; bool bIsPlainCFile = Path.GetExtension(SourceFile.AbsolutePath).ToUpperInvariant() == ".C"; // should we disable optimizations on this file? // @todo android - We wouldn't need this if we could disable optimizations per function (via pragma) bool bDisableOptimizations = false; // SourceFile.AbsolutePath.ToUpperInvariant().IndexOf("\\SLATE\\") != -1; if (bDisableOptimizations && CompileEnvironment.Config.Target.Configuration != CPPTargetConfiguration.Debug) { Log.TraceWarning("Disabling optimizations on {0}", SourceFile.AbsolutePath); } bDisableOptimizations = bDisableOptimizations || CompileEnvironment.Config.Target.Configuration == CPPTargetConfiguration.Debug; // Add C or C++ specific compiler arguments. if (CompileEnvironment.Config.PrecompiledHeaderAction == PrecompiledHeaderAction.Create) { FileArguments += GetCompileArguments_PCH(bDisableOptimizations); } else if (bIsPlainCFile) { FileArguments += GetCompileArguments_C(bDisableOptimizations); } else { FileArguments += GetCompileArguments_CPP(bDisableOptimizations); // only use PCH for .cpp files FileArguments += PCHArguments; } // Add the C++ source file and its included files to the prerequisite item list. AddPrerequisiteSourceFile(Target, BuildPlatform, CompileEnvironment, SourceFile, CompileAction.PrerequisiteItems); if (CompileEnvironment.Config.PrecompiledHeaderAction == PrecompiledHeaderAction.Create) { // Add the precompiled header file to the produced item list. FileItem PrecompiledHeaderFile = FileItem.GetItemByPath( Path.Combine( CompileEnvironment.Config.OutputDirectory, Path.GetFileName(InlineArchName(SourceFile.AbsolutePath, Arch, GPUArchitecture) + PCHExtension) ) ); CompileAction.ProducedItems.Add(PrecompiledHeaderFile); Result.PrecompiledHeaderFile = PrecompiledHeaderFile; // Add the parameters needed to compile the precompiled header file to the command-line. FileArguments += string.Format(" -o \"{0}\"", PrecompiledHeaderFile.AbsolutePath, false); } else { if (CompileEnvironment.Config.PrecompiledHeaderAction == PrecompiledHeaderAction.Include) { CompileAction.bIsUsingPCH = true; FileItem ArchPrecompiledHeaderFile = FileItem.GetItemByPath(InlineArchName(BasePCHName, Arch, GPUArchitecture) + PCHExtension); CompileAction.PrerequisiteItems.Add(ArchPrecompiledHeaderFile); } var ObjectFileExtension = UEBuildPlatform.BuildPlatformDictionary[UnrealTargetPlatform.Android].GetBinaryExtension(UEBuildBinaryType.Object); // Add the object file to the produced item list. FileItem ObjectFile = FileItem.GetItemByPath( InlineArchName(Path.Combine( CompileEnvironment.Config.OutputDirectory, Path.GetFileName(SourceFile.AbsolutePath) + ObjectFileExtension), Arch, GPUArchitecture ) ); CompileAction.ProducedItems.Add(ObjectFile); Result.ObjectFiles.Add(ObjectFile); FileArguments += string.Format(" -o \"{0}\"", ObjectFile.AbsolutePath, false); } // Add the source file path to the command-line. FileArguments += string.Format(" \"{0}\"", SourceFile.AbsolutePath); // Build a full argument list string AllArguments = Arguments + FileArguments + CompileEnvironment.Config.AdditionalArguments; AllArguments = ActionThread.ExpandEnvironmentVariables(AllArguments); AllArguments = AllArguments.Replace("\\", "/"); // Create the response file string ResponseFileName = CompileAction.ProducedItems[0].AbsolutePath + ".response"; string ResponseArgument = string.Format("@\"{0}\"", ResponseFile.Create(ResponseFileName, new List <string> { AllArguments })); CompileAction.WorkingDirectory = Path.GetFullPath("."); CompileAction.CommandPath = ClangPath; CompileAction.CommandArguments = ResponseArgument; CompileAction.StatusDescription = string.Format("{0} [{1}-{2}]", Path.GetFileName(SourceFile.AbsolutePath), Arch.Replace("-", ""), GPUArchitecture.Replace("-", "")); CompileAction.OutputEventHandler = new DataReceivedEventHandler(CompileOutputReceivedDataEventHandler); // VC++ always outputs the source file name being compiled, so we don't need to emit this ourselves CompileAction.bShouldOutputStatusDescription = true; // Don't farm out creation of pre-compiled headers as it is the critical path task. CompileAction.bCanExecuteRemotely = CompileEnvironment.Config.PrecompiledHeaderAction != PrecompiledHeaderAction.Create || BuildConfiguration.bAllowRemotelyCompiledPCHs; } } } return(Result); }
public static ExecutionResult ExecuteActions(List <Action> Actions) { ExecutionResult SNDBSResult = ExecutionResult.TasksSucceeded; if (Actions.Count > 0) { string SCERoot = Environment.GetEnvironmentVariable("SCE_ROOT_DIR"); bool bSNDBSExists = false; if (SCERoot != null) { string SNDBSExecutable = Path.Combine(SCERoot, "Common/SN-DBS/bin/dbsbuild.exe"); // Check that SN-DBS is available bSNDBSExists = File.Exists(SNDBSExecutable); } if (bSNDBSExists == false) { return(ExecutionResult.Unavailable); } // Use WMI to figure out physical cores, excluding hyper threading. int NumCores = 0; if (!Utils.IsRunningOnMono) { try { using (ManagementObjectSearcher Mos = new System.Management.ManagementObjectSearcher("Select * from Win32_Processor")) { ManagementObjectCollection MosCollection = Mos.Get(); foreach (ManagementBaseObject Item in MosCollection) { NumCores += int.Parse(Item["NumberOfCores"].ToString()); } } } catch (Exception Ex) { Log.TraceWarning("Unable to get the number of Cores: {0}", Ex.ToString()); Log.TraceWarning("Falling back to processor count."); } } // On some systems this requires a hot fix to work so we fall back to using the (logical) processor count. if (NumCores == 0) { NumCores = System.Environment.ProcessorCount; } // The number of actions to execute in parallel is trying to keep the CPU busy enough in presence of I/O stalls. MaxActionsToExecuteInParallel = 0; // The CPU has more logical cores than physical ones, aka uses hyper-threading. if (NumCores < System.Environment.ProcessorCount) { MaxActionsToExecuteInParallel = (int)(NumCores * BuildConfiguration.ProcessorCountMultiplier); } // No hyper-threading. Only kicking off a task per CPU to keep machine responsive. else { MaxActionsToExecuteInParallel = NumCores; } MaxActionsToExecuteInParallel = Math.Min(MaxActionsToExecuteInParallel, BuildConfiguration.MaxProcessorCount); JobNumber = 1; Dictionary <Action, ActionThread> ActionThreadDictionary = new Dictionary <Action, ActionThread>(); while (true) { bool bUnexecutedActions = false; foreach (Action Action in Actions) { ActionThread ActionThread = null; bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread); if (bFoundActionProcess == false) { bUnexecutedActions = true; ExecutionResult CompileResult = ExecuteActions(Actions, ActionThreadDictionary); if (CompileResult != ExecutionResult.TasksSucceeded) { return(ExecutionResult.TasksFailed); } break; } } if (bUnexecutedActions == false) { break; } } Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, LogEventType.Console, "-------- Begin Detailed Action Stats ----------------------------------------------------------"); Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, LogEventType.Console, "^Action Type^Duration (seconds)^Tool^Task^Using PCH"); double TotalThreadSeconds = 0; // Check whether any of the tasks failed and log action stats if wanted. foreach (KeyValuePair <Action, ActionThread> ActionProcess in ActionThreadDictionary) { Action Action = ActionProcess.Key; ActionThread ActionThread = ActionProcess.Value; // Check for pending actions, preemptive failure if (ActionThread == null) { SNDBSResult = ExecutionResult.TasksFailed; continue; } // Check for executed action but general failure if (ActionThread.ExitCode != 0) { SNDBSResult = ExecutionResult.TasksFailed; } // Log CPU time, tool and task. double ThreadSeconds = Action.Duration.TotalSeconds; Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, LogEventType.Console, "^{0}^{1:0.00}^{2}^{3}^{4}", Action.ActionType.ToString(), ThreadSeconds, Path.GetFileName(Action.CommandPath), Action.StatusDescription, Action.bIsUsingPCH); // Update statistics switch (Action.ActionType) { case ActionType.BuildProject: UnrealBuildTool.TotalBuildProjectTime += ThreadSeconds; break; case ActionType.Compile: UnrealBuildTool.TotalCompileTime += ThreadSeconds; break; case ActionType.CreateAppBundle: UnrealBuildTool.TotalCreateAppBundleTime += ThreadSeconds; break; case ActionType.GenerateDebugInfo: UnrealBuildTool.TotalGenerateDebugInfoTime += ThreadSeconds; break; case ActionType.Link: UnrealBuildTool.TotalLinkTime += ThreadSeconds; break; default: UnrealBuildTool.TotalOtherActionsTime += ThreadSeconds; break; } // Keep track of total thread seconds spent on tasks. TotalThreadSeconds += ThreadSeconds; } Log.TraceInformation("-------- End Detailed Actions Stats -----------------------------------------------------------"); // Log total CPU seconds and numbers of processors involved in tasks. Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats || BuildConfiguration.bPrintDebugInfo, LogEventType.Console, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds); } return(SNDBSResult); }
public static bool ExecuteActions(List<Action> Actions) { bool bDistccResult = true; if (Actions.Count > 0) { // Time to sleep after each iteration of the loop in order to not busy wait. const float LoopSleepTime = 0.1f; int MaxActionsToExecuteInParallel = 0; string UserDir = Environment.GetEnvironmentVariable("HOME"); string HostsInfo = UserDir + "/.dmucs/hosts-info"; string NumUBTBuildTasks = Environment.GetEnvironmentVariable("NumUBTBuildTasks"); Int32 MaxUBTBuildTasks = MaxActionsToExecuteInParallel; if(Int32.TryParse(NumUBTBuildTasks, out MaxUBTBuildTasks)) { MaxActionsToExecuteInParallel = MaxUBTBuildTasks; } else if (BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Mac) { using (System.Diagnostics.Process DefaultsProcess = new System.Diagnostics.Process()) { try { DefaultsProcess.StartInfo.FileName = "/usr/bin/defaults"; DefaultsProcess.StartInfo.CreateNoWindow = true; DefaultsProcess.StartInfo.UseShellExecute = false; DefaultsProcess.StartInfo.RedirectStandardOutput = true; DefaultsProcess.StartInfo.Arguments = "com.apple.dt.Xcode IDEBuildOperationMaxNumberOfConcurrentCompileTasks"; DefaultsProcess.Start(); string Output = DefaultsProcess.StandardOutput.ReadToEnd(); DefaultsProcess.WaitForExit(); if(DefaultsProcess.ExitCode == 0 && Int32.TryParse(Output, out MaxUBTBuildTasks)) { MaxActionsToExecuteInParallel = MaxUBTBuildTasks; } } catch (Exception) { } } } else if (System.IO.File.Exists (HostsInfo)) { System.IO.StreamReader File = new System.IO.StreamReader(HostsInfo); string Line = null; while((Line = File.ReadLine()) != null) { var HostInfo = Line.Split(' '); if (HostInfo.Count () == 3) { int NumCPUs = 0; if (System.Int32.TryParse (HostInfo [1], out NumCPUs)) { MaxActionsToExecuteInParallel += NumCPUs; } } } File.Close(); } else { MaxActionsToExecuteInParallel = System.Environment.ProcessorCount; } if (BuildConfiguration.bAllowDistccLocalFallback == false) { Environment.SetEnvironmentVariable("DISTCC_FALLBACK", "0"); } if (BuildConfiguration.bVerboseDistccOutput == true) { Environment.SetEnvironmentVariable("DISTCC_VERBOSE", "1"); } string DistccExecutable = BuildConfiguration.DistccExecutablesPath + "/distcc"; string GetHostExecutable = BuildConfiguration.DistccExecutablesPath + "/gethost"; Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel, DistccExecutable, GetHostExecutable); Dictionary<Action, ActionThread> ActionThreadDictionary = new Dictionary<Action, ActionThread>(); int JobNumber = 1; using(ProgressWriter ProgressWriter = new ProgressWriter("Compiling source code...", false)) { int ProgressValue = 0; while(true) { // Count the number of pending and still executing actions. int NumUnexecutedActions = 0; int NumExecutingActions = 0; foreach (Action Action in Actions) { ActionThread ActionThread = null; bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread); if (bFoundActionProcess == false) { NumUnexecutedActions++; } else if (ActionThread != null) { if (ActionThread.bComplete == false) { NumUnexecutedActions++; NumExecutingActions++; } } } // Update the current progress int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions; if (ProgressValue != NewProgressValue) { ProgressWriter.Write(ProgressValue, Actions.Count + 1); ProgressValue = NewProgressValue; } // If there aren't any pending actions left, we're done executing. if (NumUnexecutedActions == 0) { break; } // If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated // prerequisites. foreach (Action Action in Actions) { ActionThread ActionProcess = null; bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess); if (bFoundActionProcess == false) { if (NumExecutingActions < Math.Max(1,MaxActionsToExecuteInParallel)) { // Determine whether there are any prerequisites of the action that are outdated. bool bHasOutdatedPrerequisites = false; bool bHasFailedPrerequisites = false; foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems) { if (PrerequisiteItem.ProducingAction != null && Actions.Contains(PrerequisiteItem.ProducingAction)) { ActionThread PrerequisiteProcess = null; bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue( PrerequisiteItem.ProducingAction, out PrerequisiteProcess ); if (bFoundPrerequisiteProcess == true) { if (PrerequisiteProcess == null) { bHasFailedPrerequisites = true; } else if (PrerequisiteProcess.bComplete == false) { bHasOutdatedPrerequisites = true; } else if (PrerequisiteProcess.ExitCode != 0) { bHasFailedPrerequisites = true; } } else { bHasOutdatedPrerequisites = true; } } } // If there are any failed prerequisites of this action, don't execute it. if (bHasFailedPrerequisites) { // Add a null entry in the dictionary for this action. ActionThreadDictionary.Add( Action, null ); } // If there aren't any outdated prerequisites of this action, execute it. else if (!bHasOutdatedPrerequisites) { if ((Action.ActionType == ActionType.Compile || Action.ActionType == ActionType.Link) && DistccExecutable != null && GetHostExecutable != null) { string TypeCommand = String.IsNullOrEmpty(BuildConfiguration.DMUCSDistProp) ? "" : (" --type " + BuildConfiguration.DMUCSDistProp); string NewCommandArguments = "--server " + BuildConfiguration.DMUCSCoordinator + TypeCommand + " --wait -1 \"" + DistccExecutable + "\" " + Action.CommandPath + " " + Action.CommandArguments; Action.CommandPath = GetHostExecutable; Action.CommandArguments = NewCommandArguments; } ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count); JobNumber++; ActionThread.Run(); ActionThreadDictionary.Add(Action, ActionThread); NumExecutingActions++; } } } } System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime)); } } Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, LogEventType.Console, "-------- Begin Detailed Action Stats ----------------------------------------------------------"); Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, LogEventType.Console, "^Action Type^Duration (seconds)^Tool^Task^Using PCH"); double TotalThreadSeconds = 0; // Check whether any of the tasks failed and log action stats if wanted. foreach (KeyValuePair<Action, ActionThread> ActionProcess in ActionThreadDictionary) { Action Action = ActionProcess.Key; ActionThread ActionThread = ActionProcess.Value; // Check for pending actions, preemptive failure if (ActionThread == null) { bDistccResult = false; continue; } // Check for executed action but general failure if (ActionThread.ExitCode != 0) { bDistccResult = false; } // Log CPU time, tool and task. double ThreadSeconds = Action.Duration.TotalSeconds; Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, LogEventType.Console, "^{0}^{1:0.00}^{2}^{3}^{4}", Action.ActionType.ToString(), ThreadSeconds, Path.GetFileName(Action.CommandPath), Action.StatusDescription, Action.bIsUsingPCH); // Update statistics switch (Action.ActionType) { case ActionType.BuildProject: UnrealBuildTool.TotalBuildProjectTime += ThreadSeconds; break; case ActionType.Compile: UnrealBuildTool.TotalCompileTime += ThreadSeconds; break; case ActionType.CreateAppBundle: UnrealBuildTool.TotalCreateAppBundleTime += ThreadSeconds; break; case ActionType.GenerateDebugInfo: UnrealBuildTool.TotalGenerateDebugInfoTime += ThreadSeconds; break; case ActionType.Link: UnrealBuildTool.TotalLinkTime += ThreadSeconds; break; default: UnrealBuildTool.TotalOtherActionsTime += ThreadSeconds; break; } // Keep track of total thread seconds spent on tasks. TotalThreadSeconds += ThreadSeconds; } Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats || BuildConfiguration.bPrintDebugInfo, LogEventType.Console, "-------- End Detailed Actions Stats -----------------------------------------------------------"); // Log total CPU seconds and numbers of processors involved in tasks. Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats || BuildConfiguration.bPrintDebugInfo, LogEventType.Console, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds); } return bDistccResult; }
internal bool ExecuteLocalActions(List <Action> InLocalActions, Dictionary <Action, ActionThread> InActionThreadDictionary, int TotalNumJobs) { // Time to sleep after each iteration of the loop in order to not busy wait. const float LoopSleepTime = 0.1f; bool LocalActionsResult = true; while (true) { // Count the number of pending and still executing actions. int NumUnexecutedActions = 0; int NumExecutingActions = 0; foreach (Action Action in InLocalActions) { ActionThread ActionThread = null; bool bFoundActionProcess = InActionThreadDictionary.TryGetValue(Action, out ActionThread); if (bFoundActionProcess == false) { NumUnexecutedActions++; } else if (ActionThread != null) { if (ActionThread.bComplete == false) { NumUnexecutedActions++; NumExecutingActions++; } } } // If there aren't any pending actions left, we're done executing. if (NumUnexecutedActions == 0) { break; } // If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated // prerequisites. foreach (Action Action in InLocalActions) { ActionThread ActionProcess = null; bool bFoundActionProcess = InActionThreadDictionary.TryGetValue(Action, out ActionProcess); if (bFoundActionProcess == false) { if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel)) { // Determine whether there are any prerequisites of the action that are outdated. bool bHasOutdatedPrerequisites = false; bool bHasFailedPrerequisites = false; foreach (Action PrerequisiteAction in Action.PrerequisiteActions) { if (InLocalActions.Contains(PrerequisiteAction)) { ActionThread PrerequisiteProcess = null; bool bFoundPrerequisiteProcess = InActionThreadDictionary.TryGetValue(PrerequisiteAction, out PrerequisiteProcess); if (bFoundPrerequisiteProcess == true) { if (PrerequisiteProcess == null) { bHasFailedPrerequisites = true; } else if (PrerequisiteProcess.bComplete == false) { bHasOutdatedPrerequisites = true; } else if (PrerequisiteProcess.ExitCode != 0) { bHasFailedPrerequisites = true; } } else { bHasOutdatedPrerequisites = true; } } } // If there are any failed prerequisites of this action, don't execute it. if (bHasFailedPrerequisites) { // Add a null entry in the dictionary for this action. InActionThreadDictionary.Add(Action, null); } // If there aren't any outdated prerequisites of this action, execute it. else if (!bHasOutdatedPrerequisites) { ActionThread ActionThread = new ActionThread(Action, JobNumber, TotalNumJobs); ActionThread.Run(); InActionThreadDictionary.Add(Action, ActionThread); NumExecutingActions++; JobNumber++; } } } } System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime)); } return(LocalActionsResult); }
/// <summary> /// Executes the specified actions locally. /// </summary> /// <returns>True if all the tasks successfully executed, or false if any of them failed.</returns> public override bool ExecuteActions(List <Action> Actions, bool bLogDetailedActionStats) { // Time to sleep after each iteration of the loop in order to not busy wait. const float LoopSleepTime = 0.1f; // The number of actions to execute in parallel is trying to keep the CPU busy enough in presence of I/O stalls. int MaxActionsToExecuteInParallel = GetMaxActionsToExecuteInParallel(); Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel); Dictionary <Action, ActionThread> ActionThreadDictionary = new Dictionary <Action, ActionThread>(); int JobNumber = 1; using (ProgressWriter ProgressWriter = new ProgressWriter("Compiling C++ source code...", false)) { int ProgressValue = 0; while (true) { // Count the number of pending and still executing actions. int NumUnexecutedActions = 0; int NumExecutingActions = 0; foreach (Action Action in Actions) { ActionThread ActionThread = null; bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread); if (bFoundActionProcess == false) { NumUnexecutedActions++; } else if (ActionThread != null) { if (ActionThread.bComplete == false) { NumUnexecutedActions++; NumExecutingActions++; } } } // Update the current progress int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions; if (ProgressValue != NewProgressValue) { ProgressWriter.Write(ProgressValue, Actions.Count + 1); ProgressValue = NewProgressValue; } // If there aren't any pending actions left, we're done executing. if (NumUnexecutedActions == 0) { break; } // If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated // prerequisites. foreach (Action Action in Actions) { ActionThread ActionProcess = null; bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess); if (bFoundActionProcess == false) { if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel)) { // Determine whether there are any prerequisites of the action that are outdated. bool bHasOutdatedPrerequisites = false; bool bHasFailedPrerequisites = false; foreach (Action PrerequisiteAction in Action.PrerequisiteActions) { if (Actions.Contains(PrerequisiteAction)) { ActionThread PrerequisiteProcess = null; bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue(PrerequisiteAction, out PrerequisiteProcess); if (bFoundPrerequisiteProcess == true) { if (PrerequisiteProcess == null) { bHasFailedPrerequisites = true; } else if (PrerequisiteProcess.bComplete == false) { bHasOutdatedPrerequisites = true; } else if (PrerequisiteProcess.ExitCode != 0) { bHasFailedPrerequisites = true; } } else { bHasOutdatedPrerequisites = true; } } } // If there are any failed prerequisites of this action, don't execute it. if (bHasFailedPrerequisites) { // Add a null entry in the dictionary for this action. ActionThreadDictionary.Add(Action, null); } // If there aren't any outdated prerequisites of this action, execute it. else if (!bHasOutdatedPrerequisites) { ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count); JobNumber++; try { ActionThread.Run(); } catch (Exception ex) { throw new BuildException(ex, "Failed to start thread for action: {0} {1}\r\n{2}", Action.CommandPath, Action.CommandArguments, ex.ToString()); } ActionThreadDictionary.Add(Action, ActionThread); NumExecutingActions++; } } } } System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime)); } } Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "-------- Begin Detailed Action Stats ----------------------------------------------------------"); Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "^Action Type^Duration (seconds)^Tool^Task^Using PCH"); double TotalThreadSeconds = 0; // Check whether any of the tasks failed and log action stats if wanted. bool bSuccess = true; double TotalBuildProjectTime = 0; double TotalCompileTime = 0; double TotalCreateAppBundleTime = 0; double TotalGenerateDebugInfoTime = 0; double TotalLinkTime = 0; double TotalOtherActionsTime = 0; foreach (KeyValuePair <Action, ActionThread> ActionProcess in ActionThreadDictionary) { Action Action = ActionProcess.Key; ActionThread ActionThread = ActionProcess.Value; // Check for pending actions, preemptive failure if (ActionThread == null) { bSuccess = false; continue; } // Check for executed action but general failure if (ActionThread.ExitCode != 0) { bSuccess = false; } // Log CPU time, tool and task. double ThreadSeconds = Action.Duration.TotalSeconds; Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "^{0}^{1:0.00}^{2}^{3}^{4}", Action.ActionType.ToString(), ThreadSeconds, Action.CommandPath.GetFileName(), Action.StatusDescription, Action.bIsUsingPCH); // Update statistics switch (Action.ActionType) { case ActionType.BuildProject: TotalBuildProjectTime += ThreadSeconds; break; case ActionType.Compile: TotalCompileTime += ThreadSeconds; break; case ActionType.CreateAppBundle: TotalCreateAppBundleTime += ThreadSeconds; break; case ActionType.GenerateDebugInfo: TotalGenerateDebugInfoTime += ThreadSeconds; break; case ActionType.Link: TotalLinkTime += ThreadSeconds; break; default: TotalOtherActionsTime += ThreadSeconds; break; } // Keep track of total thread seconds spent on tasks. TotalThreadSeconds += ThreadSeconds; } Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "-------- End Detailed Actions Stats -----------------------------------------------------------"); // Log total CPU seconds and numbers of processors involved in tasks. Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds); // Log detailed stats Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "Cumulative action seconds ({0} processors): {1:0.00} building projects, {2:0.00} compiling, {3:0.00} creating app bundles, {4:0.00} generating debug info, {5:0.00} linking, {6:0.00} other", System.Environment.ProcessorCount, TotalBuildProjectTime, TotalCompileTime, TotalCreateAppBundleTime, TotalGenerateDebugInfoTime, TotalLinkTime, TotalOtherActionsTime ); return(bSuccess); }
static string GetFullIncludePath(string IncludePath) { return(Path.GetFullPath(ActionThread.ExpandEnvironmentVariables(IncludePath))); }
/// <summary> /// Executes the specified actions locally. /// </summary> /// <returns>True if all the tasks successfully executed, or false if any of them failed.</returns> public override bool ExecuteActions(List <Action> Actions, bool bLogDetailedActionStats) { // Time to sleep after each iteration of the loop in order to not busy wait. const float LoopSleepTime = 0.1f; // Use WMI to figure out physical cores, excluding hyper threading. int NumCores = Utils.GetPhysicalProcessorCount(); if (NumCores == -1) { NumCores = System.Environment.ProcessorCount; } // The number of actions to execute in parallel is trying to keep the CPU busy enough in presence of I/O stalls. int MaxActionsToExecuteInParallel = 0; if (NumCores < System.Environment.ProcessorCount && ProcessorCountMultiplier != 1.0) { // The CPU has more logical cores than physical ones, aka uses hyper-threading. // Use multiplier if provided MaxActionsToExecuteInParallel = (int)(NumCores * ProcessorCountMultiplier); } else if (NumCores < System.Environment.ProcessorCount && NumCores > 4) { // The CPU has more logical cores than physical ones, aka uses hyper-threading. // Use average of logical and physical if we have "lots of cores" MaxActionsToExecuteInParallel = (int)(NumCores + System.Environment.ProcessorCount) / 2; } // No hyper-threading. Only kicking off a task per CPU to keep machine responsive. else { MaxActionsToExecuteInParallel = NumCores; } if (Utils.IsRunningOnMono) { // heuristic: give each action at least 1.5GB of RAM (some clang instances will need more, actually) long MinMemoryPerActionMB = 3 * 1024 / 2; long PhysicalRAMAvailableMB = (new PerformanceCounter("Mono Memory", "Total Physical Memory").RawValue) / (1024 * 1024); int MaxActionsAffordedByMemory = (int)(Math.Max(1, (PhysicalRAMAvailableMB) / MinMemoryPerActionMB)); MaxActionsToExecuteInParallel = Math.Min(MaxActionsToExecuteInParallel, MaxActionsAffordedByMemory); } MaxActionsToExecuteInParallel = Math.Max(1, Math.Min(MaxActionsToExecuteInParallel, MaxProcessorCount)); Log.TraceInformation("Performing {0} actions ({1} in parallel)", Actions.Count, MaxActionsToExecuteInParallel); Dictionary <Action, ActionThread> ActionThreadDictionary = new Dictionary <Action, ActionThread>(); int JobNumber = 1; using (ProgressWriter ProgressWriter = new ProgressWriter("Compiling C++ source code...", false)) { int ProgressValue = 0; while (true) { // Count the number of pending and still executing actions. int NumUnexecutedActions = 0; int NumExecutingActions = 0; foreach (Action Action in Actions) { ActionThread ActionThread = null; bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread); if (bFoundActionProcess == false) { NumUnexecutedActions++; } else if (ActionThread != null) { if (ActionThread.bComplete == false) { NumUnexecutedActions++; NumExecutingActions++; } } } // Update the current progress int NewProgressValue = Actions.Count + 1 - NumUnexecutedActions; if (ProgressValue != NewProgressValue) { ProgressWriter.Write(ProgressValue, Actions.Count + 1); ProgressValue = NewProgressValue; } // If there aren't any pending actions left, we're done executing. if (NumUnexecutedActions == 0) { break; } // If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated // prerequisites. foreach (Action Action in Actions) { ActionThread ActionProcess = null; bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess); if (bFoundActionProcess == false) { if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel)) { // Determine whether there are any prerequisites of the action that are outdated. bool bHasOutdatedPrerequisites = false; bool bHasFailedPrerequisites = false; foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems) { if (PrerequisiteItem.ProducingAction != null && Actions.Contains(PrerequisiteItem.ProducingAction)) { ActionThread PrerequisiteProcess = null; bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue(PrerequisiteItem.ProducingAction, out PrerequisiteProcess); if (bFoundPrerequisiteProcess == true) { if (PrerequisiteProcess == null) { bHasFailedPrerequisites = true; } else if (PrerequisiteProcess.bComplete == false) { bHasOutdatedPrerequisites = true; } else if (PrerequisiteProcess.ExitCode != 0) { bHasFailedPrerequisites = true; } } else { bHasOutdatedPrerequisites = true; } } } // If there are any failed prerequisites of this action, don't execute it. if (bHasFailedPrerequisites) { // Add a null entry in the dictionary for this action. ActionThreadDictionary.Add(Action, null); } // If there aren't any outdated prerequisites of this action, execute it. else if (!bHasOutdatedPrerequisites) { ActionThread ActionThread = new ActionThread(Action, JobNumber, Actions.Count); JobNumber++; try { ActionThread.Run(); } catch (Exception ex) { throw new BuildException(ex, "Failed to start thread for action: {0} {1}\r\n{2}", Action.CommandPath, Action.CommandArguments, ex.ToString()); } ActionThreadDictionary.Add(Action, ActionThread); NumExecutingActions++; } } } } System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime)); } } Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "-------- Begin Detailed Action Stats ----------------------------------------------------------"); Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "^Action Type^Duration (seconds)^Tool^Task^Using PCH"); double TotalThreadSeconds = 0; // Check whether any of the tasks failed and log action stats if wanted. bool bSuccess = true; double TotalBuildProjectTime = 0; double TotalCompileTime = 0; double TotalCreateAppBundleTime = 0; double TotalGenerateDebugInfoTime = 0; double TotalLinkTime = 0; double TotalOtherActionsTime = 0; foreach (KeyValuePair <Action, ActionThread> ActionProcess in ActionThreadDictionary) { Action Action = ActionProcess.Key; ActionThread ActionThread = ActionProcess.Value; // Check for pending actions, preemptive failure if (ActionThread == null) { bSuccess = false; continue; } // Check for executed action but general failure if (ActionThread.ExitCode != 0) { bSuccess = false; } // Log CPU time, tool and task. double ThreadSeconds = Action.Duration.TotalSeconds; Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "^{0}^{1:0.00}^{2}^{3}^{4}", Action.ActionType.ToString(), ThreadSeconds, Path.GetFileName(Action.CommandPath), Action.StatusDescription, Action.bIsUsingPCH); // Update statistics switch (Action.ActionType) { case ActionType.BuildProject: TotalBuildProjectTime += ThreadSeconds; break; case ActionType.Compile: TotalCompileTime += ThreadSeconds; break; case ActionType.CreateAppBundle: TotalCreateAppBundleTime += ThreadSeconds; break; case ActionType.GenerateDebugInfo: TotalGenerateDebugInfoTime += ThreadSeconds; break; case ActionType.Link: TotalLinkTime += ThreadSeconds; break; default: TotalOtherActionsTime += ThreadSeconds; break; } // Keep track of total thread seconds spent on tasks. TotalThreadSeconds += ThreadSeconds; } Log.WriteLineIf(bLogDetailedActionStats, LogEventType.Console, "-------- End Detailed Actions Stats -----------------------------------------------------------"); // Log total CPU seconds and numbers of processors involved in tasks. Log.WriteLineIf(bLogDetailedActionStats || UnrealBuildTool.bPrintDebugInfo, LogEventType.Console, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds); // Log detailed stats Log.WriteLineIf(bLogDetailedActionStats || UnrealBuildTool.bPrintDebugInfo, LogEventType.Console, "Cumulative action seconds ({0} processors): {1:0.00} building projects, {2:0.00} compiling, {3:0.00} creating app bundles, {4:0.00} generating debug info, {5:0.00} linking, {6:0.00} other", System.Environment.ProcessorCount, TotalBuildProjectTime, TotalCompileTime, TotalCreateAppBundleTime, TotalGenerateDebugInfoTime, TotalLinkTime, TotalOtherActionsTime ); return(bSuccess); }
internal static ExecutionResult ExecuteLocalActions(List<Action> InLocalActions, Dictionary<Action, ActionThread> InActionThreadDictionary, int TotalNumJobs) { // Time to sleep after each iteration of the loop in order to not busy wait. const float LoopSleepTime = 0.1f; ExecutionResult LocalActionsResult = ExecutionResult.TasksSucceeded; while (true) { // Count the number of pending and still executing actions. int NumUnexecutedActions = 0; int NumExecutingActions = 0; foreach (Action Action in InLocalActions) { ActionThread ActionThread = null; bool bFoundActionProcess = InActionThreadDictionary.TryGetValue(Action, out ActionThread); if (bFoundActionProcess == false) { NumUnexecutedActions++; } else if (ActionThread != null) { if (ActionThread.bComplete == false) { NumUnexecutedActions++; NumExecutingActions++; } } } // If there aren't any pending actions left, we're done executing. if (NumUnexecutedActions == 0) { break; } // If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated // prerequisites. foreach (Action Action in InLocalActions) { ActionThread ActionProcess = null; bool bFoundActionProcess = InActionThreadDictionary.TryGetValue(Action, out ActionProcess); if (bFoundActionProcess == false) { if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel)) { // Determine whether there are any prerequisites of the action that are outdated. bool bHasOutdatedPrerequisites = false; bool bHasFailedPrerequisites = false; foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems) { if (PrerequisiteItem.ProducingAction != null && InLocalActions.Contains(PrerequisiteItem.ProducingAction)) { ActionThread PrerequisiteProcess = null; bool bFoundPrerequisiteProcess = InActionThreadDictionary.TryGetValue(PrerequisiteItem.ProducingAction, out PrerequisiteProcess); if (bFoundPrerequisiteProcess == true) { if (PrerequisiteProcess == null) { bHasFailedPrerequisites = true; } else if (PrerequisiteProcess.bComplete == false) { bHasOutdatedPrerequisites = true; } else if (PrerequisiteProcess.ExitCode != 0) { bHasFailedPrerequisites = true; } } else { bHasOutdatedPrerequisites = true; } } } // If there are any failed prerequisites of this action, don't execute it. if (bHasFailedPrerequisites) { // Add a null entry in the dictionary for this action. InActionThreadDictionary.Add(Action, null); } // If there aren't any outdated prerequisites of this action, execute it. else if (!bHasOutdatedPrerequisites) { ActionThread ActionThread = new ActionThread(Action, JobNumber, TotalNumJobs); ActionThread.Run(); InActionThreadDictionary.Add(Action, ActionThread); NumExecutingActions++; JobNumber++; } } } } System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime)); } return LocalActionsResult; }
/** * Executes the specified actions locally. * @return True if all the tasks successfully executed, or false if any of them failed. */ public static bool ExecuteActions(List<Action> Actions) { // Time to sleep after each iteration of the loop in order to not busy wait. const float LoopSleepTime = 0.1f; // Use WMI to figure out physical cores, excluding hyper threading. int NumCores = 0; if (!Utils.IsRunningOnMono) { foreach(var Item in new System.Management.ManagementObjectSearcher("Select * from Win32_Processor").Get()) { NumCores += int.Parse(Item["NumberOfCores"].ToString()); } } // On some systems this requires a hot fix to work so we fall back to using the (logical) processor count. if( NumCores == 0 ) { NumCores = System.Environment.ProcessorCount; } // The number of actions to execute in parallel is trying to keep the CPU busy enough in presence of I/O stalls. int MaxActionsToExecuteInParallel = 0; // The CPU has more logical cores than physical ones, aka uses hyper-threading. if( NumCores < System.Environment.ProcessorCount ) { MaxActionsToExecuteInParallel = (int) (NumCores * BuildConfiguration.ProcessorCountMultiplier); } // No hyper-threading. Only kicking off a task per CPU to keep machine responsive. else { MaxActionsToExecuteInParallel = NumCores; } if( Utils.IsRunningOnMono ) { // @todo iosmerge: this should be looking at available memory as well since some of our // Macs have a poor memory/CPU ratio MaxActionsToExecuteInParallel /= 2; } Dictionary<Action, ActionThread> ActionThreadDictionary = new Dictionary<Action, ActionThread>(); while(true) { // Count the number of pending and still executing actions. int NumUnexecutedActions = 0; int NumExecutingActions = 0; foreach (Action Action in Actions) { ActionThread ActionThread = null; bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionThread); if (bFoundActionProcess == false) { NumUnexecutedActions++; } else if (ActionThread != null) { if (ActionThread.bComplete == false) { NumUnexecutedActions++; NumExecutingActions++; } } } // If there aren't any pending actions left, we're done executing. if (NumUnexecutedActions == 0) { break; } // If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated // prerequisites. foreach (Action Action in Actions) { ActionThread ActionProcess = null; bool bFoundActionProcess = ActionThreadDictionary.TryGetValue(Action, out ActionProcess); if (bFoundActionProcess == false) { if (NumExecutingActions < Math.Max(1,MaxActionsToExecuteInParallel)) { // Determine whether there are any prerequisites of the action that are outdated. bool bHasOutdatedPrerequisites = false; bool bHasFailedPrerequisites = false; foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems) { if (PrerequisiteItem.ProducingAction != null && Actions.Contains(PrerequisiteItem.ProducingAction)) { ActionThread PrerequisiteProcess = null; bool bFoundPrerequisiteProcess = ActionThreadDictionary.TryGetValue( PrerequisiteItem.ProducingAction, out PrerequisiteProcess ); if (bFoundPrerequisiteProcess == true) { if (PrerequisiteProcess == null) { bHasFailedPrerequisites = true; } else if (PrerequisiteProcess.bComplete == false) { bHasOutdatedPrerequisites = true; } else if (PrerequisiteProcess.ExitCode != 0) { bHasFailedPrerequisites = true; } } else { bHasOutdatedPrerequisites = true; } } } // If there are any failed prerequisites of this action, don't execute it. if (bHasFailedPrerequisites) { // Add a null entry in the dictionary for this action. ActionThreadDictionary.Add( Action, null ); } // If there aren't any outdated prerequisites of this action, execute it. else if (!bHasOutdatedPrerequisites) { ActionThread ActionThread = new ActionThread(Action); ActionThread.Run(); ActionThreadDictionary.Add(Action, ActionThread); NumExecutingActions++; } } } } System.Threading.Thread.Sleep(TimeSpan.FromSeconds(LoopSleepTime)); } Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "-------- Begin Detailed Action Stats ----------------------------------------------------------"); Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "^Action Type^Duration (seconds)^Tool^Task^Using PCH^Description"); double TotalThreadSeconds = 0; // Check whether any of the tasks failed and log action stats if wanted. bool bSuccess = true; foreach (KeyValuePair<Action, ActionThread> ActionProcess in ActionThreadDictionary) { Action Action = ActionProcess.Key; ActionThread ActionThread = ActionProcess.Value; // Check for pending actions, preemptive failure if (ActionThread == null) { bSuccess = false; continue; } // Check for executed action but general failure if (ActionThread.ExitCode != 0) { bSuccess = false; } // Log CPU time, tool and task. double ThreadSeconds = Action.Duration.TotalSeconds; Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats, TraceEventType.Information, "^{0}^{1:0.00}^{2}^{3}^{4}^{5}", Action.ActionType.ToString(), ThreadSeconds, Path.GetFileName(Action.CommandPath), Action.StatusDescription, Action.bIsUsingPCH, Action.StatusDetailedDescription); // Update statistics switch (Action.ActionType) { case ActionType.BuildProject: UnrealBuildTool.TotalBuildProjectTime += ThreadSeconds; break; case ActionType.Compile: UnrealBuildTool.TotalCompileTime += ThreadSeconds; break; case ActionType.CreateAppBundle: UnrealBuildTool.TotalCreateAppBundleTime += ThreadSeconds; break; case ActionType.GenerateDebugInfo: UnrealBuildTool.TotalGenerateDebugInfoTime += ThreadSeconds; break; case ActionType.Link: UnrealBuildTool.TotalLinkTime += ThreadSeconds; break; default: UnrealBuildTool.TotalOtherActionsTime += ThreadSeconds; break; } // Keep track of total thread seconds spent on tasks. TotalThreadSeconds += ThreadSeconds; } Log.TraceInformation("-------- End Detailed Actions Stats -----------------------------------------------------------"); // Log total CPU seconds and numbers of processors involved in tasks. Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats || BuildConfiguration.bPrintDebugInfo, TraceEventType.Information, "Cumulative thread seconds ({0} processors): {1:0.00}", System.Environment.ProcessorCount, TotalThreadSeconds); return bSuccess; }