/// <summary> /// Build a list of targets /// </summary> /// <param name="TargetDescriptors">Target descriptors</param> /// <param name="BuildConfiguration">Current build configuration</param> /// <param name="WorkingSet">The source file working set</param> /// <param name="Options">Additional options for the build</param> /// <returns>Result from the compilation</returns> public static void Build(List <TargetDescriptor> TargetDescriptors, BuildConfiguration BuildConfiguration, ISourceFileWorkingSet WorkingSet, BuildOptions Options) { // Create a makefile for each target TargetMakefile[] Makefiles = new TargetMakefile[TargetDescriptors.Count]; for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++) { Makefiles[TargetIdx] = CreateMakefile(BuildConfiguration, TargetDescriptors[TargetIdx], WorkingSet); } // Output the manifest for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++) { TargetDescriptor TargetDescriptor = TargetDescriptors[TargetIdx]; if (TargetDescriptor.LiveCodingManifest != null) { TargetMakefile Makefile = Makefiles[TargetIdx]; HotReload.WriteLiveCodingManifest(TargetDescriptor.LiveCodingManifest, Makefile.Actions); } } // Execute the build if ((Options & BuildOptions.SkipBuild) == 0) { // Make sure that none of the actions conflict with any other (producing output files differently, etc...) ActionGraph.CheckForConflicts(Makefiles.SelectMany(x => x.Actions)); // Find all the actions to be executed HashSet <Action>[] ActionsToExecute = new HashSet <Action> [TargetDescriptors.Count]; for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++) { ActionsToExecute[TargetIdx] = GetActionsForTarget(BuildConfiguration, TargetDescriptors[TargetIdx], Makefiles[TargetIdx]); } // If there are multiple targets being built, merge the actions together List <Action> MergedActionsToExecute; if (TargetDescriptors.Count == 1) { MergedActionsToExecute = new List <Action>(ActionsToExecute[0]); } else { MergedActionsToExecute = MergeActionGraphs(TargetDescriptors, ActionsToExecute); } // Link all the actions together ActionGraph.Link(MergedActionsToExecute); // Make sure the appropriate executor is selected foreach (TargetDescriptor TargetDescriptor in TargetDescriptors) { UEBuildPlatform BuildPlatform = UEBuildPlatform.GetBuildPlatform(TargetDescriptor.Platform); BuildConfiguration.bAllowXGE &= BuildPlatform.CanUseXGE(); BuildConfiguration.bAllowDistcc &= BuildPlatform.CanUseDistcc(); BuildConfiguration.bAllowSNDBS &= BuildPlatform.CanUseSNDBS(); } // Delete produced items that are outdated. ActionGraph.DeleteOutdatedProducedItems(MergedActionsToExecute); // Save all the action histories now that files have been removed. We have to do this after deleting produced items to ensure that any // items created during the build don't have the wrong command line. ActionHistory.SaveAll(); // Create directories for the outdated produced items. ActionGraph.CreateDirectoriesForProducedItems(MergedActionsToExecute); // Execute the actions if ((Options & BuildOptions.XGEExport) != 0) { // Just export to an XML file using (Timeline.ScopeEvent("XGE.ExportActions()")) { XGE.ExportActions(MergedActionsToExecute); } } else { // Execute the actions if (MergedActionsToExecute.Count == 0) { if ((Options & BuildOptions.Quiet) == 0) { Log.TraceInformation((TargetDescriptors.Count == 1)? "Target is up to date" : "Targets are up to date"); } } else { if ((Options & BuildOptions.Quiet) != 0) { Log.TraceInformation("Building {0}...", StringUtils.FormatList(TargetDescriptors.Select(x => x.Name).Distinct())); } OutputToolchainInfo(TargetDescriptors, Makefiles); using (Timeline.ScopeEvent("ActionGraph.ExecuteActions()")) { ActionGraph.ExecuteActions(BuildConfiguration, MergedActionsToExecute); } } // Run the deployment steps foreach (TargetMakefile Makefile in Makefiles) { if (Makefile.bDeployAfterCompile) { TargetReceipt Receipt = TargetReceipt.Read(Makefile.ReceiptFile); Log.TraceInformation("Deploying {0} {1} {2}...", Receipt.TargetName, Receipt.Platform, Receipt.Configuration); UEBuildPlatform.GetBuildPlatform(Receipt.Platform).Deploy(Receipt); } } } } }
/// <summary> /// Determine what needs to be built for a target /// </summary> /// <param name="BuildConfiguration">The build configuration</param> /// <param name="TargetDescriptor">Target being built</param> /// <param name="Makefile">Makefile generated for this target</param> /// <returns>Set of actions to execute</returns> static HashSet <Action> GetActionsForTarget(BuildConfiguration BuildConfiguration, TargetDescriptor TargetDescriptor, TargetMakefile Makefile) { // Create the action graph ActionGraph.Link(Makefile.Actions); // Get the hot-reload mode HotReloadMode HotReloadMode = TargetDescriptor.HotReloadMode; if (HotReloadMode == HotReloadMode.Default) { if (TargetDescriptor.HotReloadModuleNameToSuffix.Count > 0 && TargetDescriptor.ForeignPlugin == null) { HotReloadMode = HotReloadMode.FromEditor; } else if (BuildConfiguration.bAllowHotReloadFromIDE && HotReload.ShouldDoHotReloadFromIDE(BuildConfiguration, TargetDescriptor)) { HotReloadMode = HotReloadMode.FromIDE; } else { HotReloadMode = HotReloadMode.Disabled; } } // Guard against a live coding session for this target being active if (BuildConfiguration.bAllowHotReloadFromIDE && HotReloadMode != HotReloadMode.LiveCoding && TargetDescriptor.ForeignPlugin == null && HotReload.IsLiveCodingSessionActive(Makefile)) { throw new BuildException("Unable to start regular build while Live Coding is active. Press Ctrl+Alt+F11 to trigger a Live Coding compile."); } // Get the root prerequisite actions List <Action> PrerequisiteActions = GatherPrerequisiteActions(TargetDescriptor, Makefile); // Get the path to the hot reload state file for this target FileReference HotReloadStateFile = global::UnrealBuildTool.HotReloadState.GetLocation(TargetDescriptor.ProjectFile, TargetDescriptor.Name, TargetDescriptor.Platform, TargetDescriptor.Configuration, TargetDescriptor.Architecture); // Apply the previous hot reload state HotReloadState HotReloadState = null; if (HotReloadMode == HotReloadMode.Disabled) { // Make sure we're not doing a partial build from the editor (eg. compiling a new plugin) if (TargetDescriptor.ForeignPlugin == null && TargetDescriptor.SingleFileToCompile == null) { // Delete the previous state file HotReload.DeleteTemporaryFiles(HotReloadStateFile); } } else { // Read the previous state file and apply it to the action graph if (FileReference.Exists(HotReloadStateFile)) { HotReloadState = HotReloadState.Load(HotReloadStateFile); } else { HotReloadState = new HotReloadState(); } // Apply the old state to the makefile HotReload.ApplyState(HotReloadState, Makefile); // If we want a specific suffix on any modules, apply that now. We'll track the outputs later, but the suffix has to be forced (and is always out of date if it doesn't exist). HotReload.PatchActionGraphWithNames(PrerequisiteActions, TargetDescriptor.HotReloadModuleNameToSuffix, Makefile); // Re-link the action graph ActionGraph.Link(PrerequisiteActions); } // Create the dependencies cache CppDependencyCache CppDependencies; using (Timeline.ScopeEvent("Reading dependency cache")) { CppDependencies = CppDependencyCache.CreateHierarchy(TargetDescriptor.ProjectFile, TargetDescriptor.Name, TargetDescriptor.Platform, TargetDescriptor.Configuration, Makefile.TargetType, TargetDescriptor.Architecture); } // Create the action history ActionHistory History; using (Timeline.ScopeEvent("Reading action history")) { History = ActionHistory.CreateHierarchy(TargetDescriptor.ProjectFile, TargetDescriptor.Name, TargetDescriptor.Platform, Makefile.TargetType, TargetDescriptor.Architecture); } // Plan the actions to execute for the build. For single file compiles, always rebuild the source file regardless of whether it's out of date. HashSet <Action> TargetActionsToExecute; if (TargetDescriptor.SingleFileToCompile == null) { TargetActionsToExecute = ActionGraph.GetActionsToExecute(Makefile.Actions, PrerequisiteActions, CppDependencies, History, BuildConfiguration.bIgnoreOutdatedImportLibraries); } else { TargetActionsToExecute = new HashSet <Action>(PrerequisiteActions); } // Additional processing for hot reload if (HotReloadMode == HotReloadMode.LiveCoding) { // Make sure we're not overwriting any lazy-loaded modules if (TargetDescriptor.LiveCodingModules != null) { // Read the list of modules that we're allowed to build string[] Lines = FileReference.ReadAllLines(TargetDescriptor.LiveCodingModules); // Parse it out into a set of filenames HashSet <string> AllowedOutputFileNames = new HashSet <string>(FileReference.Comparer); foreach (string Line in Lines) { string TrimLine = Line.Trim(); if (TrimLine.Length > 0) { AllowedOutputFileNames.Add(Path.GetFileName(TrimLine)); } } // Find all the binaries that we're actually going to build HashSet <FileReference> OutputFiles = new HashSet <FileReference>(); foreach (Action Action in TargetActionsToExecute) { if (Action.ActionType == ActionType.Link) { OutputFiles.UnionWith(Action.ProducedItems.Where(x => x.HasExtension(".exe") || x.HasExtension(".dll")).Select(x => x.Location)); } } // Find all the files that will be built that aren't allowed List <FileReference> ProtectedOutputFiles = OutputFiles.Where(x => !AllowedOutputFileNames.Contains(x.GetFileName())).ToList(); if (ProtectedOutputFiles.Count > 0) { FileReference.WriteAllLines(new FileReference(TargetDescriptor.LiveCodingModules.FullName + ".out"), ProtectedOutputFiles.Select(x => x.ToString())); foreach (FileReference ProtectedOutputFile in ProtectedOutputFiles) { Log.TraceInformation("Module {0} is not currently enabled for Live Coding", ProtectedOutputFile); } throw new CompilationResultException(CompilationResult.Canceled); } } // Filter the prerequisite actions down to just the compile actions, then recompute all the actions to execute PrerequisiteActions = new List <Action>(TargetActionsToExecute.Where(x => x.ActionType == ActionType.Compile)); TargetActionsToExecute = ActionGraph.GetActionsToExecute(Makefile.Actions, PrerequisiteActions, CppDependencies, History, BuildConfiguration.bIgnoreOutdatedImportLibraries); // Update the action graph with these new paths Dictionary <FileReference, FileReference> OriginalFileToPatchedFile = new Dictionary <FileReference, FileReference>(); HotReload.PatchActionGraphForLiveCoding(PrerequisiteActions, OriginalFileToPatchedFile); // Get a new list of actions to execute now that the graph has been modified TargetActionsToExecute = ActionGraph.GetActionsToExecute(Makefile.Actions, PrerequisiteActions, CppDependencies, History, BuildConfiguration.bIgnoreOutdatedImportLibraries); // Output the Live Coding manifest if (TargetDescriptor.LiveCodingManifest != null) { HotReload.WriteLiveCodingManifest(TargetDescriptor.LiveCodingManifest, Makefile.Actions, OriginalFileToPatchedFile); } } else if (HotReloadMode == HotReloadMode.FromEditor || HotReloadMode == HotReloadMode.FromIDE) { // Patch action history for hot reload when running in assembler mode. In assembler mode, the suffix on the output file will be // the same for every invocation on that makefile, but we need a new suffix each time. // For all the hot-reloadable modules that may need a unique suffix appended, build a mapping from output item to all the output items in that module. We can't // apply a suffix to one without applying a suffix to all of them. Dictionary <FileItem, FileItem[]> HotReloadItemToDependentItems = new Dictionary <FileItem, FileItem[]>(); foreach (string HotReloadModuleName in Makefile.HotReloadModuleNames) { int ModuleSuffix; if (!TargetDescriptor.HotReloadModuleNameToSuffix.TryGetValue(HotReloadModuleName, out ModuleSuffix) || ModuleSuffix == -1) { FileItem[] ModuleOutputItems; if (Makefile.ModuleNameToOutputItems.TryGetValue(HotReloadModuleName, out ModuleOutputItems)) { foreach (FileItem ModuleOutputItem in ModuleOutputItems) { HotReloadItemToDependentItems[ModuleOutputItem] = ModuleOutputItems; } } } } // Expand the list of actions to execute to include everything that references any files with a new suffix. Unlike a regular build, we can't ignore // dependencies on import libraries under the assumption that a header would change if the API changes, because the dependency will be on a different DLL. HashSet <FileItem> FilesRequiringSuffix = new HashSet <FileItem>(TargetActionsToExecute.SelectMany(x => x.ProducedItems).Where(x => HotReloadItemToDependentItems.ContainsKey(x))); for (int LastNumFilesWithNewSuffix = 0; FilesRequiringSuffix.Count > LastNumFilesWithNewSuffix;) { LastNumFilesWithNewSuffix = FilesRequiringSuffix.Count; foreach (Action PrerequisiteAction in PrerequisiteActions) { if (!TargetActionsToExecute.Contains(PrerequisiteAction)) { foreach (FileItem ProducedItem in PrerequisiteAction.ProducedItems) { FileItem[] DependentItems; if (HotReloadItemToDependentItems.TryGetValue(ProducedItem, out DependentItems)) { TargetActionsToExecute.Add(PrerequisiteAction); FilesRequiringSuffix.UnionWith(DependentItems); } } } } } // Build a list of file mappings Dictionary <FileReference, FileReference> OldLocationToNewLocation = new Dictionary <FileReference, FileReference>(); foreach (FileItem FileRequiringSuffix in FilesRequiringSuffix) { FileReference OldLocation = FileRequiringSuffix.Location; FileReference NewLocation = HotReload.ReplaceSuffix(OldLocation, HotReloadState.NextSuffix); OldLocationToNewLocation[OldLocation] = NewLocation; } // Update the action graph with these new paths HotReload.PatchActionGraph(PrerequisiteActions, OldLocationToNewLocation); // Get a new list of actions to execute now that the graph has been modified TargetActionsToExecute = ActionGraph.GetActionsToExecute(Makefile.Actions, PrerequisiteActions, CppDependencies, History, BuildConfiguration.bIgnoreOutdatedImportLibraries); // Build a mapping of all file items to their original Dictionary <FileReference, FileReference> HotReloadFileToOriginalFile = new Dictionary <FileReference, FileReference>(); foreach (KeyValuePair <FileReference, FileReference> Pair in HotReloadState.OriginalFileToHotReloadFile) { HotReloadFileToOriginalFile[Pair.Value] = Pair.Key; } foreach (KeyValuePair <FileReference, FileReference> Pair in OldLocationToNewLocation) { FileReference OriginalLocation; if (!HotReloadFileToOriginalFile.TryGetValue(Pair.Key, out OriginalLocation)) { OriginalLocation = Pair.Key; } HotReloadFileToOriginalFile[Pair.Value] = OriginalLocation; } // Now filter out all the hot reload files and update the state foreach (Action Action in TargetActionsToExecute) { foreach (FileItem ProducedItem in Action.ProducedItems) { FileReference OriginalLocation; if (HotReloadFileToOriginalFile.TryGetValue(ProducedItem.Location, out OriginalLocation)) { HotReloadState.OriginalFileToHotReloadFile[OriginalLocation] = ProducedItem.Location; HotReloadState.TemporaryFiles.Add(ProducedItem.Location); } } } // Increment the suffix for the next iteration if (TargetActionsToExecute.Count > 0) { HotReloadState.NextSuffix++; } // Save the new state HotReloadState.Save(HotReloadStateFile); // Prevent this target from deploying Makefile.bDeployAfterCompile = false; } return(TargetActionsToExecute); }
/// <summary> /// Build a list of targets with a given set of makefiles. /// </summary> /// <param name="Makefiles">Makefiles created with CreateMakefiles</param> /// <param name="TargetDescriptors">Target descriptors</param> /// <param name="BuildConfiguration">Current build configuration</param> /// <param name="WorkingSet">The source file working set</param> /// <param name="Options">Additional options for the build</param> /// <param name="WriteOutdatedActionsFile">Files to write the list of outdated actions to (rather than building them)</param> /// <returns>Result from the compilation</returns> static void Build(TargetMakefile[] Makefiles, List <TargetDescriptor> TargetDescriptors, BuildConfiguration BuildConfiguration, ISourceFileWorkingSet WorkingSet, BuildOptions Options, FileReference WriteOutdatedActionsFile) { // Export the actions for each target for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++) { TargetDescriptor TargetDescriptor = TargetDescriptors[TargetIdx]; foreach (FileReference WriteActionFile in TargetDescriptor.WriteActionFiles) { Log.TraceInformation("Writing actions to {0}", WriteActionFile); ActionGraph.ExportJson(Makefiles[TargetIdx].Actions, WriteActionFile); } } // Execute the build if ((Options & BuildOptions.SkipBuild) == 0) { // Make sure that none of the actions conflict with any other (producing output files differently, etc...) ActionGraph.CheckForConflicts(Makefiles.SelectMany(x => x.Actions)); // Check we don't exceed the nominal max path length using (Timeline.ScopeEvent("ActionGraph.CheckPathLengths")) { ActionGraph.CheckPathLengths(BuildConfiguration, Makefiles.SelectMany(x => x.Actions)); } // Clean up any previous hot reload runs, and reapply the current state if it's already active for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++) { HotReload.Setup(TargetDescriptors[TargetIdx], Makefiles[TargetIdx], BuildConfiguration); } // Merge the action graphs together List <Action> MergedActions; if (TargetDescriptors.Count == 1) { MergedActions = new List <Action>(Makefiles[0].Actions); } else { MergedActions = MergeActionGraphs(TargetDescriptors, Makefiles); } // Gather all the prerequisite actions that are part of the targets HashSet <FileItem> MergedOutputItems = new HashSet <FileItem>(); for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++) { GatherOutputItems(TargetDescriptors[TargetIdx], Makefiles[TargetIdx], MergedOutputItems); } // Link all the actions together ActionGraph.Link(MergedActions); // Get all the actions that are prerequisites for these targets. This forms the list of actions that we want executed. List <Action> PrerequisiteActions = ActionGraph.GatherPrerequisiteActions(MergedActions, MergedOutputItems); // Create the action history ActionHistory History = new ActionHistory(); for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++) { using (Timeline.ScopeEvent("Reading action history")) { TargetDescriptor TargetDescriptor = TargetDescriptors[TargetIdx]; if (TargetDescriptor.ProjectFile != null) { History.Mount(TargetDescriptor.ProjectFile.Directory); } } } // Figure out which actions need to be built Dictionary <Action, bool> ActionToOutdatedFlag = new Dictionary <Action, bool>(); for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++) { TargetDescriptor TargetDescriptor = TargetDescriptors[TargetIdx]; // Create the dependencies cache CppDependencyCache CppDependencies; using (Timeline.ScopeEvent("Reading dependency cache")) { CppDependencies = CppDependencyCache.CreateHierarchy(TargetDescriptor.ProjectFile, TargetDescriptor.Name, TargetDescriptor.Platform, TargetDescriptor.Configuration, Makefiles[TargetIdx].TargetType, TargetDescriptor.Architecture); } // Plan the actions to execute for the build. For single file compiles, always rebuild the source file regardless of whether it's out of date. if (TargetDescriptor.SpecificFilesToCompile.Count == 0) { ActionGraph.GatherAllOutdatedActions(PrerequisiteActions, History, ActionToOutdatedFlag, CppDependencies, BuildConfiguration.bIgnoreOutdatedImportLibraries); } else { foreach (FileReference SpecificFile in TargetDescriptor.SpecificFilesToCompile) { foreach (Action PrerequisiteAction in PrerequisiteActions.Where(x => x.PrerequisiteItems.Any(y => y.Location == SpecificFile))) { ActionToOutdatedFlag[PrerequisiteAction] = true; } } } } // Link the action graph again to sort it List <Action> MergedActionsToExecute = ActionToOutdatedFlag.Where(x => x.Value).Select(x => x.Key).ToList(); ActionGraph.Link(MergedActionsToExecute); // Allow hot reload to override the actions int HotReloadTargetIdx = -1; for (int Idx = 0; Idx < TargetDescriptors.Count; Idx++) { if (TargetDescriptors[Idx].HotReloadMode != HotReloadMode.Disabled) { if (HotReloadTargetIdx != -1) { throw new BuildException("Unable to perform hot reload with multiple targets."); } else { MergedActionsToExecute = HotReload.PatchActionsForTarget(BuildConfiguration, TargetDescriptors[Idx], Makefiles[Idx], PrerequisiteActions, MergedActionsToExecute); } HotReloadTargetIdx = Idx; } } // Make sure we're not modifying any engine files if ((Options & BuildOptions.NoEngineChanges) != 0) { List <FileItem> EngineChanges = MergedActionsToExecute.SelectMany(x => x.ProducedItems).Where(x => x.Location.IsUnderDirectory(UnrealBuildTool.EngineDirectory)).Distinct().OrderBy(x => x.FullName).ToList(); if (EngineChanges.Count > 0) { StringBuilder Result = new StringBuilder("Building would modify the following engine files:\n"); foreach (FileItem EngineChange in EngineChanges) { Result.AppendFormat("\n{0}", EngineChange.FullName); } Result.Append("\n\nPlease rebuild from an IDE instead."); Log.TraceError("{0}", Result.ToString()); throw new CompilationResultException(CompilationResult.FailedDueToEngineChange); } } // Make sure the appropriate executor is selected foreach (TargetDescriptor TargetDescriptor in TargetDescriptors) { UEBuildPlatform BuildPlatform = UEBuildPlatform.GetBuildPlatform(TargetDescriptor.Platform); BuildConfiguration.bAllowXGE &= BuildPlatform.CanUseXGE(); BuildConfiguration.bAllowDistcc &= BuildPlatform.CanUseDistcc(); BuildConfiguration.bAllowSNDBS &= BuildPlatform.CanUseSNDBS(); } // Delete produced items that are outdated. ActionGraph.DeleteOutdatedProducedItems(MergedActionsToExecute); // Save all the action histories now that files have been removed. We have to do this after deleting produced items to ensure that any // items created during the build don't have the wrong command line. History.Save(); // Create directories for the outdated produced items. ActionGraph.CreateDirectoriesForProducedItems(MergedActionsToExecute); // Execute the actions if ((Options & BuildOptions.XGEExport) != 0) { OutputToolchainInfo(TargetDescriptors, Makefiles); // Just export to an XML file using (Timeline.ScopeEvent("XGE.ExportActions()")) { XGE.ExportActions(MergedActionsToExecute); } } else if (WriteOutdatedActionsFile != null) { OutputToolchainInfo(TargetDescriptors, Makefiles); // Write actions to an output file using (Timeline.ScopeEvent("ActionGraph.WriteActions")) { ActionGraph.ExportJson(MergedActionsToExecute, WriteOutdatedActionsFile); } } else { // Execute the actions if (MergedActionsToExecute.Count == 0) { if (TargetDescriptors.Any(x => !x.bQuiet)) { Log.TraceInformation((TargetDescriptors.Count == 1)? "Target is up to date" : "Targets are up to date"); } } else { if (TargetDescriptors.Any(x => !x.bQuiet)) { Log.TraceInformation("Building {0}...", StringUtils.FormatList(TargetDescriptors.Select(x => x.Name).Distinct())); } OutputToolchainInfo(TargetDescriptors, Makefiles); using (Timeline.ScopeEvent("ActionGraph.ExecuteActions()")) { ActionGraph.ExecuteActions(BuildConfiguration, MergedActionsToExecute); } } // Run the deployment steps foreach (TargetMakefile Makefile in Makefiles) { if (Makefile.bDeployAfterCompile) { TargetReceipt Receipt = TargetReceipt.Read(Makefile.ReceiptFile); Log.TraceInformation("Deploying {0} {1} {2}...", Receipt.TargetName, Receipt.Platform, Receipt.Configuration); UEBuildPlatform.GetBuildPlatform(Receipt.Platform).Deploy(Receipt); } } } } }