/// <summary>
/// Creates a cache hierarchy for a particular target
/// </summary>
/// <param name="ProjectFile">Project file for the target being built</param>
/// <param name="TargetName">Name of the target</param>
/// <param name="Platform">Platform being built</param>
/// <param name="Configuration">Configuration being built</param>
/// <param name="TargetType">The target type</param>
/// <param name="Architecture">The target architecture</param>
/// <returns>Dependency cache hierarchy for the given project</returns>
public static CppDependencyCache CreateHierarchy(FileReference ProjectFile, string TargetName, UnrealTargetPlatform Platform, UnrealTargetConfiguration Configuration, TargetType TargetType, string Architecture)
{
CppDependencyCache Cache = null;
if (ProjectFile == null || !UnrealBuildTool.IsEngineInstalled())
{
string AppName;
if (TargetType == TargetType.Program)
{
AppName = TargetName;
}
else
{
AppName = UEBuildTarget.GetAppNameForTargetType(TargetType);
}
FileReference EngineCacheLocation = FileReference.Combine(UnrealBuildTool.EngineDirectory, UEBuildTarget.GetPlatformIntermediateFolder(Platform, Architecture), AppName, Configuration.ToString(), "DependencyCache.bin");
Cache = FindOrAddCache(EngineCacheLocation, UnrealBuildTool.EngineDirectory, Cache);
}
if (ProjectFile != null)
{
FileReference ProjectCacheLocation = FileReference.Combine(ProjectFile.Directory, UEBuildTarget.GetPlatformIntermediateFolder(Platform, Architecture), TargetName, Configuration.ToString(), "DependencyCache.bin");
Cache = FindOrAddCache(ProjectCacheLocation, ProjectFile.Directory, Cache);
}
return(Cache);
}
/// <summary>
/// Constructs a dependency cache. This method is private; call CppDependencyCache.Create() to create a cache hierarchy for a given project.
/// </summary>
/// <param name="Location">File to store the cache</param>
/// <param name="BaseDir">Base directory for files that this cache should store data for</param>
/// <param name="Parent">The parent cache to use</param>
private CppDependencyCache(FileReference Location, DirectoryReference BaseDir, CppDependencyCache Parent)
{
this.Location = Location;
this.BaseDir = BaseDir;
this.Parent = Parent;
if (FileReference.Exists(Location))
{
Read();
}
}
/// <summary>
/// Builds a list of actions that need to be executed to produce the specified output items.
/// </summary>
public static List <Action> GetActionsToExecute(List <Action> Actions, CppDependencyCache CppDependencies, ActionHistory History, bool bIgnoreOutdatedImportLibraries)
{
using (Timeline.ScopeEvent("ActionGraph.GetActionsToExecute()"))
{
// For all targets, build a set of all actions that are outdated.
Dictionary <Action, bool> OutdatedActionDictionary = new Dictionary <Action, bool>();
GatherAllOutdatedActions(Actions, History, OutdatedActionDictionary, CppDependencies, bIgnoreOutdatedImportLibraries);
// Build a list of actions that are both needed for this target and outdated.
return(Actions.Where(Action => Action.CommandPath != null && OutdatedActionDictionary[Action]).ToList());
}
}
/// <summary>
/// Reads a cache from the given location, or creates it with the given settings
/// </summary>
/// <param name="Location">File to store the cache</param>
/// <param name="BaseDir">Base directory for files that this cache should store data for</param>
/// <param name="Parent">The parent cache to use</param>
/// <returns>Reference to a dependency cache with the given settings</returns>
static CppDependencyCache FindOrAddCache(FileReference Location, DirectoryReference BaseDir, CppDependencyCache Parent)
{
lock (Caches)
{
CppDependencyCache Cache;
if (Caches.TryGetValue(Location, out Cache))
{
Debug.Assert(Cache.BaseDir == BaseDir);
Debug.Assert(Cache.Parent == Parent);
}
else
{
Cache = new CppDependencyCache(Location, BaseDir, Parent);
Caches.Add(Location, Cache);
}
return(Cache);
}
}
/// <summary>
/// Builds a list of actions that need to be executed to produce the specified output items.
/// </summary>
public static HashSet <Action> GetActionsToExecute(List <Action> Actions, List <Action> PrerequisiteActions, CppDependencyCache CppDependencies, ActionHistory History, bool bIgnoreOutdatedImportLibraries)
{
ITimelineEvent GetActionsToExecuteTimer = Timeline.ScopeEvent("ActionGraph.GetActionsToExecute()");
// Build a set of all actions needed for this target.
Dictionary <Action, bool> IsActionOutdatedMap = new Dictionary <Action, bool>();
foreach (Action Action in PrerequisiteActions)
{
IsActionOutdatedMap.Add(Action, true);
}
// For all targets, build a set of all actions that are outdated.
Dictionary <Action, bool> OutdatedActionDictionary = new Dictionary <Action, bool>();
GatherAllOutdatedActions(Actions, History, OutdatedActionDictionary, CppDependencies, bIgnoreOutdatedImportLibraries);
// Build a list of actions that are both needed for this target and outdated.
HashSet <Action> ActionsToExecute = new HashSet <Action>(Actions.Where(Action => Action.CommandPath != null && IsActionOutdatedMap.ContainsKey(Action) && OutdatedActionDictionary[Action]));
GetActionsToExecuteTimer.Finish();
return(ActionsToExecute);
}
/// <summary>
/// Builds a dictionary containing the actions from AllActions that are outdated by calling
/// IsActionOutdated.
/// </summary>
static void GatherAllOutdatedActions(List <Action> Actions, ActionHistory ActionHistory, Dictionary <Action, bool> OutdatedActions, CppDependencyCache CppDependencies, bool bIgnoreOutdatedImportLibraries)
{
using (Timeline.ScopeEvent("Prefetching include dependencies"))
{
List <FileItem> Dependencies = new List <FileItem>();
foreach (Action Action in Actions)
{
if (Action.DependencyListFile != null)
{
Dependencies.Add(Action.DependencyListFile);
}
}
Parallel.ForEach(Dependencies, File => { List <FileItem> Temp; CppDependencies.TryGetDependencies(File, out Temp); });
}
using (Timeline.ScopeEvent("Cache outdated actions"))
{
Parallel.ForEach(Actions, Action => IsActionOutdated(Action, OutdatedActions, ActionHistory, CppDependencies, bIgnoreOutdatedImportLibraries));
}
}
/// <summary>
/// Determines whether an action is outdated based on the modification times for its prerequisite
/// and produced items.
/// </summary>
/// <param name="RootAction">- The action being considered.</param>
/// <param name="OutdatedActionDictionary">-</param>
/// <param name="ActionHistory"></param>
/// <param name="CppDependencies"></param>
/// <param name="bIgnoreOutdatedImportLibraries"></param>
/// <returns>true if outdated</returns>
public static bool IsActionOutdated(Action RootAction, Dictionary <Action, bool> OutdatedActionDictionary, ActionHistory ActionHistory, CppDependencyCache CppDependencies, bool bIgnoreOutdatedImportLibraries)
{
// Only compute the outdated-ness for actions that don't aren't cached in the outdated action dictionary.
bool bIsOutdated = false;
lock (OutdatedActionDictionary)
{
if (OutdatedActionDictionary.TryGetValue(RootAction, out bIsOutdated))
{
return(bIsOutdated);
}
}
// Determine the last time the action was run based on the write times of its produced files.
string LatestUpdatedProducedItemName = null;
DateTimeOffset LastExecutionTimeUtc = DateTimeOffset.MaxValue;
foreach (FileItem ProducedItem in RootAction.ProducedItems)
{
// Check if the command-line of the action previously used to produce the item is outdated.
string NewProducingCommandLine = RootAction.CommandPath.FullName + " " + RootAction.CommandArguments;
if (ActionHistory.UpdateProducingCommandLine(ProducedItem, NewProducingCommandLine))
{
if (ProducedItem.Exists)
{
Log.TraceLog(
"{0}: Produced item \"{1}\" was produced by outdated command-line.\n New command-line: {2}",
RootAction.StatusDescription,
Path.GetFileName(ProducedItem.AbsolutePath),
NewProducingCommandLine
);
}
bIsOutdated = true;
}
// If the produced file doesn't exist or has zero size, consider it outdated. The zero size check is to detect cases
// where aborting an earlier compile produced invalid zero-sized obj files, but that may cause actions where that's
// legitimate output to always be considered outdated.
if (ProducedItem.Exists && (RootAction.ActionType != ActionType.Compile || ProducedItem.Length > 0 || (!ProducedItem.Location.HasExtension(".obj") && !ProducedItem.Location.HasExtension(".o"))))
{
// Use the oldest produced item's time as the last execution time.
if (ProducedItem.LastWriteTimeUtc < LastExecutionTimeUtc)
{
LastExecutionTimeUtc = ProducedItem.LastWriteTimeUtc;
LatestUpdatedProducedItemName = ProducedItem.AbsolutePath;
}
}
else
{
// If any of the produced items doesn't exist, the action is outdated.
Log.TraceLog(
"{0}: Produced item \"{1}\" doesn't exist.",
RootAction.StatusDescription,
Path.GetFileName(ProducedItem.AbsolutePath)
);
bIsOutdated = true;
}
}
// Check if any of the prerequisite actions are out of date
if (!bIsOutdated)
{
foreach (Action PrerequisiteAction in RootAction.PrerequisiteActions)
{
if (IsActionOutdated(PrerequisiteAction, OutdatedActionDictionary, ActionHistory, CppDependencies, bIgnoreOutdatedImportLibraries))
{
// Only check for outdated import libraries if we were configured to do so. Often, a changed import library
// won't affect a dependency unless a public header file was also changed, in which case we would be forced
// to recompile anyway. This just allows for faster iteration when working on a subsystem in a DLL, as we
// won't have to wait for dependent targets to be relinked after each change.
if (!bIgnoreOutdatedImportLibraries || !IsImportLibraryDependency(RootAction, PrerequisiteAction))
{
Log.TraceLog("{0}: Prerequisite {1} is produced by outdated action.", RootAction.StatusDescription, PrerequisiteAction.StatusDescription);
bIsOutdated = true;
break;
}
}
}
}
// Check if any prerequisite item has a newer timestamp than the last execution time of this action
if (!bIsOutdated)
{
foreach (FileItem PrerequisiteItem in RootAction.PrerequisiteItems)
{
if (PrerequisiteItem.Exists)
{
// allow a 1 second slop for network copies
TimeSpan TimeDifference = PrerequisiteItem.LastWriteTimeUtc - LastExecutionTimeUtc;
bool bPrerequisiteItemIsNewerThanLastExecution = TimeDifference.TotalSeconds > 1;
if (bPrerequisiteItemIsNewerThanLastExecution)
{
// Need to check for import libraries here too
if (!bIgnoreOutdatedImportLibraries || !IsImportLibraryDependency(RootAction, PrerequisiteItem))
{
Log.TraceLog("{0}: Prerequisite {1} is newer than the last execution of the action: {2} vs {3}", RootAction.StatusDescription, Path.GetFileName(PrerequisiteItem.AbsolutePath), PrerequisiteItem.LastWriteTimeUtc.ToLocalTime(), LastExecutionTimeUtc.LocalDateTime);
bIsOutdated = true;
break;
}
}
}
}
}
// Check the dependency list
if (!bIsOutdated && RootAction.DependencyListFile != null)
{
List <FileItem> DependencyFiles;
if (!CppDependencies.TryGetDependencies(RootAction.DependencyListFile, out DependencyFiles))
{
Log.TraceLog("{0}: Missing dependency list file \"{1}\"", RootAction.StatusDescription, RootAction.DependencyListFile);
bIsOutdated = true;
}
else
{
foreach (FileItem DependencyFile in DependencyFiles)
{
if (!DependencyFile.Exists || DependencyFile.LastWriteTimeUtc > LastExecutionTimeUtc)
{
Log.TraceLog(
"{0}: Dependency {1} is newer than the last execution of the action: {2} vs {3}",
RootAction.StatusDescription,
Path.GetFileName(DependencyFile.AbsolutePath),
DependencyFile.LastWriteTimeUtc.ToLocalTime(),
LastExecutionTimeUtc.LocalDateTime
);
bIsOutdated = true;
break;
}
}
}
}
// Cache the outdated-ness of this action.
lock (OutdatedActionDictionary)
{
if (!OutdatedActionDictionary.ContainsKey(RootAction))
{
OutdatedActionDictionary.Add(RootAction, bIsOutdated);
}
}
return(bIsOutdated);
}
/// <summary>
/// Main entry point
/// </summary>
/// <param name="Arguments">Command-line arguments</param>
/// <returns>One of the values of ECompilationResult</returns>
public override int Execute(CommandLineArguments Arguments)
{
Arguments.ApplyTo(this);
// Initialize the log system, buffering the output until we can create the log file
StartupTraceListener StartupListener = new StartupTraceListener();
Trace.Listeners.Add(StartupListener);
// Write the command line
Log.TraceLog("Command line: {0}", Environment.CommandLine);
// Grab the environment.
UnrealBuildTool.InitialEnvironment = Environment.GetEnvironmentVariables();
if (UnrealBuildTool.InitialEnvironment.Count < 1)
{
throw new BuildException("Environment could not be read");
}
// Read the XML configuration files
XmlConfig.ApplyTo(this);
// Fixup the log path if it wasn't overridden by a config file
if (BaseLogFileName == null)
{
BaseLogFileName = FileReference.Combine(UnrealBuildTool.EngineProgramSavedDirectory, "UnrealBuildTool", "Log.txt").FullName;
}
// Create the log file, and flush the startup listener to it
if (!Arguments.HasOption("-NoLog") && !Log.HasFileWriter())
{
FileReference LogFile = new FileReference(BaseLogFileName);
foreach (string LogSuffix in Arguments.GetValues("-LogSuffix="))
{
LogFile = LogFile.ChangeExtension(null) + "_" + LogSuffix + LogFile.GetExtension();
}
TextWriterTraceListener LogTraceListener = Log.AddFileWriter("DefaultLogTraceListener", LogFile);
StartupListener.CopyTo(LogTraceListener);
}
Trace.Listeners.Remove(StartupListener);
// Create the build configuration object, and read the settings
BuildConfiguration BuildConfiguration = new BuildConfiguration();
XmlConfig.ApplyTo(BuildConfiguration);
Arguments.ApplyTo(BuildConfiguration);
// Check the root path length isn't too long
if (BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Win64 && UnrealBuildTool.RootDirectory.FullName.Length > BuildConfiguration.MaxRootPathLength)
{
Log.TraceWarning("Running from a path with a long directory name (\"{0}\" = {1} characters). Root paths shorter than {2} characters are recommended to avoid exceeding maximum path lengths on Windows.", UnrealBuildTool.RootDirectory, UnrealBuildTool.RootDirectory.FullName.Length, BuildConfiguration.MaxRootPathLength);
}
// now that we know the available platforms, we can delete other platforms' junk. if we're only building specific modules from the editor, don't touch anything else (it may be in use).
if (!bIgnoreJunk && !UnrealBuildTool.IsEngineInstalled())
{
using (Timeline.ScopeEvent("DeleteJunk()"))
{
JunkDeleter.DeleteJunk();
}
}
// Parse and build the targets
try
{
List <TargetDescriptor> TargetDescriptors;
// Parse all the target descriptors
using (Timeline.ScopeEvent("TargetDescriptor.ParseCommandLine()"))
{
TargetDescriptors = TargetDescriptor.ParseCommandLine(Arguments, BuildConfiguration.bUsePrecompiled, BuildConfiguration.bSkipRulesCompile);
}
// Hack for specific files compile; don't build the ShaderCompileWorker target that's added to the command line for generated project files
if (TargetDescriptors.Count >= 2)
{
TargetDescriptors.RemoveAll(x => (x.Name == "ShaderCompileWorker" || x.Name == "LiveCodingConsole") && x.SpecificFilesToCompile.Count > 0);
}
// Handle remote builds
for (int Idx = 0; Idx < TargetDescriptors.Count; ++Idx)
{
TargetDescriptor TargetDesc = TargetDescriptors[Idx];
if (RemoteMac.HandlesTargetPlatform(TargetDesc.Platform))
{
FileReference BaseLogFile = Log.OutputFile ?? new FileReference(BaseLogFileName);
FileReference RemoteLogFile = FileReference.Combine(BaseLogFile.Directory, BaseLogFile.GetFileNameWithoutExtension() + "_Remote.txt");
RemoteMac RemoteMac = new RemoteMac(TargetDesc.ProjectFile);
if (!RemoteMac.Build(TargetDesc, RemoteLogFile, bSkipPreBuildTargets))
{
return((int)CompilationResult.Unknown);
}
TargetDescriptors.RemoveAt(Idx--);
}
}
// Handle local builds
if (TargetDescriptors.Count > 0)
{
// Get a set of all the project directories
HashSet <DirectoryReference> ProjectDirs = new HashSet <DirectoryReference>();
foreach (TargetDescriptor TargetDesc in TargetDescriptors)
{
if (TargetDesc.ProjectFile != null)
{
DirectoryReference ProjectDirectory = TargetDesc.ProjectFile.Directory;
FileMetadataPrefetch.QueueProjectDirectory(ProjectDirectory);
ProjectDirs.Add(ProjectDirectory);
}
}
// Get all the build options
BuildOptions Options = BuildOptions.None;
if (bSkipBuild)
{
Options |= BuildOptions.SkipBuild;
}
if (bXGEExport)
{
Options |= BuildOptions.XGEExport;
}
if (bNoEngineChanges)
{
Options |= BuildOptions.NoEngineChanges;
}
// Create the working set provider per group.
using (ISourceFileWorkingSet WorkingSet = SourceFileWorkingSet.Create(UnrealBuildTool.RootDirectory, ProjectDirs))
{
Build(TargetDescriptors, BuildConfiguration, WorkingSet, Options, WriteOutdatedActionsFile, bSkipPreBuildTargets);
}
}
}
finally
{
// Save all the caches
SourceFileMetadataCache.SaveAll();
CppDependencyCache.SaveAll();
}
return(0);
}
/// <summary>
/// Build a list of targets with a given set of makefiles.
/// </summary>
/// <param name="Makefiles">Makefiles created with CreateMakefiles</param>
/// <param name="TargetDescriptors">Target descriptors</param>
/// <param name="BuildConfiguration">Current build configuration</param>
/// <param name="WorkingSet">The source file working set</param>
/// <param name="Options">Additional options for the build</param>
/// <param name="WriteOutdatedActionsFile">Files to write the list of outdated actions to (rather than building them)</param>
/// <returns>Result from the compilation</returns>
static void Build(TargetMakefile[] Makefiles, List <TargetDescriptor> TargetDescriptors, BuildConfiguration BuildConfiguration, ISourceFileWorkingSet WorkingSet, BuildOptions Options, FileReference WriteOutdatedActionsFile)
{
// Export the actions for each target
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
TargetDescriptor TargetDescriptor = TargetDescriptors[TargetIdx];
foreach (FileReference WriteActionFile in TargetDescriptor.WriteActionFiles)
{
Log.TraceInformation("Writing actions to {0}", WriteActionFile);
ActionGraph.ExportJson(Makefiles[TargetIdx].Actions, WriteActionFile);
}
}
// Execute the build
if ((Options & BuildOptions.SkipBuild) == 0)
{
// Make sure that none of the actions conflict with any other (producing output files differently, etc...)
ActionGraph.CheckForConflicts(Makefiles.SelectMany(x => x.Actions));
// Check we don't exceed the nominal max path length
using (Timeline.ScopeEvent("ActionGraph.CheckPathLengths"))
{
ActionGraph.CheckPathLengths(BuildConfiguration, Makefiles.SelectMany(x => x.Actions));
}
// Clean up any previous hot reload runs, and reapply the current state if it's already active
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
HotReload.Setup(TargetDescriptors[TargetIdx], Makefiles[TargetIdx], BuildConfiguration);
}
// Merge the action graphs together
List <Action> MergedActions;
if (TargetDescriptors.Count == 1)
{
MergedActions = new List <Action>(Makefiles[0].Actions);
}
else
{
MergedActions = MergeActionGraphs(TargetDescriptors, Makefiles);
}
// Gather all the prerequisite actions that are part of the targets
HashSet <FileItem> MergedOutputItems = new HashSet <FileItem>();
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
GatherOutputItems(TargetDescriptors[TargetIdx], Makefiles[TargetIdx], MergedOutputItems);
}
// Link all the actions together
ActionGraph.Link(MergedActions);
// Get all the actions that are prerequisites for these targets. This forms the list of actions that we want executed.
List <Action> PrerequisiteActions = ActionGraph.GatherPrerequisiteActions(MergedActions, MergedOutputItems);
// Create the action history
ActionHistory History = new ActionHistory();
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
using (Timeline.ScopeEvent("Reading action history"))
{
TargetDescriptor TargetDescriptor = TargetDescriptors[TargetIdx];
if (TargetDescriptor.ProjectFile != null)
{
History.Mount(TargetDescriptor.ProjectFile.Directory);
}
}
}
// Figure out which actions need to be built
Dictionary <Action, bool> ActionToOutdatedFlag = new Dictionary <Action, bool>();
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
TargetDescriptor TargetDescriptor = TargetDescriptors[TargetIdx];
// Create the dependencies cache
CppDependencyCache CppDependencies;
using (Timeline.ScopeEvent("Reading dependency cache"))
{
CppDependencies = CppDependencyCache.CreateHierarchy(TargetDescriptor.ProjectFile, TargetDescriptor.Name, TargetDescriptor.Platform, TargetDescriptor.Configuration, Makefiles[TargetIdx].TargetType, TargetDescriptor.Architecture);
}
// Plan the actions to execute for the build. For single file compiles, always rebuild the source file regardless of whether it's out of date.
if (TargetDescriptor.SpecificFilesToCompile.Count == 0)
{
ActionGraph.GatherAllOutdatedActions(PrerequisiteActions, History, ActionToOutdatedFlag, CppDependencies, BuildConfiguration.bIgnoreOutdatedImportLibraries);
}
else
{
foreach (FileReference SpecificFile in TargetDescriptor.SpecificFilesToCompile)
{
foreach (Action PrerequisiteAction in PrerequisiteActions.Where(x => x.PrerequisiteItems.Any(y => y.Location == SpecificFile)))
{
ActionToOutdatedFlag[PrerequisiteAction] = true;
}
}
}
}
// Link the action graph again to sort it
List <Action> MergedActionsToExecute = ActionToOutdatedFlag.Where(x => x.Value).Select(x => x.Key).ToList();
ActionGraph.Link(MergedActionsToExecute);
// Allow hot reload to override the actions
int HotReloadTargetIdx = -1;
for (int Idx = 0; Idx < TargetDescriptors.Count; Idx++)
{
if (TargetDescriptors[Idx].HotReloadMode != HotReloadMode.Disabled)
{
if (HotReloadTargetIdx != -1)
{
throw new BuildException("Unable to perform hot reload with multiple targets.");
}
else
{
MergedActionsToExecute = HotReload.PatchActionsForTarget(BuildConfiguration, TargetDescriptors[Idx], Makefiles[Idx], PrerequisiteActions, MergedActionsToExecute);
}
HotReloadTargetIdx = Idx;
}
}
// Make sure we're not modifying any engine files
if ((Options & BuildOptions.NoEngineChanges) != 0)
{
List <FileItem> EngineChanges = MergedActionsToExecute.SelectMany(x => x.ProducedItems).Where(x => x.Location.IsUnderDirectory(UnrealBuildTool.EngineDirectory)).Distinct().OrderBy(x => x.FullName).ToList();
if (EngineChanges.Count > 0)
{
StringBuilder Result = new StringBuilder("Building would modify the following engine files:\n");
foreach (FileItem EngineChange in EngineChanges)
{
Result.AppendFormat("\n{0}", EngineChange.FullName);
}
Result.Append("\n\nPlease rebuild from an IDE instead.");
Log.TraceError("{0}", Result.ToString());
throw new CompilationResultException(CompilationResult.FailedDueToEngineChange);
}
}
// Make sure the appropriate executor is selected
foreach (TargetDescriptor TargetDescriptor in TargetDescriptors)
{
UEBuildPlatform BuildPlatform = UEBuildPlatform.GetBuildPlatform(TargetDescriptor.Platform);
BuildConfiguration.bAllowXGE &= BuildPlatform.CanUseXGE();
BuildConfiguration.bAllowDistcc &= BuildPlatform.CanUseDistcc();
BuildConfiguration.bAllowSNDBS &= BuildPlatform.CanUseSNDBS();
}
// Delete produced items that are outdated.
ActionGraph.DeleteOutdatedProducedItems(MergedActionsToExecute);
// Save all the action histories now that files have been removed. We have to do this after deleting produced items to ensure that any
// items created during the build don't have the wrong command line.
History.Save();
// Create directories for the outdated produced items.
ActionGraph.CreateDirectoriesForProducedItems(MergedActionsToExecute);
// Execute the actions
if ((Options & BuildOptions.XGEExport) != 0)
{
OutputToolchainInfo(TargetDescriptors, Makefiles);
// Just export to an XML file
using (Timeline.ScopeEvent("XGE.ExportActions()"))
{
XGE.ExportActions(MergedActionsToExecute);
}
}
else if (WriteOutdatedActionsFile != null)
{
OutputToolchainInfo(TargetDescriptors, Makefiles);
// Write actions to an output file
using (Timeline.ScopeEvent("ActionGraph.WriteActions"))
{
ActionGraph.ExportJson(MergedActionsToExecute, WriteOutdatedActionsFile);
}
}
else
{
// Execute the actions
if (MergedActionsToExecute.Count == 0)
{
if (TargetDescriptors.Any(x => !x.bQuiet))
{
Log.TraceInformation((TargetDescriptors.Count == 1)? "Target is up to date" : "Targets are up to date");
}
}
else
{
if (TargetDescriptors.Any(x => !x.bQuiet))
{
Log.TraceInformation("Building {0}...", StringUtils.FormatList(TargetDescriptors.Select(x => x.Name).Distinct()));
}
OutputToolchainInfo(TargetDescriptors, Makefiles);
using (Timeline.ScopeEvent("ActionGraph.ExecuteActions()"))
{
ActionGraph.ExecuteActions(BuildConfiguration, MergedActionsToExecute);
}
}
// Run the deployment steps
foreach (TargetMakefile Makefile in Makefiles)
{
if (Makefile.bDeployAfterCompile)
{
TargetReceipt Receipt = TargetReceipt.Read(Makefile.ReceiptFile);
Log.TraceInformation("Deploying {0} {1} {2}...", Receipt.TargetName, Receipt.Platform, Receipt.Configuration);
UEBuildPlatform.GetBuildPlatform(Receipt.Platform).Deploy(Receipt);
}
}
}
}
}
/// <summary>
/// Determine what needs to be built for a target
/// </summary>
/// <param name="BuildConfiguration">The build configuration</param>
/// <param name="TargetDescriptor">Target being built</param>
/// <param name="Makefile">Makefile generated for this target</param>
/// <returns>Set of actions to execute</returns>
static HashSet <Action> GetActionsForTarget(BuildConfiguration BuildConfiguration, TargetDescriptor TargetDescriptor, TargetMakefile Makefile)
{
// Create the action graph
ActionGraph.Link(Makefile.Actions);
// Get the hot-reload mode
HotReloadMode HotReloadMode = TargetDescriptor.HotReloadMode;
if (HotReloadMode == HotReloadMode.Default)
{
if (TargetDescriptor.HotReloadModuleNameToSuffix.Count > 0 && TargetDescriptor.ForeignPlugin == null)
{
HotReloadMode = HotReloadMode.FromEditor;
}
else if (BuildConfiguration.bAllowHotReloadFromIDE && HotReload.ShouldDoHotReloadFromIDE(BuildConfiguration, TargetDescriptor))
{
HotReloadMode = HotReloadMode.FromIDE;
}
else
{
HotReloadMode = HotReloadMode.Disabled;
}
}
// Get the root prerequisite actions
List <Action> PrerequisiteActions = GatherPrerequisiteActions(TargetDescriptor, Makefile);
// Get the path to the hot reload state file for this target
FileReference HotReloadStateFile = global::UnrealBuildTool.HotReloadState.GetLocation(TargetDescriptor.ProjectFile, TargetDescriptor.Name, TargetDescriptor.Platform, TargetDescriptor.Configuration, TargetDescriptor.Architecture);
// Apply the previous hot reload state
HotReloadState HotReloadState = null;
if (HotReloadMode == HotReloadMode.Disabled)
{
// Make sure we're not doing a partial build from the editor (eg. compiling a new plugin)
if (TargetDescriptor.ForeignPlugin == null && TargetDescriptor.SingleFileToCompile == null)
{
// Delete the previous state file
HotReload.DeleteTemporaryFiles(HotReloadStateFile);
}
}
else
{
// Read the previous state file and apply it to the action graph
if (FileReference.Exists(HotReloadStateFile))
{
HotReloadState = HotReloadState.Load(HotReloadStateFile);
}
else
{
HotReloadState = new HotReloadState();
}
// Apply the old state to the makefile
HotReload.ApplyState(HotReloadState, Makefile);
// If we want a specific suffix on any modules, apply that now. We'll track the outputs later, but the suffix has to be forced (and is always out of date if it doesn't exist).
HotReload.PatchActionGraphWithNames(PrerequisiteActions, TargetDescriptor.HotReloadModuleNameToSuffix, Makefile);
// Re-link the action graph
ActionGraph.Link(PrerequisiteActions);
}
// Create the dependencies cache
CppDependencyCache CppDependencies;
using (Timeline.ScopeEvent("Reading dependency cache"))
{
CppDependencies = CppDependencyCache.CreateHierarchy(TargetDescriptor.ProjectFile, TargetDescriptor.Name, TargetDescriptor.Platform, TargetDescriptor.Configuration, Makefile.TargetType);
}
// Create the action history
ActionHistory History;
using (Timeline.ScopeEvent("Reading action history"))
{
History = ActionHistory.CreateHierarchy(TargetDescriptor.ProjectFile, TargetDescriptor.Name, TargetDescriptor.Platform, Makefile.TargetType);
}
// Plan the actions to execute for the build. For single file compiles, always rebuild the source file regardless of whether it's out of date.
HashSet <Action> TargetActionsToExecute;
if (TargetDescriptor.SingleFileToCompile == null)
{
TargetActionsToExecute = ActionGraph.GetActionsToExecute(Makefile.Actions, PrerequisiteActions, CppDependencies, History, BuildConfiguration.bIgnoreOutdatedImportLibraries);
}
else
{
TargetActionsToExecute = new HashSet <Action>(PrerequisiteActions);
}
// Additional processing for hot reload
if (HotReloadMode == HotReloadMode.LiveCoding)
{
// Filter the prerequisite actions down to just the compile actions, then recompute all the actions to execute
PrerequisiteActions = new List <Action>(TargetActionsToExecute.Where(x => x.ActionType == ActionType.Compile));
TargetActionsToExecute = ActionGraph.GetActionsToExecute(Makefile.Actions, PrerequisiteActions, CppDependencies, History, BuildConfiguration.bIgnoreOutdatedImportLibraries);
}
else if (HotReloadMode == HotReloadMode.FromEditor || HotReloadMode == HotReloadMode.FromIDE)
{
// Patch action history for hot reload when running in assembler mode. In assembler mode, the suffix on the output file will be
// the same for every invocation on that makefile, but we need a new suffix each time.
// For all the hot-reloadable modules that may need a unique suffix appended, build a mapping from output item to all the output items in that module. We can't
// apply a suffix to one without applying a suffix to all of them.
Dictionary <FileItem, FileItem[]> HotReloadItemToDependentItems = new Dictionary <FileItem, FileItem[]>();
foreach (string HotReloadModuleName in Makefile.HotReloadModuleNames)
{
int ModuleSuffix;
if (!TargetDescriptor.HotReloadModuleNameToSuffix.TryGetValue(HotReloadModuleName, out ModuleSuffix) || ModuleSuffix == -1)
{
FileItem[] ModuleOutputItems;
if (Makefile.ModuleNameToOutputItems.TryGetValue(HotReloadModuleName, out ModuleOutputItems))
{
foreach (FileItem ModuleOutputItem in ModuleOutputItems)
{
HotReloadItemToDependentItems[ModuleOutputItem] = ModuleOutputItems;
}
}
}
}
// Expand the list of actions to execute to include everything that references any files with a new suffix. Unlike a regular build, we can't ignore
// dependencies on import libraries under the assumption that a header would change if the API changes, because the dependency will be on a different DLL.
HashSet <FileItem> FilesRequiringSuffix = new HashSet <FileItem>(TargetActionsToExecute.SelectMany(x => x.ProducedItems).Where(x => HotReloadItemToDependentItems.ContainsKey(x)));
for (int LastNumFilesWithNewSuffix = 0; FilesRequiringSuffix.Count > LastNumFilesWithNewSuffix;)
{
LastNumFilesWithNewSuffix = FilesRequiringSuffix.Count;
foreach (Action PrerequisiteAction in PrerequisiteActions)
{
if (!TargetActionsToExecute.Contains(PrerequisiteAction))
{
foreach (FileItem ProducedItem in PrerequisiteAction.ProducedItems)
{
FileItem[] DependentItems;
if (HotReloadItemToDependentItems.TryGetValue(ProducedItem, out DependentItems))
{
TargetActionsToExecute.Add(PrerequisiteAction);
FilesRequiringSuffix.UnionWith(DependentItems);
}
}
}
}
}
// Build a list of file mappings
Dictionary <FileReference, FileReference> OldLocationToNewLocation = new Dictionary <FileReference, FileReference>();
foreach (FileItem FileRequiringSuffix in FilesRequiringSuffix)
{
FileReference OldLocation = FileRequiringSuffix.Location;
FileReference NewLocation = HotReload.ReplaceSuffix(OldLocation, HotReloadState.NextSuffix);
OldLocationToNewLocation[OldLocation] = NewLocation;
}
// Update the action graph with these new paths
HotReload.PatchActionGraph(PrerequisiteActions, OldLocationToNewLocation);
// Get a new list of actions to execute now that the graph has been modified
TargetActionsToExecute = ActionGraph.GetActionsToExecute(Makefile.Actions, PrerequisiteActions, CppDependencies, History, BuildConfiguration.bIgnoreOutdatedImportLibraries);
// Build a mapping of all file items to their original
Dictionary <FileReference, FileReference> HotReloadFileToOriginalFile = new Dictionary <FileReference, FileReference>();
foreach (KeyValuePair <FileReference, FileReference> Pair in HotReloadState.OriginalFileToHotReloadFile)
{
HotReloadFileToOriginalFile[Pair.Value] = Pair.Key;
}
foreach (KeyValuePair <FileReference, FileReference> Pair in OldLocationToNewLocation)
{
FileReference OriginalLocation;
if (!HotReloadFileToOriginalFile.TryGetValue(Pair.Key, out OriginalLocation))
{
OriginalLocation = Pair.Key;
}
HotReloadFileToOriginalFile[Pair.Value] = OriginalLocation;
}
// Now filter out all the hot reload files and update the state
foreach (Action Action in TargetActionsToExecute)
{
foreach (FileItem ProducedItem in Action.ProducedItems)
{
FileReference OriginalLocation;
if (HotReloadFileToOriginalFile.TryGetValue(ProducedItem.Location, out OriginalLocation))
{
HotReloadState.OriginalFileToHotReloadFile[OriginalLocation] = ProducedItem.Location;
HotReloadState.TemporaryFiles.Add(ProducedItem.Location);
}
}
}
// Increment the suffix for the next iteration
if (TargetActionsToExecute.Count > 0)
{
HotReloadState.NextSuffix++;
}
// Save the new state
HotReloadState.Save(HotReloadStateFile);
// Prevent this target from deploying
Makefile.bDeployAfterCompile = false;
}
return(TargetActionsToExecute);
}
/// <summary>
/// Main entry point
/// </summary>
/// <param name="Arguments">Command-line arguments</param>
/// <returns>One of the values of ECompilationResult</returns>
public override int Execute(CommandLineArguments Arguments)
{
Arguments.ApplyTo(this);
// Initialize the log system, buffering the output until we can create the log file
StartupTraceListener StartupListener = new StartupTraceListener();
Trace.Listeners.Add(StartupListener);
// Write the command line
Log.TraceLog("Command line: {0}", Environment.CommandLine);
// Grab the environment.
UnrealBuildTool.InitialEnvironment = Environment.GetEnvironmentVariables();
if (UnrealBuildTool.InitialEnvironment.Count < 1)
{
throw new BuildException("Environment could not be read");
}
// Read the XML configuration files
XmlConfig.ApplyTo(this);
// Create the log file, and flush the startup listener to it
if (!Arguments.HasOption("-NoLog") && !Log.HasFileWriter())
{
FileReference LogFile = new FileReference(BaseLogFileName);
foreach (string LogSuffix in Arguments.GetValues("-LogSuffix="))
{
LogFile = LogFile.ChangeExtension(null) + "_" + LogSuffix + LogFile.GetExtension();
}
TextWriterTraceListener LogTraceListener = Log.AddFileWriter("DefaultLogTraceListener", LogFile);
StartupListener.CopyTo(LogTraceListener);
}
Trace.Listeners.Remove(StartupListener);
// Create the build configuration object, and read the settings
BuildConfiguration BuildConfiguration = new BuildConfiguration();
XmlConfig.ApplyTo(BuildConfiguration);
Arguments.ApplyTo(BuildConfiguration);
// now that we know the available platforms, we can delete other platforms' junk. if we're only building specific modules from the editor, don't touch anything else (it may be in use).
if (!bIgnoreJunk && !UnrealBuildTool.IsEngineInstalled())
{
using (Timeline.ScopeEvent("DeleteJunk()"))
{
JunkDeleter.DeleteJunk();
}
}
// Parse and build the targets
try
{
// Parse all the target descriptors
List <TargetDescriptor> TargetDescriptors;
using (Timeline.ScopeEvent("TargetDescriptor.ParseCommandLine()"))
{
TargetDescriptors = TargetDescriptor.ParseCommandLine(Arguments, BuildConfiguration.bUsePrecompiled, BuildConfiguration.bSkipRulesCompile);
}
// Hack for single file compile; don't build the ShaderCompileWorker target that's added to the command line for generated project files
if (TargetDescriptors.Count >= 2)
{
TargetDescriptors.RemoveAll(x => x.Name == "ShaderCompileWorker" && x.SingleFileToCompile != null);
}
// Handle remote builds
for (int Idx = 0; Idx < TargetDescriptors.Count; Idx++)
{
TargetDescriptor TargetDesc = TargetDescriptors[Idx];
if (RemoteMac.HandlesTargetPlatform(TargetDesc.Platform))
{
FileReference BaseLogFile = Log.OutputFile ?? new FileReference(BaseLogFileName);
FileReference RemoteLogFile = FileReference.Combine(BaseLogFile.Directory, BaseLogFile.GetFileNameWithoutExtension() + "_Remote.txt");
RemoteMac RemoteMac = new RemoteMac(TargetDesc.ProjectFile);
if (!RemoteMac.Build(TargetDesc, RemoteLogFile))
{
return((int)CompilationResult.Unknown);
}
TargetDescriptors.RemoveAt(Idx--);
}
}
// Handle local builds
if (TargetDescriptors.Count > 0)
{
// Get a set of all the project directories
HashSet <DirectoryReference> ProjectDirs = new HashSet <DirectoryReference>();
foreach (TargetDescriptor TargetDesc in TargetDescriptors)
{
if (TargetDesc.ProjectFile != null)
{
DirectoryReference ProjectDirectory = TargetDesc.ProjectFile.Directory;
FileMetadataPrefetch.QueueProjectDirectory(ProjectDirectory);
ProjectDirs.Add(ProjectDirectory);
}
}
// Get all the build options
BuildOptions Options = BuildOptions.None;
if (bSkipBuild)
{
Options |= BuildOptions.SkipBuild;
}
if (bXGEExport)
{
Options |= BuildOptions.XGEExport;
}
// Create the working set provider
using (ISourceFileWorkingSet WorkingSet = SourceFileWorkingSet.Create(UnrealBuildTool.RootDirectory, ProjectDirs))
{
Build(TargetDescriptors, BuildConfiguration, WorkingSet, Options);
}
}
}
finally
{
// Save all the caches
SourceFileMetadataCache.SaveAll();
CppDependencyCache.SaveAll();
}
return(0);
}