/// <summary>
/// Main entry point
/// </summary>
/// <param name="Arguments">Command-line arguments</param>
/// <returns>One of the values of ECompilationResult</returns>
public override int Execute(CommandLineArguments Arguments)
{
Arguments.ApplyTo(this);
// Read the XML configuration files
XmlConfig.ApplyTo(this);
// Create the build configuration object, and read the settings
BuildConfiguration BuildConfiguration = new BuildConfiguration();
XmlConfig.ApplyTo(BuildConfiguration);
Arguments.ApplyTo(BuildConfiguration);
// Read the actions file
List <Action> Actions;
using (Timeline.ScopeEvent("ActionGraph.ReadActions()"))
{
Actions = ActionGraph.ImportJson(ActionsFile);
}
// Link the action graph
using (Timeline.ScopeEvent("ActionGraph.Link()"))
{
ActionGraph.Link(Actions);
}
// Execute the actions
using (Timeline.ScopeEvent("ActionGraph.ExecuteActions()"))
{
ActionGraph.ExecuteActions(BuildConfiguration, Actions);
}
return(0);
}
/// <summary>
/// Builds a list of actions that need to be executed to produce the specified output items.
/// </summary>
public static List <Action> GetActionsToExecute(List <Action> Actions, CppDependencyCache CppDependencies, ActionHistory History, bool bIgnoreOutdatedImportLibraries)
{
using (Timeline.ScopeEvent("ActionGraph.GetActionsToExecute()"))
{
// For all targets, build a set of all actions that are outdated.
Dictionary <Action, bool> OutdatedActionDictionary = new Dictionary <Action, bool>();
GatherAllOutdatedActions(Actions, History, OutdatedActionDictionary, CppDependencies, bIgnoreOutdatedImportLibraries);
// Build a list of actions that are both needed for this target and outdated.
return(Actions.Where(Action => Action.CommandPath != null && OutdatedActionDictionary[Action]).ToList());
}
}
/// <summary>
/// Creates a rules assembly
/// </summary>
/// <param name="RootDirectory">The root directory to create rules for</param>
/// <param name="AssemblyPrefix">A prefix for the assembly file name</param>
/// <param name="Plugins">List of plugins to include in this assembly</param>
/// <param name="bReadOnly">Whether the assembly should be marked as installed</param>
/// <param name="bSkipCompile">Whether to skip compilation for this assembly</param>
/// <param name="Parent">The parent rules assembly</param>
/// <returns>New rules assembly</returns>
private static RulesAssembly CreateEngineOrEnterpriseRulesAssembly(DirectoryReference RootDirectory, string AssemblyPrefix, IReadOnlyList <PluginInfo> Plugins, bool bReadOnly, bool bSkipCompile, RulesAssembly Parent)
{
DirectoryReference SourceDirectory = DirectoryReference.Combine(RootDirectory, "Source");
DirectoryReference ProgramsDirectory = DirectoryReference.Combine(SourceDirectory, "Programs");
// Find the shared modules, excluding the programs directory. These are used to create an assembly with the bContainsEngineModules flag set to true.
List <FileReference> EngineModuleFiles = new List <FileReference>();
using (Timeline.ScopeEvent("Finding engine modules"))
{
foreach (DirectoryReference SubDirectory in DirectoryLookupCache.EnumerateDirectories(SourceDirectory))
{
if (SubDirectory != ProgramsDirectory)
{
EngineModuleFiles.AddRange(FindAllRulesFiles(SubDirectory, RulesFileType.Module));
}
}
}
// Add all the plugin modules too
Dictionary <FileReference, PluginInfo> ModuleFileToPluginInfo = new Dictionary <FileReference, PluginInfo>();
using (Timeline.ScopeEvent("Finding plugin modules"))
{
FindModuleRulesForPlugins(Plugins, EngineModuleFiles, ModuleFileToPluginInfo);
}
// Create the assembly
FileReference EngineAssemblyFileName = FileReference.Combine(RootDirectory, "Intermediate", "Build", "BuildRules", AssemblyPrefix + "Rules" + FrameworkAssemblyExtension);
RulesAssembly EngineAssembly = new RulesAssembly(RootDirectory, Plugins, EngineModuleFiles, new List <FileReference>(), ModuleFileToPluginInfo, EngineAssemblyFileName, bContainsEngineModules: true, bUseBackwardsCompatibleDefaults: false, bReadOnly: bReadOnly, bSkipCompile: bSkipCompile, Parent: Parent);
// Find all the rules files
List <FileReference> ProgramModuleFiles;
using (Timeline.ScopeEvent("Finding program modules"))
{
ProgramModuleFiles = new List <FileReference>(FindAllRulesFiles(ProgramsDirectory, RulesFileType.Module));
}
List <FileReference> ProgramTargetFiles;
using (Timeline.ScopeEvent("Finding program targets"))
{
ProgramTargetFiles = new List <FileReference>(FindAllRulesFiles(SourceDirectory, RulesFileType.Target));
}
// Create a path to the assembly that we'll either load or compile
FileReference ProgramAssemblyFileName = FileReference.Combine(RootDirectory, "Intermediate", "Build", "BuildRules", AssemblyPrefix + "ProgramRules" + FrameworkAssemblyExtension);
RulesAssembly ProgramAssembly = new RulesAssembly(RootDirectory, new List <PluginInfo>().AsReadOnly(), ProgramModuleFiles, ProgramTargetFiles, new Dictionary <FileReference, PluginInfo>(), ProgramAssemblyFileName, bContainsEngineModules: false, bUseBackwardsCompatibleDefaults: false, bReadOnly: bReadOnly, bSkipCompile: bSkipCompile, Parent: EngineAssembly);
// Return the combined assembly
return(ProgramAssembly);
}
/// <summary>
/// Constructs a dependency cache. This method is private; call CppDependencyCache.Create() to create a cache hierarchy for a given project.
/// </summary>
/// <param name="Location">File to store the cache</param>
/// <param name="BaseDir">Base directory for files that this cache should store data for</param>
/// <param name="Parent">The parent cache to use</param>
private SourceFileMetadataCache(FileReference Location, DirectoryReference BaseDir, SourceFileMetadataCache Parent)
{
this.Location = Location;
this.BaseDirectory = BaseDir;
this.Parent = Parent;
if (FileReference.Exists(Location))
{
using (Timeline.ScopeEvent("Reading source file metadata cache"))
{
Read();
}
}
}
/// <summary>
/// Finds all the UEBuildPlatformFactory types in this assembly and uses them to register all the available platforms
/// </summary>
/// <param name="bIncludeNonInstalledPlatforms">Whether to register platforms that are not installed</param>
/// <param name="bHostPlatformOnly">Only register the host platform</param>
public static void RegisterPlatforms(bool bIncludeNonInstalledPlatforms, bool bHostPlatformOnly)
{
// Initialize the installed platform info
using (Timeline.ScopeEvent("Initializing InstalledPlatformInfo"))
{
InstalledPlatformInfo.Initialize();
}
// Find and register all tool chains and build platforms that are present
Type[] AllTypes;
using (Timeline.ScopeEvent("Querying types"))
{
AllTypes = Assembly.GetExecutingAssembly().GetTypes();
}
// register all build platforms first, since they implement SDK-switching logic that can set environment variables
foreach (Type CheckType in AllTypes)
{
if (CheckType.IsClass && !CheckType.IsAbstract)
{
if (CheckType.IsSubclassOf(typeof(UEBuildPlatformFactory)))
{
Log.TraceVerbose(" Registering build platform: {0}", CheckType.ToString());
using (Timeline.ScopeEvent(CheckType.Name))
{
UEBuildPlatformFactory TempInst = (UEBuildPlatformFactory)Activator.CreateInstance(CheckType);
if (bHostPlatformOnly && TempInst.TargetPlatform != BuildHostPlatform.Current.Platform)
{
continue;
}
// We need all platforms to be registered when we run -validateplatform command to check SDK status of each
if (bIncludeNonInstalledPlatforms || InstalledPlatformInfo.IsValidPlatform(TempInst.TargetPlatform))
{
TempInst.RegisterBuildPlatforms();
}
}
}
}
}
}
/// <summary>
/// Builds a dictionary containing the actions from AllActions that are outdated by calling
/// IsActionOutdated.
/// </summary>
static void GatherAllOutdatedActions(List <Action> Actions, ActionHistory ActionHistory, Dictionary <Action, bool> OutdatedActions, CppDependencyCache CppDependencies, bool bIgnoreOutdatedImportLibraries)
{
using (Timeline.ScopeEvent("Prefetching include dependencies"))
{
List <FileItem> Dependencies = new List <FileItem>();
foreach (Action Action in Actions)
{
if (Action.DependencyListFile != null)
{
Dependencies.Add(Action.DependencyListFile);
}
}
Parallel.ForEach(Dependencies, File => { List <FileItem> Temp; CppDependencies.TryGetDependencies(File, out Temp); });
}
using (Timeline.ScopeEvent("Cache outdated actions"))
{
Parallel.ForEach(Actions, Action => IsActionOutdated(Action, OutdatedActions, ActionHistory, CppDependencies, bIgnoreOutdatedImportLibraries));
}
}
/// <summary>
/// Builds a list of actions that need to be executed to produce the specified output items.
/// </summary>
public static HashSet <Action> GetActionsToExecute(List <Action> Actions, List <Action> PrerequisiteActions, CppDependencyCache CppDependencies, ActionHistory History, bool bIgnoreOutdatedImportLibraries)
{
ITimelineEvent GetActionsToExecuteTimer = Timeline.ScopeEvent("ActionGraph.GetActionsToExecute()");
// Build a set of all actions needed for this target.
Dictionary <Action, bool> IsActionOutdatedMap = new Dictionary <Action, bool>();
foreach (Action Action in PrerequisiteActions)
{
IsActionOutdatedMap.Add(Action, true);
}
// For all targets, build a set of all actions that are outdated.
Dictionary <Action, bool> OutdatedActionDictionary = new Dictionary <Action, bool>();
GatherAllOutdatedActions(Actions, History, OutdatedActionDictionary, CppDependencies, bIgnoreOutdatedImportLibraries);
// Build a list of actions that are both needed for this target and outdated.
HashSet <Action> ActionsToExecute = new HashSet <Action>(Actions.Where(Action => Action.CommandPath != null && IsActionOutdatedMap.ContainsKey(Action) && OutdatedActionDictionary[Action]));
GetActionsToExecuteTimer.Finish();
return(ActionsToExecute);
}
/// <summary>
/// Main entry point
/// </summary>
/// <param name="Arguments">Command-line arguments</param>
/// <returns>One of the values of ECompilationResult</returns>
public override int Execute(CommandLineArguments Arguments)
{
Arguments.ApplyTo(this);
// Initialize the log system, buffering the output until we can create the log file
StartupTraceListener StartupListener = new StartupTraceListener();
Trace.Listeners.Add(StartupListener);
// Write the command line
Log.TraceLog("Command line: {0}", Environment.CommandLine);
// Grab the environment.
UnrealBuildTool.InitialEnvironment = Environment.GetEnvironmentVariables();
if (UnrealBuildTool.InitialEnvironment.Count < 1)
{
throw new BuildException("Environment could not be read");
}
// Read the XML configuration files
XmlConfig.ApplyTo(this);
// Fixup the log path if it wasn't overridden by a config file
if (BaseLogFileName == null)
{
BaseLogFileName = FileReference.Combine(UnrealBuildTool.EngineProgramSavedDirectory, "UnrealBuildTool", "Log.txt").FullName;
}
// Create the log file, and flush the startup listener to it
if (!Arguments.HasOption("-NoLog") && !Log.HasFileWriter())
{
FileReference LogFile = new FileReference(BaseLogFileName);
foreach (string LogSuffix in Arguments.GetValues("-LogSuffix="))
{
LogFile = LogFile.ChangeExtension(null) + "_" + LogSuffix + LogFile.GetExtension();
}
TextWriterTraceListener LogTraceListener = Log.AddFileWriter("DefaultLogTraceListener", LogFile);
StartupListener.CopyTo(LogTraceListener);
}
Trace.Listeners.Remove(StartupListener);
// Create the build configuration object, and read the settings
BuildConfiguration BuildConfiguration = new BuildConfiguration();
XmlConfig.ApplyTo(BuildConfiguration);
Arguments.ApplyTo(BuildConfiguration);
// Check the root path length isn't too long
if (BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Win64 && UnrealBuildTool.RootDirectory.FullName.Length > BuildConfiguration.MaxRootPathLength)
{
Log.TraceWarning("Running from a path with a long directory name (\"{0}\" = {1} characters). Root paths shorter than {2} characters are recommended to avoid exceeding maximum path lengths on Windows.", UnrealBuildTool.RootDirectory, UnrealBuildTool.RootDirectory.FullName.Length, BuildConfiguration.MaxRootPathLength);
}
// now that we know the available platforms, we can delete other platforms' junk. if we're only building specific modules from the editor, don't touch anything else (it may be in use).
if (!bIgnoreJunk && !UnrealBuildTool.IsEngineInstalled())
{
using (Timeline.ScopeEvent("DeleteJunk()"))
{
JunkDeleter.DeleteJunk();
}
}
// Parse and build the targets
try
{
List <TargetDescriptor> TargetDescriptors;
// Parse all the target descriptors
using (Timeline.ScopeEvent("TargetDescriptor.ParseCommandLine()"))
{
TargetDescriptors = TargetDescriptor.ParseCommandLine(Arguments, BuildConfiguration.bUsePrecompiled, BuildConfiguration.bSkipRulesCompile);
}
// Hack for specific files compile; don't build the ShaderCompileWorker target that's added to the command line for generated project files
if (TargetDescriptors.Count >= 2)
{
TargetDescriptors.RemoveAll(x => (x.Name == "ShaderCompileWorker" || x.Name == "LiveCodingConsole") && x.SpecificFilesToCompile.Count > 0);
}
// Handle remote builds
for (int Idx = 0; Idx < TargetDescriptors.Count; ++Idx)
{
TargetDescriptor TargetDesc = TargetDescriptors[Idx];
if (RemoteMac.HandlesTargetPlatform(TargetDesc.Platform))
{
FileReference BaseLogFile = Log.OutputFile ?? new FileReference(BaseLogFileName);
FileReference RemoteLogFile = FileReference.Combine(BaseLogFile.Directory, BaseLogFile.GetFileNameWithoutExtension() + "_Remote.txt");
RemoteMac RemoteMac = new RemoteMac(TargetDesc.ProjectFile);
if (!RemoteMac.Build(TargetDesc, RemoteLogFile, bSkipPreBuildTargets))
{
return((int)CompilationResult.Unknown);
}
TargetDescriptors.RemoveAt(Idx--);
}
}
// Handle local builds
if (TargetDescriptors.Count > 0)
{
// Get a set of all the project directories
HashSet <DirectoryReference> ProjectDirs = new HashSet <DirectoryReference>();
foreach (TargetDescriptor TargetDesc in TargetDescriptors)
{
if (TargetDesc.ProjectFile != null)
{
DirectoryReference ProjectDirectory = TargetDesc.ProjectFile.Directory;
FileMetadataPrefetch.QueueProjectDirectory(ProjectDirectory);
ProjectDirs.Add(ProjectDirectory);
}
}
// Get all the build options
BuildOptions Options = BuildOptions.None;
if (bSkipBuild)
{
Options |= BuildOptions.SkipBuild;
}
if (bXGEExport)
{
Options |= BuildOptions.XGEExport;
}
if (bNoEngineChanges)
{
Options |= BuildOptions.NoEngineChanges;
}
// Create the working set provider per group.
using (ISourceFileWorkingSet WorkingSet = SourceFileWorkingSet.Create(UnrealBuildTool.RootDirectory, ProjectDirs))
{
Build(TargetDescriptors, BuildConfiguration, WorkingSet, Options, WriteOutdatedActionsFile, bSkipPreBuildTargets);
}
}
}
finally
{
// Save all the caches
SourceFileMetadataCache.SaveAll();
CppDependencyCache.SaveAll();
}
return(0);
}
/// <summary>
/// Creates the makefile for a target. If an existing, valid makefile already exists on disk, loads that instead.
/// </summary>
/// <param name="BuildConfiguration">The build configuration</param>
/// <param name="TargetDescriptor">Target being built</param>
/// <param name="WorkingSet">Set of source files which are part of the working set</param>
/// <returns>Makefile for the given target</returns>
static TargetMakefile CreateMakefile(BuildConfiguration BuildConfiguration, TargetDescriptor TargetDescriptor, ISourceFileWorkingSet WorkingSet)
{
// Get the path to the makefile for this target
FileReference MakefileLocation = null;
if (BuildConfiguration.bUseUBTMakefiles && TargetDescriptor.SpecificFilesToCompile.Count == 0)
{
MakefileLocation = TargetMakefile.GetLocation(TargetDescriptor.ProjectFile, TargetDescriptor.Name, TargetDescriptor.Platform, TargetDescriptor.Architecture, TargetDescriptor.Configuration);
}
// Try to load an existing makefile
TargetMakefile Makefile = null;
if (MakefileLocation != null)
{
using (Timeline.ScopeEvent("TargetMakefile.Load()"))
{
string ReasonNotLoaded;
Makefile = TargetMakefile.Load(MakefileLocation, TargetDescriptor.ProjectFile, TargetDescriptor.Platform, TargetDescriptor.AdditionalArguments.GetRawArray(), out ReasonNotLoaded);
if (Makefile == null)
{
Log.TraceInformation("Creating makefile for {0} ({1})", TargetDescriptor.Name, ReasonNotLoaded);
}
}
}
// If we have a makefile, execute the pre-build steps and check it's still valid
bool bHasRunPreBuildScripts = false;
if (Makefile != null)
{
// Execute the scripts. We have to invalidate all cached file info after doing so, because we don't know what may have changed.
if (Makefile.PreBuildScripts.Length > 0)
{
Utils.ExecuteCustomBuildSteps(Makefile.PreBuildScripts);
DirectoryItem.ResetAllCachedInfo_SLOW();
}
// Don't run the pre-build steps again, even if we invalidate the makefile.
bHasRunPreBuildScripts = true;
// Check that the makefile is still valid
string Reason;
if (!TargetMakefile.IsValidForSourceFiles(Makefile, TargetDescriptor.ProjectFile, TargetDescriptor.Platform, WorkingSet, out Reason))
{
Log.TraceInformation("Invalidating makefile for {0} ({1})", TargetDescriptor.Name, Reason);
Makefile = null;
}
}
// If we couldn't load a makefile, create a new one
if (Makefile == null)
{
// Create the target
UEBuildTarget Target;
using (Timeline.ScopeEvent("UEBuildTarget.Create()"))
{
Target = UEBuildTarget.Create(TargetDescriptor, BuildConfiguration.bSkipRulesCompile, BuildConfiguration.bUsePrecompiled);
}
// Create the pre-build scripts
FileReference[] PreBuildScripts = Target.CreatePreBuildScripts();
// Execute the pre-build scripts
if (!bHasRunPreBuildScripts)
{
Utils.ExecuteCustomBuildSteps(PreBuildScripts);
bHasRunPreBuildScripts = true;
}
// Build the target
using (Timeline.ScopeEvent("UEBuildTarget.Build()"))
{
const bool bIsAssemblingBuild = true;
Makefile = Target.Build(BuildConfiguration, WorkingSet, bIsAssemblingBuild, TargetDescriptor.SpecificFilesToCompile);
}
// Save the pre-build scripts onto the makefile
Makefile.PreBuildScripts = PreBuildScripts;
// Save the additional command line arguments
Makefile.AdditionalArguments = TargetDescriptor.AdditionalArguments.GetRawArray();
// Save the environment variables
foreach (System.Collections.DictionaryEntry EnvironmentVariable in Environment.GetEnvironmentVariables())
{
Makefile.EnvironmentVariables.Add(Tuple.Create((string)EnvironmentVariable.Key, (string)EnvironmentVariable.Value));
}
// Save the makefile for next time
if (MakefileLocation != null)
{
using (Timeline.ScopeEvent("TargetMakefile.Save()"))
{
Makefile.Save(MakefileLocation);
}
}
}
else
{
// Restore the environment variables
foreach (Tuple <string, string> EnvironmentVariable in Makefile.EnvironmentVariables)
{
Environment.SetEnvironmentVariable(EnvironmentVariable.Item1, EnvironmentVariable.Item2);
}
// If the target needs UHT to be run, we'll go ahead and do that now
if (Makefile.UObjectModules.Count > 0)
{
const bool bIsGatheringBuild = false;
const bool bIsAssemblingBuild = true;
FileReference ModuleInfoFileName = FileReference.Combine(Makefile.ProjectIntermediateDirectory, TargetDescriptor.Name + ".uhtmanifest");
ExternalExecution.ExecuteHeaderToolIfNecessary(BuildConfiguration, TargetDescriptor.ProjectFile, TargetDescriptor.Name, Makefile.TargetType, Makefile.bHasProjectScriptPlugin, UObjectModules: Makefile.UObjectModules, ModuleInfoFileName: ModuleInfoFileName, bIsGatheringBuild: bIsGatheringBuild, bIsAssemblingBuild: bIsAssemblingBuild, WorkingSet: WorkingSet);
}
}
return(Makefile);
}
/// <summary>
/// Build a list of targets with a given set of makefiles.
/// </summary>
/// <param name="Makefiles">Makefiles created with CreateMakefiles</param>
/// <param name="TargetDescriptors">Target descriptors</param>
/// <param name="BuildConfiguration">Current build configuration</param>
/// <param name="WorkingSet">The source file working set</param>
/// <param name="Options">Additional options for the build</param>
/// <param name="WriteOutdatedActionsFile">Files to write the list of outdated actions to (rather than building them)</param>
/// <returns>Result from the compilation</returns>
static void Build(TargetMakefile[] Makefiles, List <TargetDescriptor> TargetDescriptors, BuildConfiguration BuildConfiguration, ISourceFileWorkingSet WorkingSet, BuildOptions Options, FileReference WriteOutdatedActionsFile)
{
// Export the actions for each target
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
TargetDescriptor TargetDescriptor = TargetDescriptors[TargetIdx];
foreach (FileReference WriteActionFile in TargetDescriptor.WriteActionFiles)
{
Log.TraceInformation("Writing actions to {0}", WriteActionFile);
ActionGraph.ExportJson(Makefiles[TargetIdx].Actions, WriteActionFile);
}
}
// Execute the build
if ((Options & BuildOptions.SkipBuild) == 0)
{
// Make sure that none of the actions conflict with any other (producing output files differently, etc...)
ActionGraph.CheckForConflicts(Makefiles.SelectMany(x => x.Actions));
// Check we don't exceed the nominal max path length
using (Timeline.ScopeEvent("ActionGraph.CheckPathLengths"))
{
ActionGraph.CheckPathLengths(BuildConfiguration, Makefiles.SelectMany(x => x.Actions));
}
// Clean up any previous hot reload runs, and reapply the current state if it's already active
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
HotReload.Setup(TargetDescriptors[TargetIdx], Makefiles[TargetIdx], BuildConfiguration);
}
// Merge the action graphs together
List <Action> MergedActions;
if (TargetDescriptors.Count == 1)
{
MergedActions = new List <Action>(Makefiles[0].Actions);
}
else
{
MergedActions = MergeActionGraphs(TargetDescriptors, Makefiles);
}
// Gather all the prerequisite actions that are part of the targets
HashSet <FileItem> MergedOutputItems = new HashSet <FileItem>();
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
GatherOutputItems(TargetDescriptors[TargetIdx], Makefiles[TargetIdx], MergedOutputItems);
}
// Link all the actions together
ActionGraph.Link(MergedActions);
// Get all the actions that are prerequisites for these targets. This forms the list of actions that we want executed.
List <Action> PrerequisiteActions = ActionGraph.GatherPrerequisiteActions(MergedActions, MergedOutputItems);
// Create the action history
ActionHistory History = new ActionHistory();
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
using (Timeline.ScopeEvent("Reading action history"))
{
TargetDescriptor TargetDescriptor = TargetDescriptors[TargetIdx];
if (TargetDescriptor.ProjectFile != null)
{
History.Mount(TargetDescriptor.ProjectFile.Directory);
}
}
}
// Figure out which actions need to be built
Dictionary <Action, bool> ActionToOutdatedFlag = new Dictionary <Action, bool>();
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
TargetDescriptor TargetDescriptor = TargetDescriptors[TargetIdx];
// Create the dependencies cache
CppDependencyCache CppDependencies;
using (Timeline.ScopeEvent("Reading dependency cache"))
{
CppDependencies = CppDependencyCache.CreateHierarchy(TargetDescriptor.ProjectFile, TargetDescriptor.Name, TargetDescriptor.Platform, TargetDescriptor.Configuration, Makefiles[TargetIdx].TargetType, TargetDescriptor.Architecture);
}
// Plan the actions to execute for the build. For single file compiles, always rebuild the source file regardless of whether it's out of date.
if (TargetDescriptor.SpecificFilesToCompile.Count == 0)
{
ActionGraph.GatherAllOutdatedActions(PrerequisiteActions, History, ActionToOutdatedFlag, CppDependencies, BuildConfiguration.bIgnoreOutdatedImportLibraries);
}
else
{
foreach (FileReference SpecificFile in TargetDescriptor.SpecificFilesToCompile)
{
foreach (Action PrerequisiteAction in PrerequisiteActions.Where(x => x.PrerequisiteItems.Any(y => y.Location == SpecificFile)))
{
ActionToOutdatedFlag[PrerequisiteAction] = true;
}
}
}
}
// Link the action graph again to sort it
List <Action> MergedActionsToExecute = ActionToOutdatedFlag.Where(x => x.Value).Select(x => x.Key).ToList();
ActionGraph.Link(MergedActionsToExecute);
// Allow hot reload to override the actions
int HotReloadTargetIdx = -1;
for (int Idx = 0; Idx < TargetDescriptors.Count; Idx++)
{
if (TargetDescriptors[Idx].HotReloadMode != HotReloadMode.Disabled)
{
if (HotReloadTargetIdx != -1)
{
throw new BuildException("Unable to perform hot reload with multiple targets.");
}
else
{
MergedActionsToExecute = HotReload.PatchActionsForTarget(BuildConfiguration, TargetDescriptors[Idx], Makefiles[Idx], PrerequisiteActions, MergedActionsToExecute);
}
HotReloadTargetIdx = Idx;
}
}
// Make sure we're not modifying any engine files
if ((Options & BuildOptions.NoEngineChanges) != 0)
{
List <FileItem> EngineChanges = MergedActionsToExecute.SelectMany(x => x.ProducedItems).Where(x => x.Location.IsUnderDirectory(UnrealBuildTool.EngineDirectory)).Distinct().OrderBy(x => x.FullName).ToList();
if (EngineChanges.Count > 0)
{
StringBuilder Result = new StringBuilder("Building would modify the following engine files:\n");
foreach (FileItem EngineChange in EngineChanges)
{
Result.AppendFormat("\n{0}", EngineChange.FullName);
}
Result.Append("\n\nPlease rebuild from an IDE instead.");
Log.TraceError("{0}", Result.ToString());
throw new CompilationResultException(CompilationResult.FailedDueToEngineChange);
}
}
// Make sure the appropriate executor is selected
foreach (TargetDescriptor TargetDescriptor in TargetDescriptors)
{
UEBuildPlatform BuildPlatform = UEBuildPlatform.GetBuildPlatform(TargetDescriptor.Platform);
BuildConfiguration.bAllowXGE &= BuildPlatform.CanUseXGE();
BuildConfiguration.bAllowDistcc &= BuildPlatform.CanUseDistcc();
BuildConfiguration.bAllowSNDBS &= BuildPlatform.CanUseSNDBS();
}
// Delete produced items that are outdated.
ActionGraph.DeleteOutdatedProducedItems(MergedActionsToExecute);
// Save all the action histories now that files have been removed. We have to do this after deleting produced items to ensure that any
// items created during the build don't have the wrong command line.
History.Save();
// Create directories for the outdated produced items.
ActionGraph.CreateDirectoriesForProducedItems(MergedActionsToExecute);
// Execute the actions
if ((Options & BuildOptions.XGEExport) != 0)
{
OutputToolchainInfo(TargetDescriptors, Makefiles);
// Just export to an XML file
using (Timeline.ScopeEvent("XGE.ExportActions()"))
{
XGE.ExportActions(MergedActionsToExecute);
}
}
else if (WriteOutdatedActionsFile != null)
{
OutputToolchainInfo(TargetDescriptors, Makefiles);
// Write actions to an output file
using (Timeline.ScopeEvent("ActionGraph.WriteActions"))
{
ActionGraph.ExportJson(MergedActionsToExecute, WriteOutdatedActionsFile);
}
}
else
{
// Execute the actions
if (MergedActionsToExecute.Count == 0)
{
if (TargetDescriptors.Any(x => !x.bQuiet))
{
Log.TraceInformation((TargetDescriptors.Count == 1)? "Target is up to date" : "Targets are up to date");
}
}
else
{
if (TargetDescriptors.Any(x => !x.bQuiet))
{
Log.TraceInformation("Building {0}...", StringUtils.FormatList(TargetDescriptors.Select(x => x.Name).Distinct()));
}
OutputToolchainInfo(TargetDescriptors, Makefiles);
using (Timeline.ScopeEvent("ActionGraph.ExecuteActions()"))
{
ActionGraph.ExecuteActions(BuildConfiguration, MergedActionsToExecute);
}
}
// Run the deployment steps
foreach (TargetMakefile Makefile in Makefiles)
{
if (Makefile.bDeployAfterCompile)
{
TargetReceipt Receipt = TargetReceipt.Read(Makefile.ReceiptFile);
Log.TraceInformation("Deploying {0} {1} {2}...", Receipt.TargetName, Receipt.Platform, Receipt.Configuration);
UEBuildPlatform.GetBuildPlatform(Receipt.Platform).Deploy(Receipt);
}
}
}
}
}
/// <summary>
/// Determine what needs to be built for a target
/// </summary>
/// <param name="BuildConfiguration">The build configuration</param>
/// <param name="TargetDescriptor">Target being built</param>
/// <param name="Makefile">Makefile generated for this target</param>
/// <returns>Set of actions to execute</returns>
static HashSet <Action> GetActionsForTarget(BuildConfiguration BuildConfiguration, TargetDescriptor TargetDescriptor, TargetMakefile Makefile)
{
// Create the action graph
ActionGraph.Link(Makefile.Actions);
// Get the hot-reload mode
HotReloadMode HotReloadMode = TargetDescriptor.HotReloadMode;
if (HotReloadMode == HotReloadMode.Default)
{
if (TargetDescriptor.HotReloadModuleNameToSuffix.Count > 0 && TargetDescriptor.ForeignPlugin == null)
{
HotReloadMode = HotReloadMode.FromEditor;
}
else if (BuildConfiguration.bAllowHotReloadFromIDE && HotReload.ShouldDoHotReloadFromIDE(BuildConfiguration, TargetDescriptor))
{
HotReloadMode = HotReloadMode.FromIDE;
}
else
{
HotReloadMode = HotReloadMode.Disabled;
}
}
// Get the root prerequisite actions
List <Action> PrerequisiteActions = GatherPrerequisiteActions(TargetDescriptor, Makefile);
// Get the path to the hot reload state file for this target
FileReference HotReloadStateFile = global::UnrealBuildTool.HotReloadState.GetLocation(TargetDescriptor.ProjectFile, TargetDescriptor.Name, TargetDescriptor.Platform, TargetDescriptor.Configuration, TargetDescriptor.Architecture);
// Apply the previous hot reload state
HotReloadState HotReloadState = null;
if (HotReloadMode == HotReloadMode.Disabled)
{
// Make sure we're not doing a partial build from the editor (eg. compiling a new plugin)
if (TargetDescriptor.ForeignPlugin == null && TargetDescriptor.SingleFileToCompile == null)
{
// Delete the previous state file
HotReload.DeleteTemporaryFiles(HotReloadStateFile);
}
}
else
{
// Read the previous state file and apply it to the action graph
if (FileReference.Exists(HotReloadStateFile))
{
HotReloadState = HotReloadState.Load(HotReloadStateFile);
}
else
{
HotReloadState = new HotReloadState();
}
// Apply the old state to the makefile
HotReload.ApplyState(HotReloadState, Makefile);
// If we want a specific suffix on any modules, apply that now. We'll track the outputs later, but the suffix has to be forced (and is always out of date if it doesn't exist).
HotReload.PatchActionGraphWithNames(PrerequisiteActions, TargetDescriptor.HotReloadModuleNameToSuffix, Makefile);
// Re-link the action graph
ActionGraph.Link(PrerequisiteActions);
}
// Create the dependencies cache
CppDependencyCache CppDependencies;
using (Timeline.ScopeEvent("Reading dependency cache"))
{
CppDependencies = CppDependencyCache.CreateHierarchy(TargetDescriptor.ProjectFile, TargetDescriptor.Name, TargetDescriptor.Platform, TargetDescriptor.Configuration, Makefile.TargetType);
}
// Create the action history
ActionHistory History;
using (Timeline.ScopeEvent("Reading action history"))
{
History = ActionHistory.CreateHierarchy(TargetDescriptor.ProjectFile, TargetDescriptor.Name, TargetDescriptor.Platform, Makefile.TargetType);
}
// Plan the actions to execute for the build. For single file compiles, always rebuild the source file regardless of whether it's out of date.
HashSet <Action> TargetActionsToExecute;
if (TargetDescriptor.SingleFileToCompile == null)
{
TargetActionsToExecute = ActionGraph.GetActionsToExecute(Makefile.Actions, PrerequisiteActions, CppDependencies, History, BuildConfiguration.bIgnoreOutdatedImportLibraries);
}
else
{
TargetActionsToExecute = new HashSet <Action>(PrerequisiteActions);
}
// Additional processing for hot reload
if (HotReloadMode == HotReloadMode.LiveCoding)
{
// Filter the prerequisite actions down to just the compile actions, then recompute all the actions to execute
PrerequisiteActions = new List <Action>(TargetActionsToExecute.Where(x => x.ActionType == ActionType.Compile));
TargetActionsToExecute = ActionGraph.GetActionsToExecute(Makefile.Actions, PrerequisiteActions, CppDependencies, History, BuildConfiguration.bIgnoreOutdatedImportLibraries);
}
else if (HotReloadMode == HotReloadMode.FromEditor || HotReloadMode == HotReloadMode.FromIDE)
{
// Patch action history for hot reload when running in assembler mode. In assembler mode, the suffix on the output file will be
// the same for every invocation on that makefile, but we need a new suffix each time.
// For all the hot-reloadable modules that may need a unique suffix appended, build a mapping from output item to all the output items in that module. We can't
// apply a suffix to one without applying a suffix to all of them.
Dictionary <FileItem, FileItem[]> HotReloadItemToDependentItems = new Dictionary <FileItem, FileItem[]>();
foreach (string HotReloadModuleName in Makefile.HotReloadModuleNames)
{
int ModuleSuffix;
if (!TargetDescriptor.HotReloadModuleNameToSuffix.TryGetValue(HotReloadModuleName, out ModuleSuffix) || ModuleSuffix == -1)
{
FileItem[] ModuleOutputItems;
if (Makefile.ModuleNameToOutputItems.TryGetValue(HotReloadModuleName, out ModuleOutputItems))
{
foreach (FileItem ModuleOutputItem in ModuleOutputItems)
{
HotReloadItemToDependentItems[ModuleOutputItem] = ModuleOutputItems;
}
}
}
}
// Expand the list of actions to execute to include everything that references any files with a new suffix. Unlike a regular build, we can't ignore
// dependencies on import libraries under the assumption that a header would change if the API changes, because the dependency will be on a different DLL.
HashSet <FileItem> FilesRequiringSuffix = new HashSet <FileItem>(TargetActionsToExecute.SelectMany(x => x.ProducedItems).Where(x => HotReloadItemToDependentItems.ContainsKey(x)));
for (int LastNumFilesWithNewSuffix = 0; FilesRequiringSuffix.Count > LastNumFilesWithNewSuffix;)
{
LastNumFilesWithNewSuffix = FilesRequiringSuffix.Count;
foreach (Action PrerequisiteAction in PrerequisiteActions)
{
if (!TargetActionsToExecute.Contains(PrerequisiteAction))
{
foreach (FileItem ProducedItem in PrerequisiteAction.ProducedItems)
{
FileItem[] DependentItems;
if (HotReloadItemToDependentItems.TryGetValue(ProducedItem, out DependentItems))
{
TargetActionsToExecute.Add(PrerequisiteAction);
FilesRequiringSuffix.UnionWith(DependentItems);
}
}
}
}
}
// Build a list of file mappings
Dictionary <FileReference, FileReference> OldLocationToNewLocation = new Dictionary <FileReference, FileReference>();
foreach (FileItem FileRequiringSuffix in FilesRequiringSuffix)
{
FileReference OldLocation = FileRequiringSuffix.Location;
FileReference NewLocation = HotReload.ReplaceSuffix(OldLocation, HotReloadState.NextSuffix);
OldLocationToNewLocation[OldLocation] = NewLocation;
}
// Update the action graph with these new paths
HotReload.PatchActionGraph(PrerequisiteActions, OldLocationToNewLocation);
// Get a new list of actions to execute now that the graph has been modified
TargetActionsToExecute = ActionGraph.GetActionsToExecute(Makefile.Actions, PrerequisiteActions, CppDependencies, History, BuildConfiguration.bIgnoreOutdatedImportLibraries);
// Build a mapping of all file items to their original
Dictionary <FileReference, FileReference> HotReloadFileToOriginalFile = new Dictionary <FileReference, FileReference>();
foreach (KeyValuePair <FileReference, FileReference> Pair in HotReloadState.OriginalFileToHotReloadFile)
{
HotReloadFileToOriginalFile[Pair.Value] = Pair.Key;
}
foreach (KeyValuePair <FileReference, FileReference> Pair in OldLocationToNewLocation)
{
FileReference OriginalLocation;
if (!HotReloadFileToOriginalFile.TryGetValue(Pair.Key, out OriginalLocation))
{
OriginalLocation = Pair.Key;
}
HotReloadFileToOriginalFile[Pair.Value] = OriginalLocation;
}
// Now filter out all the hot reload files and update the state
foreach (Action Action in TargetActionsToExecute)
{
foreach (FileItem ProducedItem in Action.ProducedItems)
{
FileReference OriginalLocation;
if (HotReloadFileToOriginalFile.TryGetValue(ProducedItem.Location, out OriginalLocation))
{
HotReloadState.OriginalFileToHotReloadFile[OriginalLocation] = ProducedItem.Location;
HotReloadState.TemporaryFiles.Add(ProducedItem.Location);
}
}
}
// Increment the suffix for the next iteration
if (TargetActionsToExecute.Count > 0)
{
HotReloadState.NextSuffix++;
}
// Save the new state
HotReloadState.Save(HotReloadStateFile);
// Prevent this target from deploying
Makefile.bDeployAfterCompile = false;
}
return(TargetActionsToExecute);
}
/// <summary>
/// Build a list of targets
/// </summary>
/// <param name="TargetDescriptors">Target descriptors</param>
/// <param name="BuildConfiguration">Current build configuration</param>
/// <param name="WorkingSet">The source file working set</param>
/// <param name="Options">Additional options for the build</param>
/// <param name="LiveCodingManifest">Path to write the live coding manifest to</param>
/// <param name="WriteOutdatedActionsFile">Files to write the list of outdated actions to (rather than building them)</param>
/// <returns>Result from the compilation</returns>
public static void Build(List <TargetDescriptor> TargetDescriptors, BuildConfiguration BuildConfiguration, ISourceFileWorkingSet WorkingSet, BuildOptions Options, FileReference LiveCodingManifest, FileReference WriteOutdatedActionsFile)
{
// Create a makefile for each target
TargetMakefile[] Makefiles = new TargetMakefile[TargetDescriptors.Count];
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
Makefiles[TargetIdx] = CreateMakefile(BuildConfiguration, TargetDescriptors[TargetIdx], WorkingSet);
}
// Output the Live Coding manifest
if (LiveCodingManifest != null)
{
List <Action> AllActions = Makefiles.SelectMany(x => x.Actions).ToList();
HotReload.WriteLiveCodingManifest(LiveCodingManifest, AllActions);
}
// Export the actions for each target
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
TargetDescriptor TargetDescriptor = TargetDescriptors[TargetIdx];
foreach (FileReference WriteActionFile in TargetDescriptor.WriteActionFiles)
{
Log.TraceInformation("Writing actions to {0}", WriteActionFile);
ActionGraph.ExportJson(Makefiles[TargetIdx].Actions, WriteActionFile);
}
}
// Execute the build
if ((Options & BuildOptions.SkipBuild) == 0)
{
// Make sure that none of the actions conflict with any other (producing output files differently, etc...)
ActionGraph.CheckForConflicts(Makefiles.SelectMany(x => x.Actions));
// Find all the actions to be executed
HashSet <Action>[] ActionsToExecute = new HashSet <Action> [TargetDescriptors.Count];
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
ActionsToExecute[TargetIdx] = GetActionsForTarget(BuildConfiguration, TargetDescriptors[TargetIdx], Makefiles[TargetIdx]);
}
// If there are multiple targets being built, merge the actions together
List <Action> MergedActionsToExecute;
if (TargetDescriptors.Count == 1)
{
MergedActionsToExecute = new List <Action>(ActionsToExecute[0]);
}
else
{
MergedActionsToExecute = MergeActionGraphs(TargetDescriptors, ActionsToExecute);
}
// Link all the actions together
ActionGraph.Link(MergedActionsToExecute);
// Make sure the appropriate executor is selected
foreach (TargetDescriptor TargetDescriptor in TargetDescriptors)
{
UEBuildPlatform BuildPlatform = UEBuildPlatform.GetBuildPlatform(TargetDescriptor.Platform);
BuildConfiguration.bAllowXGE &= BuildPlatform.CanUseXGE();
BuildConfiguration.bAllowDistcc &= BuildPlatform.CanUseDistcc();
BuildConfiguration.bAllowSNDBS &= BuildPlatform.CanUseSNDBS();
}
// Delete produced items that are outdated.
ActionGraph.DeleteOutdatedProducedItems(MergedActionsToExecute);
// Save all the action histories now that files have been removed. We have to do this after deleting produced items to ensure that any
// items created during the build don't have the wrong command line.
ActionHistory.SaveAll();
// Create directories for the outdated produced items.
ActionGraph.CreateDirectoriesForProducedItems(MergedActionsToExecute);
// Execute the actions
if ((Options & BuildOptions.XGEExport) != 0)
{
// Just export to an XML file
using (Timeline.ScopeEvent("XGE.ExportActions()"))
{
XGE.ExportActions(MergedActionsToExecute);
}
}
else if (WriteOutdatedActionsFile != null)
{
// Write actions to an output file
using (Timeline.ScopeEvent("ActionGraph.WriteActions"))
{
ActionGraph.ExportJson(MergedActionsToExecute, WriteOutdatedActionsFile);
}
}
else
{
// Execute the actions
if (MergedActionsToExecute.Count == 0)
{
if ((Options & BuildOptions.Quiet) == 0)
{
Log.TraceInformation((TargetDescriptors.Count == 1)? "Target is up to date" : "Targets are up to date");
}
}
else
{
if ((Options & BuildOptions.Quiet) != 0)
{
Log.TraceInformation("Building {0}...", StringUtils.FormatList(TargetDescriptors.Select(x => x.Name).Distinct()));
}
OutputToolchainInfo(TargetDescriptors, Makefiles);
using (Timeline.ScopeEvent("ActionGraph.ExecuteActions()"))
{
ActionGraph.ExecuteActions(BuildConfiguration, MergedActionsToExecute);
}
}
// Run the deployment steps
foreach (TargetMakefile Makefile in Makefiles)
{
if (Makefile.bDeployAfterCompile)
{
TargetReceipt Receipt = TargetReceipt.Read(Makefile.ReceiptFile);
Log.TraceInformation("Deploying {0} {1} {2}...", Receipt.TargetName, Receipt.Platform, Receipt.Configuration);
UEBuildPlatform.GetBuildPlatform(Receipt.Platform).Deploy(Receipt);
}
}
}
}
}
/// <summary>
/// Dynamically compiles an assembly for the specified source file and loads that assembly into the application's
/// current domain. If an assembly has already been compiled and is not out of date, then it will be loaded and
/// no compilation is necessary.
/// </summary>
/// <param name="OutputAssemblyPath">Full path to the assembly to be created</param>
/// <param name="SourceFileNames">List of source file name</param>
/// <param name="ReferencedAssembies"></param>
/// <param name="PreprocessorDefines"></param>
/// <param name="DoNotCompile"></param>
/// <param name="TreatWarningsAsErrors"></param>
/// <returns>The assembly that was loaded</returns>
public static Assembly CompileAndLoadAssembly(FileReference OutputAssemblyPath, HashSet <FileReference> SourceFileNames, List <string> ReferencedAssembies = null, List <string> PreprocessorDefines = null, bool DoNotCompile = false, bool TreatWarningsAsErrors = false)
{
// Check to see if the resulting assembly is compiled and up to date
FileReference AssemblyManifestFilePath = FileReference.Combine(OutputAssemblyPath.Directory, Path.GetFileNameWithoutExtension(OutputAssemblyPath.FullName) + "Manifest.json");
bool bNeedsCompilation = false;
if (!DoNotCompile)
{
bNeedsCompilation = RequiresCompilation(SourceFileNames, AssemblyManifestFilePath, OutputAssemblyPath);
}
// Load the assembly to ensure it is correct
Assembly CompiledAssembly = null;
if (!bNeedsCompilation)
{
try
{
// Load the previously-compiled assembly from disk
CompiledAssembly = Assembly.LoadFile(OutputAssemblyPath.FullName);
}
catch (FileLoadException Ex)
{
Log.TraceInformation(String.Format("Unable to load the previously-compiled assembly file '{0}'. Unreal Build Tool will try to recompile this assembly now. (Exception: {1})", OutputAssemblyPath, Ex.Message));
bNeedsCompilation = true;
}
catch (BadImageFormatException Ex)
{
Log.TraceInformation(String.Format("Compiled assembly file '{0}' appears to be for a newer CLR version or is otherwise invalid. Unreal Build Tool will try to recompile this assembly now. (Exception: {1})", OutputAssemblyPath, Ex.Message));
bNeedsCompilation = true;
}
catch (FileNotFoundException)
{
throw new BuildException("Precompiled rules assembly '{0}' does not exist.", OutputAssemblyPath);
}
catch (Exception Ex)
{
throw new BuildException(Ex, "Error while loading previously-compiled assembly file '{0}'. (Exception: {1})", OutputAssemblyPath, Ex.Message);
}
}
// Compile the assembly if me
if (bNeedsCompilation)
{
using (Timeline.ScopeEvent(String.Format("Compiling rules assembly ({0})", OutputAssemblyPath.GetFileName())))
{
CompiledAssembly = CompileAssembly(OutputAssemblyPath, SourceFileNames, ReferencedAssembies, PreprocessorDefines, TreatWarningsAsErrors);
}
using (JsonWriter Writer = new JsonWriter(AssemblyManifestFilePath))
{
ReadOnlyBuildVersion Version = ReadOnlyBuildVersion.Current;
Writer.WriteObjectStart();
// Save out a list of all the source files we compiled. This is so that we can tell if whole files were added or removed
// since the previous time we compiled the assembly. In that case, we'll always want to recompile it!
Writer.WriteStringArrayField("SourceFiles", SourceFileNames.Select(x => x.FullName));
Writer.WriteValue("EngineVersion", FormatVersionNumber(Version));
Writer.WriteObjectEnd();
}
}
#if !NET_CORE
// Load the assembly into our app domain
try
{
AppDomain.CurrentDomain.Load(CompiledAssembly.GetName());
}
catch (Exception Ex)
{
throw new BuildException(Ex, "Unable to load the compiled build assembly '{0}' into our application's domain. (Exception: {1})", OutputAssemblyPath, Ex.Message);
}
#endif
return(CompiledAssembly);
}
/// <summary>
/// Creates a rules assembly
/// </summary>
/// <param name="Scope">Scope for items created from this assembly</param>
/// <param name="RootDirectories">The root directories to create rules for</param>
/// <param name="AssemblyPrefix">A prefix for the assembly file name</param>
/// <param name="Plugins">List of plugins to include in this assembly</param>
/// <param name="bReadOnly">Whether the assembly should be marked as installed</param>
/// <param name="bSkipCompile">Whether to skip compilation for this assembly</param>
/// <param name="Parent">The parent rules assembly</param>
/// <returns>New rules assembly</returns>
private static RulesAssembly CreateEngineOrEnterpriseRulesAssembly(RulesScope Scope, List <DirectoryReference> RootDirectories, string AssemblyPrefix, IReadOnlyList <PluginInfo> Plugins, bool bReadOnly, bool bSkipCompile, RulesAssembly Parent)
{
// Scope hierarchy
RulesScope PluginsScope = new RulesScope(Scope.Name + " Plugins", Scope);
RulesScope ProgramsScope = new RulesScope(Scope.Name + " Programs", PluginsScope);
// Find the shared modules, excluding the programs directory. These are used to create an assembly with the bContainsEngineModules flag set to true.
Dictionary <FileReference, ModuleRulesContext> ModuleFileToContext = new Dictionary <FileReference, ModuleRulesContext>();
ModuleRulesContext DefaultModuleContext = new ModuleRulesContext(Scope, RootDirectories[0]);
foreach (DirectoryReference RootDirectory in RootDirectories)
{
using (Timeline.ScopeEvent("Finding engine modules"))
{
DirectoryReference SourceDirectory = DirectoryReference.Combine(RootDirectory, "Source");
AddEngineModuleRulesWithContext(SourceDirectory, "Runtime", DefaultModuleContext, UHTModuleType.EngineRuntime, ModuleFileToContext);
AddEngineModuleRulesWithContext(SourceDirectory, "Developer", DefaultModuleContext, UHTModuleType.EngineDeveloper, ModuleFileToContext);
AddEngineModuleRulesWithContext(SourceDirectory, "Editor", DefaultModuleContext, UHTModuleType.EngineEditor, ModuleFileToContext);
AddEngineModuleRulesWithContext(SourceDirectory, "ThirdParty", DefaultModuleContext, UHTModuleType.EngineThirdParty, ModuleFileToContext);
}
}
// Add all the plugin modules too (don't need to loop over RootDirectories since the plugins come in already found
using (Timeline.ScopeEvent("Finding plugin modules"))
{
ModuleRulesContext PluginsModuleContext = new ModuleRulesContext(PluginsScope, RootDirectories[0]);
FindModuleRulesForPlugins(Plugins, PluginsModuleContext, ModuleFileToContext);
}
// Get the path to store any generated assemblies
DirectoryReference AssemblyDir = RootDirectories[0];
if (UnrealBuildTool.IsFileInstalled(FileReference.Combine(AssemblyDir, AssemblyPrefix)))
{
DirectoryReference UserDir = Utils.GetUserSettingDirectory();
if (UserDir != null)
{
ReadOnlyBuildVersion Version = ReadOnlyBuildVersion.Current;
AssemblyDir = DirectoryReference.Combine(UserDir, "UnrealEngine", String.Format("{0}.{1}", Version.MajorVersion, Version.MinorVersion));
}
}
// Create the assembly
FileReference EngineAssemblyFileName = FileReference.Combine(AssemblyDir, "Intermediate", "Build", "BuildRules", AssemblyPrefix + "Rules" + FrameworkAssemblyExtension);
RulesAssembly EngineAssembly = new RulesAssembly(Scope, RootDirectories, Plugins, ModuleFileToContext, new List <FileReference>(), EngineAssemblyFileName, bContainsEngineModules: true, DefaultBuildSettings: BuildSettingsVersion.Latest, bReadOnly: bReadOnly, bSkipCompile: bSkipCompile, Parent: Parent);
List <FileReference> ProgramTargetFiles = new List <FileReference>();
Dictionary <FileReference, ModuleRulesContext> ProgramModuleFiles = new Dictionary <FileReference, ModuleRulesContext>();
foreach (DirectoryReference RootDirectory in RootDirectories)
{
DirectoryReference SourceDirectory = DirectoryReference.Combine(RootDirectory, "Source");
DirectoryReference ProgramsDirectory = DirectoryReference.Combine(SourceDirectory, "Programs");
// Also create a scope for them, and update the UHT module type
ModuleRulesContext ProgramsModuleContext = new ModuleRulesContext(ProgramsScope, RootDirectory);
ProgramsModuleContext.DefaultUHTModuleType = UHTModuleType.Program;
using (Timeline.ScopeEvent("Finding program modules"))
{
// Find all the rules files
AddModuleRulesWithContext(ProgramsDirectory, ProgramsModuleContext, ProgramModuleFiles);
}
using (Timeline.ScopeEvent("Finding program targets"))
{
ProgramTargetFiles.AddRange(FindAllRulesFiles(SourceDirectory, RulesFileType.Target));
}
}
// Create a path to the assembly that we'll either load or compile
FileReference ProgramAssemblyFileName = FileReference.Combine(AssemblyDir, "Intermediate", "Build", "BuildRules", AssemblyPrefix + "ProgramRules" + FrameworkAssemblyExtension);
RulesAssembly ProgramAssembly = new RulesAssembly(ProgramsScope, RootDirectories, new List <PluginInfo>().AsReadOnly(), ProgramModuleFiles, ProgramTargetFiles, ProgramAssemblyFileName, bContainsEngineModules: false, DefaultBuildSettings: BuildSettingsVersion.Latest, bReadOnly: bReadOnly, bSkipCompile: bSkipCompile, Parent: EngineAssembly);
// Return the combined assembly
return(ProgramAssembly);
}
/// <summary>
/// Main entry point
/// </summary>
/// <param name="Arguments">Command-line arguments</param>
/// <returns>One of the values of ECompilationResult</returns>
public override int Execute(CommandLineArguments Arguments)
{
Arguments.ApplyTo(this);
// Initialize the log system, buffering the output until we can create the log file
StartupTraceListener StartupListener = new StartupTraceListener();
Trace.Listeners.Add(StartupListener);
// Write the command line
Log.TraceLog("Command line: {0}", Environment.CommandLine);
// Grab the environment.
UnrealBuildTool.InitialEnvironment = Environment.GetEnvironmentVariables();
if (UnrealBuildTool.InitialEnvironment.Count < 1)
{
throw new BuildException("Environment could not be read");
}
// Read the XML configuration files
XmlConfig.ApplyTo(this);
// Create the log file, and flush the startup listener to it
if (!Arguments.HasOption("-NoLog") && !Log.HasFileWriter())
{
FileReference LogFile = new FileReference(BaseLogFileName);
foreach (string LogSuffix in Arguments.GetValues("-LogSuffix="))
{
LogFile = LogFile.ChangeExtension(null) + "_" + LogSuffix + LogFile.GetExtension();
}
TextWriterTraceListener LogTraceListener = Log.AddFileWriter("DefaultLogTraceListener", LogFile);
StartupListener.CopyTo(LogTraceListener);
}
Trace.Listeners.Remove(StartupListener);
// Create the build configuration object, and read the settings
BuildConfiguration BuildConfiguration = new BuildConfiguration();
XmlConfig.ApplyTo(BuildConfiguration);
Arguments.ApplyTo(BuildConfiguration);
// now that we know the available platforms, we can delete other platforms' junk. if we're only building specific modules from the editor, don't touch anything else (it may be in use).
if (!bIgnoreJunk && !UnrealBuildTool.IsEngineInstalled())
{
using (Timeline.ScopeEvent("DeleteJunk()"))
{
JunkDeleter.DeleteJunk();
}
}
// Parse and build the targets
try
{
// Parse all the target descriptors
List <TargetDescriptor> TargetDescriptors;
using (Timeline.ScopeEvent("TargetDescriptor.ParseCommandLine()"))
{
TargetDescriptors = TargetDescriptor.ParseCommandLine(Arguments, BuildConfiguration.bUsePrecompiled, BuildConfiguration.bSkipRulesCompile);
}
// Hack for single file compile; don't build the ShaderCompileWorker target that's added to the command line for generated project files
if (TargetDescriptors.Count >= 2)
{
TargetDescriptors.RemoveAll(x => x.Name == "ShaderCompileWorker" && x.SingleFileToCompile != null);
}
// Handle remote builds
for (int Idx = 0; Idx < TargetDescriptors.Count; Idx++)
{
TargetDescriptor TargetDesc = TargetDescriptors[Idx];
if (RemoteMac.HandlesTargetPlatform(TargetDesc.Platform))
{
FileReference BaseLogFile = Log.OutputFile ?? new FileReference(BaseLogFileName);
FileReference RemoteLogFile = FileReference.Combine(BaseLogFile.Directory, BaseLogFile.GetFileNameWithoutExtension() + "_Remote.txt");
RemoteMac RemoteMac = new RemoteMac(TargetDesc.ProjectFile);
if (!RemoteMac.Build(TargetDesc, RemoteLogFile))
{
return((int)CompilationResult.Unknown);
}
TargetDescriptors.RemoveAt(Idx--);
}
}
// Handle local builds
if (TargetDescriptors.Count > 0)
{
// Get a set of all the project directories
HashSet <DirectoryReference> ProjectDirs = new HashSet <DirectoryReference>();
foreach (TargetDescriptor TargetDesc in TargetDescriptors)
{
if (TargetDesc.ProjectFile != null)
{
DirectoryReference ProjectDirectory = TargetDesc.ProjectFile.Directory;
FileMetadataPrefetch.QueueProjectDirectory(ProjectDirectory);
ProjectDirs.Add(ProjectDirectory);
}
}
// Get all the build options
BuildOptions Options = BuildOptions.None;
if (bSkipBuild)
{
Options |= BuildOptions.SkipBuild;
}
if (bXGEExport)
{
Options |= BuildOptions.XGEExport;
}
// Create the working set provider
using (ISourceFileWorkingSet WorkingSet = SourceFileWorkingSet.Create(UnrealBuildTool.RootDirectory, ProjectDirs))
{
Build(TargetDescriptors, BuildConfiguration, WorkingSet, Options);
}
}
}
finally
{
// Save all the caches
SourceFileMetadataCache.SaveAll();
CppDependencyCache.SaveAll();
}
return(0);
}
/// <summary>
/// Checks if the makefile is valid for the current set of source files. This is done separately to the Load() method to allow pre-build steps to modify source files.
/// </summary>
/// <param name="Makefile">The makefile that has been loaded</param>
/// <param name="ProjectFile">Path to the project file</param>
/// <param name="WorkingSet">The current working set of source files</param>
/// <param name="ReasonNotLoaded">If the makefile is not valid, is set to a message describing why</param>
/// <returns>True if the makefile is valid, false otherwise</returns>
public static bool IsValidForSourceFiles(TargetMakefile Makefile, FileReference ProjectFile, ISourceFileWorkingSet WorkingSet, out string ReasonNotLoaded)
{
using (Timeline.ScopeEvent("TargetMakefile.IsValidForSourceFiles()"))
{
// Check if any source files have been added or removed
foreach (KeyValuePair <DirectoryItem, FileItem[]> Pair in Makefile.DirectoryToSourceFiles)
{
DirectoryItem InputDirectory = Pair.Key;
if (!InputDirectory.Exists || InputDirectory.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
FileItem[] SourceFiles = UEBuildModuleCPP.GetSourceFiles(InputDirectory);
if (SourceFiles.Length < Pair.Value.Length)
{
ReasonNotLoaded = "source file removed";
return(false);
}
else if (SourceFiles.Length > Pair.Value.Length)
{
ReasonNotLoaded = "source file added";
return(false);
}
else if (SourceFiles.Intersect(Pair.Value).Count() != SourceFiles.Length)
{
ReasonNotLoaded = "source file modified";
return(false);
}
}
}
// Check if any of the additional dependencies has changed
foreach (FileItem AdditionalDependency in Makefile.AdditionalDependencies)
{
if (!AdditionalDependency.Exists)
{
Log.TraceLog("{0} has been deleted since makefile was built.", AdditionalDependency.Location);
ReasonNotLoaded = string.Format("{0} deleted", AdditionalDependency.Location.GetFileName());
return(false);
}
if (AdditionalDependency.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
Log.TraceLog("{0} has been modified since makefile was built.", AdditionalDependency.Location);
ReasonNotLoaded = string.Format("{0} modified", AdditionalDependency.Location.GetFileName());
return(false);
}
}
// Check that no new plugins have been added
foreach (FileReference PluginFile in Plugins.EnumeratePlugins(ProjectFile))
{
FileItem PluginFileItem = FileItem.GetItemByFileReference(PluginFile);
if (!Makefile.PluginFiles.Contains(PluginFileItem))
{
Log.TraceLog("{0} has been added", PluginFile.GetFileName());
ReasonNotLoaded = string.Format("{0} has been added", PluginFile.GetFileName());
return(false);
}
}
// We do a check to see if any modules' headers have changed which have
// acquired or lost UHT types. If so, which should be rare,
// we'll just invalidate the entire makefile and force it to be rebuilt.
// Get all H files in processed modules newer than the makefile itself
HashSet <FileItem> HFilesNewerThanMakefile = new HashSet <FileItem>();
foreach (UHTModuleHeaderInfo ModuleHeaderInfo in Makefile.UObjectModuleHeaders)
{
foreach (FileItem HeaderFile in ModuleHeaderInfo.SourceFolder.EnumerateFiles())
{
if (HeaderFile.HasExtension(".h") && HeaderFile.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
HFilesNewerThanMakefile.Add(HeaderFile);
}
}
}
// Get all H files in all modules processed in the last makefile build
HashSet <FileItem> AllUHTHeaders = new HashSet <FileItem>(Makefile.UObjectModuleHeaders.SelectMany(x => x.HeaderFiles));
// Check whether any headers have been deleted. If they have, we need to regenerate the makefile since the module might now be empty. If we don't,
// and the file has been moved to a different module, we may include stale generated headers.
foreach (FileItem HeaderFile in AllUHTHeaders)
{
if (!HeaderFile.Exists)
{
Log.TraceLog("File processed by UHT was deleted ({0}); invalidating makefile", HeaderFile);
ReasonNotLoaded = string.Format("UHT file was deleted");
return(false);
}
}
// Makefile is invalid if:
// * There are any newer files which contain no UHT data, but were previously in the makefile
// * There are any newer files contain data which needs processing by UHT, but weren't not previously in the makefile
SourceFileMetadataCache MetadataCache = SourceFileMetadataCache.CreateHierarchy(ProjectFile);
foreach (FileItem HeaderFile in HFilesNewerThanMakefile)
{
bool bContainsUHTData = MetadataCache.ContainsReflectionMarkup(HeaderFile);
bool bWasProcessed = AllUHTHeaders.Contains(HeaderFile);
if (bContainsUHTData != bWasProcessed)
{
Log.TraceLog("{0} {1} contain UHT types and now {2} , ignoring it", HeaderFile, bWasProcessed ? "used to" : "didn't", bWasProcessed ? "doesn't" : "does");
ReasonNotLoaded = string.Format("new files with reflected types");
return(false);
}
}
// If adaptive unity build is enabled, do a check to see if there are any source files that became part of the
// working set since the Makefile was created (or, source files were removed from the working set.) If anything
// changed, then we'll force a new Makefile to be created so that we have fresh unity build blobs. We always
// want to make sure that source files in the working set are excluded from those unity blobs (for fastest possible
// iteration times.)
// Check if any source files in the working set no longer belong in it
foreach (FileItem SourceFile in Makefile.WorkingSet)
{
if (!WorkingSet.Contains(SourceFile) && SourceFile.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
Log.TraceLog("{0} was part of source working set and now is not; invalidating makefile", SourceFile.AbsolutePath);
ReasonNotLoaded = string.Format("working set of source files changed");
return(false);
}
}
// Check if any source files that are eligible for being in the working set have been modified
foreach (FileItem SourceFile in Makefile.CandidatesForWorkingSet)
{
if (WorkingSet.Contains(SourceFile) && SourceFile.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
Log.TraceLog("{0} was part of source working set and now is not", SourceFile.AbsolutePath);
ReasonNotLoaded = string.Format("working set of source files changed");
return(false);
}
}
}
ReasonNotLoaded = null;
return(true);
}
/// <summary>
/// Loads a makefile from disk
/// </summary>
/// <param name="MakefilePath">Path to the makefile to load</param>
/// <param name="ProjectFile">Path to the project file</param>
/// <param name="Platform">Platform for this makefile</param>
/// <param name="Arguments">Command line arguments for this target</param>
/// <param name="ReasonNotLoaded">If the function returns null, this string will contain the reason why</param>
/// <returns>The loaded makefile, or null if it failed for some reason. On failure, the 'ReasonNotLoaded' variable will contain information about why</returns>
public static TargetMakefile Load(FileReference MakefilePath, FileReference ProjectFile, UnrealTargetPlatform Platform, string[] Arguments, out string ReasonNotLoaded)
{
using (Timeline.ScopeEvent("Checking dependent timestamps"))
{
// Check the directory timestamp on the project files directory. If the user has generated project files more recently than the makefile, then we need to consider the file to be out of date
FileInfo MakefileInfo = new FileInfo(MakefilePath.FullName);
if (!MakefileInfo.Exists)
{
// Makefile doesn't even exist, so we won't bother loading it
ReasonNotLoaded = "no existing makefile";
return(null);
}
// Check the build version
FileInfo BuildVersionFileInfo = new FileInfo(BuildVersion.GetDefaultFileName().FullName);
if (BuildVersionFileInfo.Exists && MakefileInfo.LastWriteTime.CompareTo(BuildVersionFileInfo.LastWriteTime) < 0)
{
Log.TraceLog("Existing makefile is older than Build.version, ignoring it");
ReasonNotLoaded = "Build.version is newer";
return(null);
}
// @todo ubtmake: This will only work if the directory timestamp actually changes with every single GPF. Force delete existing files before creating new ones? Eh... really we probably just want to delete + create a file in that folder
// -> UPDATE: Seems to work OK right now though on Windows platform, maybe due to GUID changes
// @todo ubtmake: Some platforms may not save any files into this folder. We should delete + generate a "touch" file to force the directory timestamp to be updated (or just check the timestamp file itself. We could put it ANYWHERE, actually)
// Installed Build doesn't need to check engine projects for outdatedness
if (!UnrealBuildTool.IsEngineInstalled())
{
if (DirectoryReference.Exists(ProjectFileGenerator.IntermediateProjectFilesPath))
{
DateTime EngineProjectFilesLastUpdateTime = new FileInfo(ProjectFileGenerator.ProjectTimestampFile).LastWriteTime;
if (MakefileInfo.LastWriteTime.CompareTo(EngineProjectFilesLastUpdateTime) < 0)
{
// Engine project files are newer than makefile
Log.TraceLog("Existing makefile is older than generated engine project files, ignoring it");
ReasonNotLoaded = "project files are newer";
return(null);
}
}
}
// Check the game project directory too
if (ProjectFile != null)
{
string ProjectFilename = ProjectFile.FullName;
FileInfo ProjectFileInfo = new FileInfo(ProjectFilename);
if (!ProjectFileInfo.Exists || MakefileInfo.LastWriteTime.CompareTo(ProjectFileInfo.LastWriteTime) < 0)
{
// .uproject file is newer than makefile
Log.TraceLog("Makefile is older than .uproject file, ignoring it");
ReasonNotLoaded = ".uproject file is newer";
return(null);
}
DirectoryReference MasterProjectRelativePath = ProjectFile.Directory;
string GameIntermediateProjectFilesPath = Path.Combine(MasterProjectRelativePath.FullName, "Intermediate", "ProjectFiles");
if (Directory.Exists(GameIntermediateProjectFilesPath))
{
DateTime GameProjectFilesLastUpdateTime = new DirectoryInfo(GameIntermediateProjectFilesPath).LastWriteTime;
if (MakefileInfo.LastWriteTime.CompareTo(GameProjectFilesLastUpdateTime) < 0)
{
// Game project files are newer than makefile
Log.TraceLog("Makefile is older than generated game project files, ignoring it");
ReasonNotLoaded = "game project files are newer";
return(null);
}
}
}
// Check to see if UnrealBuildTool.exe was compiled more recently than the makefile
DateTime UnrealBuildToolTimestamp = new FileInfo(Assembly.GetExecutingAssembly().Location).LastWriteTime;
if (MakefileInfo.LastWriteTime.CompareTo(UnrealBuildToolTimestamp) < 0)
{
// UnrealBuildTool.exe was compiled more recently than the makefile
Log.TraceLog("Makefile is older than UnrealBuildTool.exe, ignoring it");
ReasonNotLoaded = "UnrealBuildTool.exe is newer";
return(null);
}
// Check to see if any BuildConfiguration files have changed since the last build
List <XmlConfig.InputFile> InputFiles = XmlConfig.FindInputFiles();
foreach (XmlConfig.InputFile InputFile in InputFiles)
{
FileInfo InputFileInfo = new FileInfo(InputFile.Location.FullName);
if (InputFileInfo.LastWriteTime > MakefileInfo.LastWriteTime)
{
Log.TraceLog("Makefile is older than BuildConfiguration.xml, ignoring it");
ReasonNotLoaded = "BuildConfiguration.xml is newer";
return(null);
}
}
}
TargetMakefile Makefile;
using (Timeline.ScopeEvent("Loading makefile"))
{
try
{
using (BinaryArchiveReader Reader = new BinaryArchiveReader(MakefilePath))
{
int Version = Reader.ReadInt();
if (Version != CurrentVersion)
{
ReasonNotLoaded = "makefile version does not match";
return(null);
}
Makefile = new TargetMakefile(Reader);
}
}
catch (Exception Ex)
{
Log.TraceWarning("Failed to read makefile: {0}", Ex.Message);
Log.TraceLog("Exception: {0}", Ex.ToString());
ReasonNotLoaded = "couldn't read existing makefile";
return(null);
}
}
using (Timeline.ScopeEvent("Checking makefile validity"))
{
// Check if the arguments are different
if (!Enumerable.SequenceEqual(Makefile.AdditionalArguments, Arguments))
{
ReasonNotLoaded = "command line arguments changed";
return(null);
}
// Check if ini files are newer. Ini files contain build settings too.
DirectoryReference ProjectDirectory = DirectoryReference.FromFile(ProjectFile);
foreach (ConfigHierarchyType IniType in (ConfigHierarchyType[])Enum.GetValues(typeof(ConfigHierarchyType)))
{
foreach (FileReference IniFilename in ConfigHierarchy.EnumerateConfigFileLocations(IniType, ProjectDirectory, Platform))
{
FileInfo IniFileInfo = new FileInfo(IniFilename.FullName);
if (IniFileInfo.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
// Ini files are newer than makefile
ReasonNotLoaded = "ini files are newer than makefile";
return(null);
}
}
}
// Get the current build metadata from the platform
string CurrentExternalMetadata = UEBuildPlatform.GetBuildPlatform(Platform).GetExternalBuildMetadata(ProjectFile);
if (String.Compare(CurrentExternalMetadata, Makefile.ExternalMetadata, StringComparison.Ordinal) != 0)
{
Log.TraceLog("Old metadata:\n", Makefile.ExternalMetadata);
Log.TraceLog("New metadata:\n", CurrentExternalMetadata);
ReasonNotLoaded = "build metadata has changed";
return(null);
}
}
// The makefile is ok
ReasonNotLoaded = null;
return(Makefile);
}
/// <summary>
/// Checks if the makefile is valid for the current set of source files. This is done separately to the Load() method to allow pre-build steps to modify source files.
/// </summary>
/// <param name="Makefile">The makefile that has been loaded</param>
/// <param name="ProjectFile">Path to the project file</param>
/// <param name="Platform">The platform being built</param>
/// <param name="WorkingSet">The current working set of source files</param>
/// <param name="ReasonNotLoaded">If the makefile is not valid, is set to a message describing why</param>
/// <returns>True if the makefile is valid, false otherwise</returns>
public static bool IsValidForSourceFiles(TargetMakefile Makefile, FileReference ProjectFile, UnrealTargetPlatform Platform, ISourceFileWorkingSet WorkingSet, out string ReasonNotLoaded)
{
using (Timeline.ScopeEvent("TargetMakefile.IsValidForSourceFiles()"))
{
// Get the list of excluded folder names for this platform
ReadOnlyHashSet <string> ExcludedFolderNames = UEBuildPlatform.GetBuildPlatform(Platform).GetExcludedFolderNames();
// Check if any source files have been added or removed
foreach (KeyValuePair <DirectoryItem, FileItem[]> Pair in Makefile.DirectoryToSourceFiles)
{
DirectoryItem InputDirectory = Pair.Key;
if (!InputDirectory.Exists || InputDirectory.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
FileItem[] SourceFiles = UEBuildModuleCPP.GetSourceFiles(InputDirectory);
if (SourceFiles.Length < Pair.Value.Length)
{
ReasonNotLoaded = "source file removed";
return(false);
}
else if (SourceFiles.Length > Pair.Value.Length)
{
ReasonNotLoaded = "source file added";
return(false);
}
else if (SourceFiles.Intersect(Pair.Value).Count() != SourceFiles.Length)
{
ReasonNotLoaded = "source file modified";
return(false);
}
foreach (DirectoryItem Directory in InputDirectory.EnumerateDirectories())
{
if (!Makefile.DirectoryToSourceFiles.ContainsKey(Directory) && ContainsSourceFiles(Directory, ExcludedFolderNames))
{
ReasonNotLoaded = "directory added";
return(false);
}
}
}
}
// Check if any external dependencies have changed. These comparisons are done against the makefile creation time.
foreach (FileItem ExternalDependency in Makefile.ExternalDependencies)
{
if (!ExternalDependency.Exists)
{
Log.TraceLog("{0} has been deleted since makefile was built.", ExternalDependency.Location);
ReasonNotLoaded = string.Format("{0} deleted", ExternalDependency.Location.GetFileName());
return(false);
}
if (ExternalDependency.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
Log.TraceLog("{0} has been modified since makefile was built.", ExternalDependency.Location);
ReasonNotLoaded = string.Format("{0} modified", ExternalDependency.Location.GetFileName());
return(false);
}
}
// Check if any internal dependencies has changed. These comparisons are done against the makefile modified time.
foreach (FileItem InternalDependency in Makefile.InternalDependencies)
{
if (!InternalDependency.Exists)
{
Log.TraceLog("{0} has been deleted since makefile was written.", InternalDependency.Location);
ReasonNotLoaded = string.Format("{0} deleted", InternalDependency.Location.GetFileName());
return(false);
}
if (InternalDependency.LastWriteTimeUtc > Makefile.ModifiedTimeUtc)
{
Log.TraceLog("{0} has been modified since makefile was written.", InternalDependency.Location);
ReasonNotLoaded = string.Format("{0} modified", InternalDependency.Location.GetFileName());
return(false);
}
}
// Check that no new plugins have been added
foreach (FileReference PluginFile in Plugins.EnumeratePlugins(ProjectFile))
{
FileItem PluginFileItem = FileItem.GetItemByFileReference(PluginFile);
if (!Makefile.PluginFiles.Contains(PluginFileItem))
{
Log.TraceLog("{0} has been added", PluginFile.GetFileName());
ReasonNotLoaded = string.Format("{0} has been added", PluginFile.GetFileName());
return(false);
}
}
// Load the metadata cache
SourceFileMetadataCache MetadataCache = SourceFileMetadataCache.CreateHierarchy(ProjectFile);
// Find the set of files that contain reflection markup
ConcurrentBag <FileItem> NewFilesWithMarkupBag = new ConcurrentBag <FileItem>();
using (ThreadPoolWorkQueue Queue = new ThreadPoolWorkQueue())
{
foreach (DirectoryItem SourceDirectory in Makefile.SourceDirectories)
{
Queue.Enqueue(() => FindFilesWithMarkup(SourceDirectory, MetadataCache, ExcludedFolderNames, NewFilesWithMarkupBag, Queue));
}
}
// Check whether the list has changed
List <FileItem> PrevFilesWithMarkup = Makefile.UObjectModuleHeaders.Where(x => !x.bUsePrecompiled).SelectMany(x => x.HeaderFiles).ToList();
List <FileItem> NextFilesWithMarkup = NewFilesWithMarkupBag.ToList();
if (NextFilesWithMarkup.Count != PrevFilesWithMarkup.Count || NextFilesWithMarkup.Intersect(PrevFilesWithMarkup).Count() != PrevFilesWithMarkup.Count)
{
ReasonNotLoaded = "UHT files changed";
return(false);
}
// If adaptive unity build is enabled, do a check to see if there are any source files that became part of the
// working set since the Makefile was created (or, source files were removed from the working set.) If anything
// changed, then we'll force a new Makefile to be created so that we have fresh unity build blobs. We always
// want to make sure that source files in the working set are excluded from those unity blobs (for fastest possible
// iteration times.)
// Check if any source files in the working set no longer belong in it
foreach (FileItem SourceFile in Makefile.WorkingSet)
{
if (!WorkingSet.Contains(SourceFile) && SourceFile.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
Log.TraceLog("{0} was part of source working set and now is not; invalidating makefile", SourceFile.AbsolutePath);
ReasonNotLoaded = string.Format("working set of source files changed");
return(false);
}
}
// Check if any source files that are eligible for being in the working set have been modified
foreach (FileItem SourceFile in Makefile.CandidatesForWorkingSet)
{
if (WorkingSet.Contains(SourceFile) && SourceFile.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
Log.TraceLog("{0} was part of source working set and now is not", SourceFile.AbsolutePath);
ReasonNotLoaded = string.Format("working set of source files changed");
return(false);
}
}
}
ReasonNotLoaded = null;
return(true);
}
/// <summary>
/// Build a list of targets
/// </summary>
/// <param name="TargetDescriptors">Target descriptors</param>
/// <param name="BuildConfiguration">Current build configuration</param>
/// <param name="WorkingSet">The source file working set</param>
/// <param name="Options">Additional options for the build</param>
/// <param name="WriteOutdatedActionsFile">Files to write the list of outdated actions to (rather than building them)</param>
/// <returns>Result from the compilation</returns>
public static void Build(List <TargetDescriptor> TargetDescriptors, BuildConfiguration BuildConfiguration, ISourceFileWorkingSet WorkingSet, BuildOptions Options, FileReference WriteOutdatedActionsFile)
{
// Create a makefile for each target
TargetMakefile[] Makefiles = new TargetMakefile[TargetDescriptors.Count];
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
Makefiles[TargetIdx] = CreateMakefile(BuildConfiguration, TargetDescriptors[TargetIdx], WorkingSet);
}
// Export the actions for each target
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
TargetDescriptor TargetDescriptor = TargetDescriptors[TargetIdx];
foreach (FileReference WriteActionFile in TargetDescriptor.WriteActionFiles)
{
Log.TraceInformation("Writing actions to {0}", WriteActionFile);
ActionGraph.ExportJson(Makefiles[TargetIdx].Actions, WriteActionFile);
}
}
// Execute the build
if ((Options & BuildOptions.SkipBuild) == 0)
{
// Make sure that none of the actions conflict with any other (producing output files differently, etc...)
ActionGraph.CheckForConflicts(Makefiles.SelectMany(x => x.Actions));
// Check we don't exceed the nominal max path length
using (Timeline.ScopeEvent("ActionGraph.CheckPathLengths"))
{
ActionGraph.CheckPathLengths(BuildConfiguration, Makefiles.SelectMany(x => x.Actions));
}
// Find all the actions to be executed
HashSet <Action>[] ActionsToExecute = new HashSet <Action> [TargetDescriptors.Count];
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
ActionsToExecute[TargetIdx] = GetActionsForTarget(BuildConfiguration, TargetDescriptors[TargetIdx], Makefiles[TargetIdx]);
}
// If there are multiple targets being built, merge the actions together
List <Action> MergedActionsToExecute;
if (TargetDescriptors.Count == 1)
{
MergedActionsToExecute = new List <Action>(ActionsToExecute[0]);
}
else
{
MergedActionsToExecute = MergeActionGraphs(TargetDescriptors, ActionsToExecute);
}
// Link all the actions together
ActionGraph.Link(MergedActionsToExecute);
// Make sure we're not modifying any engine files
if ((Options & BuildOptions.NoEngineChanges) != 0)
{
List <FileItem> EngineChanges = MergedActionsToExecute.SelectMany(x => x.ProducedItems).Where(x => x.Location.IsUnderDirectory(UnrealBuildTool.EngineDirectory)).Distinct().OrderBy(x => x.FullName).ToList();
if (EngineChanges.Count > 0)
{
StringBuilder Result = new StringBuilder("Building would modify the following engine files:\n");
foreach (FileItem EngineChange in EngineChanges)
{
Result.AppendFormat("\n{0}", EngineChange.FullName);
}
Result.Append("\n\nPlease rebuild from an IDE instead.");
Log.TraceError("{0}", Result.ToString());
throw new CompilationResultException(CompilationResult.FailedDueToEngineChange);
}
}
// Make sure the appropriate executor is selected
foreach (TargetDescriptor TargetDescriptor in TargetDescriptors)
{
UEBuildPlatform BuildPlatform = UEBuildPlatform.GetBuildPlatform(TargetDescriptor.Platform);
BuildConfiguration.bAllowXGE &= BuildPlatform.CanUseXGE();
BuildConfiguration.bAllowDistcc &= BuildPlatform.CanUseDistcc();
BuildConfiguration.bAllowSNDBS &= BuildPlatform.CanUseSNDBS();
}
// Delete produced items that are outdated.
ActionGraph.DeleteOutdatedProducedItems(MergedActionsToExecute);
// Save all the action histories now that files have been removed. We have to do this after deleting produced items to ensure that any
// items created during the build don't have the wrong command line.
ActionHistory.SaveAll();
// Create directories for the outdated produced items.
ActionGraph.CreateDirectoriesForProducedItems(MergedActionsToExecute);
// Execute the actions
if ((Options & BuildOptions.XGEExport) != 0)
{
OutputToolchainInfo(TargetDescriptors, Makefiles);
// Just export to an XML file
using (Timeline.ScopeEvent("XGE.ExportActions()"))
{
XGE.ExportActions(MergedActionsToExecute);
}
}
else if (WriteOutdatedActionsFile != null)
{
OutputToolchainInfo(TargetDescriptors, Makefiles);
// Write actions to an output file
using (Timeline.ScopeEvent("ActionGraph.WriteActions"))
{
ActionGraph.ExportJson(MergedActionsToExecute, WriteOutdatedActionsFile);
}
}
else
{
// Execute the actions
if (MergedActionsToExecute.Count == 0)
{
if (TargetDescriptors.Any(x => !x.bQuiet))
{
Log.TraceInformation((TargetDescriptors.Count == 1)? "Target is up to date" : "Targets are up to date");
}
}
else
{
if (TargetDescriptors.Any(x => !x.bQuiet))
{
Log.TraceInformation("Building {0}...", StringUtils.FormatList(TargetDescriptors.Select(x => x.Name).Distinct()));
}
OutputToolchainInfo(TargetDescriptors, Makefiles);
using (Timeline.ScopeEvent("ActionGraph.ExecuteActions()"))
{
ActionGraph.ExecuteActions(BuildConfiguration, MergedActionsToExecute);
}
}
// Run the deployment steps
foreach (TargetMakefile Makefile in Makefiles)
{
if (Makefile.bDeployAfterCompile)
{
TargetReceipt Receipt = TargetReceipt.Read(Makefile.ReceiptFile);
Log.TraceInformation("Deploying {0} {1} {2}...", Receipt.TargetName, Receipt.Platform, Receipt.Configuration);
UEBuildPlatform.GetBuildPlatform(Receipt.Platform).Deploy(Receipt);
}
}
}
}
}
/// <summary>
/// Checks if the editor is currently running and this is a hot-reload
/// </summary>
public static bool ShouldDoHotReloadFromIDE(BuildConfiguration BuildConfiguration, TargetDescriptor TargetDesc)
{
// Check if Hot-reload is disabled globally for this project
ConfigHierarchy Hierarchy = ConfigCache.ReadHierarchy(ConfigHierarchyType.Engine, DirectoryReference.FromFile(TargetDesc.ProjectFile), TargetDesc.Platform);
bool bAllowHotReloadFromIDE;
if (Hierarchy.TryGetValue("BuildConfiguration", "bAllowHotReloadFromIDE", out bAllowHotReloadFromIDE) && !bAllowHotReloadFromIDE)
{
return(false);
}
if (!BuildConfiguration.bAllowHotReloadFromIDE)
{
return(false);
}
// Check if we're using LiveCode instead
ConfigHierarchy EditorPerProjectHierarchy = ConfigCache.ReadHierarchy(ConfigHierarchyType.EditorPerProjectUserSettings, DirectoryReference.FromFile(TargetDesc.ProjectFile), TargetDesc.Platform);
bool bEnableLiveCode;
if (EditorPerProjectHierarchy.GetBool("/Script/LiveCoding.LiveCodingSettings", "bEnabled", out bEnableLiveCode) && bEnableLiveCode)
{
return(false);
}
bool bIsRunning = false;
// @todo ubtmake: Kind of cheating here to figure out if an editor target. At this point we don't have access to the actual target description, and
// this code must be able to execute before we create or load module rules DLLs so that hot reload can work with bUseUBTMakefiles
if (TargetDesc.Name.EndsWith("Editor", StringComparison.OrdinalIgnoreCase))
{
string EditorBaseFileName = "UE4Editor";
if (TargetDesc.Configuration != UnrealTargetConfiguration.Development)
{
EditorBaseFileName = String.Format("{0}-{1}-{2}", EditorBaseFileName, TargetDesc.Platform, TargetDesc.Configuration);
}
FileReference EditorLocation;
if (TargetDesc.Platform == UnrealTargetPlatform.Win64)
{
EditorLocation = FileReference.Combine(UnrealBuildTool.EngineDirectory, "Binaries", "Win64", String.Format("{0}.exe", EditorBaseFileName));
}
else if (TargetDesc.Platform == UnrealTargetPlatform.Mac)
{
EditorLocation = FileReference.Combine(UnrealBuildTool.EngineDirectory, "Binaries", "Mac", String.Format("{0}.app/Contents/MacOS/{0}", EditorBaseFileName));
}
else if (TargetDesc.Platform == UnrealTargetPlatform.Linux)
{
EditorLocation = FileReference.Combine(UnrealBuildTool.EngineDirectory, "Binaries", "Linux", EditorBaseFileName);
}
else
{
throw new BuildException("Unknown editor filename for this platform");
}
using (Timeline.ScopeEvent("Finding editor processes for hot-reload"))
{
DirectoryReference EditorRunsDir = DirectoryReference.Combine(UnrealBuildTool.EngineDirectory, "Intermediate", "EditorRuns");
if (!DirectoryReference.Exists(EditorRunsDir))
{
return(false);
}
if (BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Win64)
{
foreach (FileReference EditorInstanceFile in DirectoryReference.EnumerateFiles(EditorRunsDir))
{
int ProcessId;
if (!Int32.TryParse(EditorInstanceFile.GetFileName(), out ProcessId))
{
FileReference.Delete(EditorInstanceFile);
continue;
}
Process RunningProcess;
try
{
RunningProcess = Process.GetProcessById(ProcessId);
}
catch
{
RunningProcess = null;
}
if (RunningProcess == null)
{
FileReference.Delete(EditorInstanceFile);
continue;
}
FileReference MainModuleFile;
try
{
MainModuleFile = new FileReference(RunningProcess.MainModule.FileName);
}
catch
{
MainModuleFile = null;
}
if (!bIsRunning && EditorLocation == MainModuleFile)
{
bIsRunning = true;
}
}
}
else
{
FileInfo[] EditorRunsFiles = new DirectoryInfo(EditorRunsDir.FullName).GetFiles();
BuildHostPlatform.ProcessInfo[] Processes = BuildHostPlatform.Current.GetProcesses();
foreach (FileInfo File in EditorRunsFiles)
{
int PID;
BuildHostPlatform.ProcessInfo Proc = null;
if (!Int32.TryParse(File.Name, out PID) || (Proc = Processes.FirstOrDefault(P => P.PID == PID)) == default(BuildHostPlatform.ProcessInfo))
{
// Delete stale files (it may happen if editor crashes).
File.Delete();
continue;
}
// Don't break here to allow clean-up of other stale files.
if (!bIsRunning)
{
// Otherwise check if the path matches.
bIsRunning = new FileReference(Proc.Filename) == EditorLocation;
}
}
}
}
}
return(bIsRunning);
}
/// <summary>
/// Main entry point. Parses any global options and initializes the logging system, then invokes the appropriate command.
/// </summary>
/// <param name="ArgumentsArray">Command line arguments</param>
/// <returns>Zero on success, non-zero on error</returns>
private static int Main(string[] ArgumentsArray)
{
SingleInstanceMutex Mutex = null;
try
{
// Start capturing performance info
Timeline.Start();
// Parse the command line arguments
CommandLineArguments Arguments = new CommandLineArguments(ArgumentsArray);
// Parse the global options
GlobalOptions Options = new GlobalOptions(Arguments);
// Configure the log system
Log.OutputLevel = Options.LogOutputLevel;
Log.IncludeTimestamps = Options.bLogTimestamps;
Log.IncludeProgramNameWithSeverityPrefix = Options.bLogFromMsBuild;
// Configure the progress writer
ProgressWriter.bWriteMarkup = Options.bWriteProgressMarkup;
// Add the log writer if requested. When building a target, we'll create the writer for the default log file later.
if (Options.LogFileName != null)
{
Log.AddFileWriter("LogTraceListener", Options.LogFileName);
}
// Ensure we can resolve any external assemblies that are not in the same folder as our assembly.
AssemblyUtils.InstallAssemblyResolver(Path.GetDirectoryName(Assembly.GetEntryAssembly().GetOriginalLocation()));
// Change the working directory to be the Engine/Source folder. We are likely running from Engine/Binaries/DotNET
// This is critical to be done early so any code that relies on the current directory being Engine/Source will work.
DirectoryReference.SetCurrentDirectory(UnrealBuildTool.EngineSourceDirectory);
// Get the type of the mode to execute, using a fast-path for the build mode.
Type ModeType = typeof(BuildMode);
if (Options.Mode != null)
{
// Find all the valid modes
Dictionary <string, Type> ModeNameToType = new Dictionary <string, Type>(StringComparer.OrdinalIgnoreCase);
foreach (Type Type in Assembly.GetExecutingAssembly().GetTypes())
{
if (Type.IsClass && !Type.IsAbstract && Type.IsSubclassOf(typeof(ToolMode)))
{
ToolModeAttribute Attribute = Type.GetCustomAttribute <ToolModeAttribute>();
if (Attribute == null)
{
throw new BuildException("Class '{0}' should have a ToolModeAttribute", Type.Name);
}
ModeNameToType.Add(Attribute.Name, Type);
}
}
// Try to get the correct mode
if (!ModeNameToType.TryGetValue(Options.Mode, out ModeType))
{
Log.TraceError("No mode named '{0}'. Available modes are:\n {1}", Options.Mode, String.Join("\n ", ModeNameToType.Keys));
return(1);
}
}
// Get the options for which systems have to be initialized for this mode
ToolModeOptions ModeOptions = ModeType.GetCustomAttribute <ToolModeAttribute>().Options;
// Start prefetching the contents of the engine folder
if ((ModeOptions & ToolModeOptions.StartPrefetchingEngine) != 0)
{
using (Timeline.ScopeEvent("FileMetadataPrefetch.QueueEngineDirectory()"))
{
FileMetadataPrefetch.QueueEngineDirectory();
}
}
// Read the XML configuration files
if ((ModeOptions & ToolModeOptions.XmlConfig) != 0)
{
using (Timeline.ScopeEvent("XmlConfig.ReadConfigFiles()"))
{
string XmlConfigMutexName = SingleInstanceMutex.GetUniqueMutexForPath("UnrealBuildTool_Mutex_XmlConfig", Assembly.GetExecutingAssembly().CodeBase);
using (SingleInstanceMutex XmlConfigMutex = new SingleInstanceMutex(XmlConfigMutexName, true))
{
FileReference XmlConfigCache = Arguments.GetFileReferenceOrDefault("-XmlConfigCache=", null);
XmlConfig.ReadConfigFiles(XmlConfigCache);
}
}
}
// Acquire a lock for this branch
if ((ModeOptions & ToolModeOptions.SingleInstance) != 0 && !Options.bNoMutex)
{
using (Timeline.ScopeEvent("SingleInstanceMutex.Acquire()"))
{
string MutexName = SingleInstanceMutex.GetUniqueMutexForPath("UnrealBuildTool_Mutex", Assembly.GetExecutingAssembly().CodeBase);
Mutex = new SingleInstanceMutex(MutexName, Options.bWaitMutex);
}
}
// Register all the build platforms
if ((ModeOptions & ToolModeOptions.BuildPlatforms) != 0)
{
using (Timeline.ScopeEvent("UEBuildPlatform.RegisterPlatforms()"))
{
UEBuildPlatform.RegisterPlatforms(false);
}
}
if ((ModeOptions & ToolModeOptions.BuildPlatformsForValidation) != 0)
{
using (Timeline.ScopeEvent("UEBuildPlatform.RegisterPlatforms()"))
{
UEBuildPlatform.RegisterPlatforms(true);
}
}
// Create the appropriate handler
ToolMode Mode = (ToolMode)Activator.CreateInstance(ModeType);
// Execute the mode
int Result = Mode.Execute(Arguments);
if ((ModeOptions & ToolModeOptions.ShowExecutionTime) != 0)
{
Log.TraceInformation("Total execution time: {0:0.00} seconds", Timeline.Elapsed.TotalSeconds);
}
return(Result);
}
catch (CompilationResultException Ex)
{
// Used to return a propagate a specific exit code after an error has occurred. Does not log any message.
Log.TraceLog(ExceptionUtils.FormatExceptionDetails(Ex));
return((int)Ex.Result);
}
catch (BuildException Ex)
{
// BuildExceptions should have nicely formatted messages. We can log these directly.
Log.TraceError(ExceptionUtils.FormatException(Ex));
Log.TraceLog(ExceptionUtils.FormatExceptionDetails(Ex));
return((int)CompilationResult.OtherCompilationError);
}
catch (Exception Ex)
{
// Unhandled exception.
Log.TraceError("Unhandled exception: {0}", ExceptionUtils.FormatException(Ex));
Log.TraceLog(ExceptionUtils.FormatExceptionDetails(Ex));
return((int)CompilationResult.OtherCompilationError);
}
finally
{
// Cancel the prefetcher
using (Timeline.ScopeEvent("FileMetadataPrefetch.Stop()"))
{
FileMetadataPrefetch.Stop();
}
// Print out all the performance info
Timeline.Print(TimeSpan.FromMilliseconds(20.0), LogEventType.Log);
// Make sure we flush the logs however we exit
Trace.Close();
// Dispose of the mutex. Must be done last to ensure that another process does not startup and start trying to write to the same log file.
if (Mutex != null)
{
Mutex.Dispose();
}
}
}
