/// <summary>
/// Caches the subdirectories of this directories
/// </summary>
public void CacheDirectories()
{
if (Directories == null)
{
Dictionary <string, DirectoryItem> NewDirectories = new Dictionary <string, DirectoryItem>(DirectoryReference.Comparer);
if (Info.Exists)
{
foreach (DirectoryInfo SubDirectoryInfo in Info.EnumerateDirectories())
{
if (SubDirectoryInfo.Name.Length == 1 && SubDirectoryInfo.Name[0] == '.')
{
continue;
}
else if (SubDirectoryInfo.Name.Length == 2 && SubDirectoryInfo.Name[0] == '.' && SubDirectoryInfo.Name[1] == '.')
{
continue;
}
else
{
NewDirectories[SubDirectoryInfo.Name] = DirectoryItem.GetItemByDirectoryInfo(SubDirectoryInfo);
}
}
}
Directories = NewDirectories;
}
}
/// <summary>
/// Finds all the rules of the given type under a given directory
/// </summary>
/// <param name="Directory">Directory to search</param>
/// <param name="Type">Type of rules to return</param>
/// <returns>List of rules files of the given type</returns>
private static IReadOnlyList <FileReference> FindAllRulesFiles(DirectoryReference Directory, RulesFileType Type)
{
// Check to see if we've already cached source files for this folder
RulesFileCache Cache;
if (!RootFolderToRulesFileCache.TryGetValue(Directory, out Cache))
{
Cache = new RulesFileCache();
using (ThreadPoolWorkQueue Queue = new ThreadPoolWorkQueue())
{
DirectoryItem BaseDirectory = DirectoryItem.GetItemByDirectoryReference(Directory);
Queue.Enqueue(() => FindAllRulesFilesRecursively(BaseDirectory, Cache, Queue));
}
RootFolderToRulesFileCache[Directory] = Cache;
}
// Get the list of files of the type we're looking for
if (Type == RulesCompiler.RulesFileType.Module)
{
return(Cache.ModuleRules);
}
else if (Type == RulesCompiler.RulesFileType.Target)
{
return(Cache.TargetRules);
}
else if (Type == RulesCompiler.RulesFileType.AutomationModule)
{
return(Cache.AutomationModules);
}
else
{
throw new BuildException("Unhandled rules type: {0}", Type);
}
}
/// <summary>
/// Prefetch multiple directories in parallel
/// </summary>
/// <param name="Directories">The directories to cache</param>
private static void PrefetchRulesFiles(IEnumerable <DirectoryReference> Directories)
{
ThreadPoolWorkQueue Queue = null;
try
{
foreach (DirectoryReference Directory in Directories)
{
if (!RootFolderToRulesFileCache.ContainsKey(Directory))
{
RulesFileCache Cache = new RulesFileCache();
RootFolderToRulesFileCache[Directory] = Cache;
if (Queue == null)
{
Queue = new ThreadPoolWorkQueue();
}
DirectoryItem DirectoryItem = DirectoryItem.GetItemByDirectoryReference(Directory);
Queue.Enqueue(() => FindAllRulesFilesRecursively(DirectoryItem, Cache, Queue));
}
}
}
finally
{
if (Queue != null)
{
Queue.Dispose();
Queue = null;
}
}
}
/// <summary>
/// Scans an arbitrary directory tree
/// </summary>
/// <param name="Directory">Root of the directory tree</param>
static void ScanDirectoryTree(DirectoryItem Directory)
{
foreach (DirectoryItem SubDirectory in Directory.EnumerateDirectories())
{
Enqueue(() => ScanDirectoryTree(SubDirectory));
}
Directory.CacheFiles();
}
/// <summary>
/// Read a file item as a DirectoryItem and name. This is slower than reading it directly, but results in a significantly smaller archive
/// where most files are in the same directories.
/// </summary>
/// <param name="Reader">Archive to read from</param>
/// <returns>FileItem read from the archive</returns>
static FileItem ReadCompactFileItemData(this BinaryArchiveReader Reader)
{
DirectoryItem Directory = Reader.ReadDirectoryItem();
string Name = Reader.ReadString();
FileItem FileItem = FileItem.GetItemByFileReference(FileReference.Combine(Directory.Location, Name));
FileItem.UpdateCachedDirectory(Directory);
return(FileItem);
}
/// <summary>
/// Scans a project directory, adding tasks for subdirectories
/// </summary>
/// <param name="ProjectDirectory">The project directory to search</param>
static void ScanProjectDirectory(DirectoryItem ProjectDirectory)
{
DirectoryItem ProjectPluginsDirectory = DirectoryItem.Combine(ProjectDirectory, "Plugins");
Enqueue(() => ScanPluginFolder(ProjectPluginsDirectory));
DirectoryItem ProjectSourceDirectory = DirectoryItem.Combine(ProjectDirectory, "Source");
Enqueue(() => ScanDirectoryTree(ProjectSourceDirectory));
}
/// <summary>
/// Enqueue a project directory for prefetching
/// </summary>
/// <param name="ProjectDirectory">The project directory to prefetch</param>
public static void QueueProjectDirectory(DirectoryReference ProjectDirectory)
{
lock (QueuedDirectories)
{
if (QueuedDirectories.Add(ProjectDirectory))
{
Enqueue(() => ScanProjectDirectory(DirectoryItem.GetItemByDirectoryReference(ProjectDirectory)));
}
}
}
/// <summary>
/// Enqueue a directory tree for prefetching
/// </summary>
/// <param name="Directory">Directory to start searching from</param>
public static void QueueDirectoryTree(DirectoryReference Directory)
{
lock (QueuedDirectories)
{
if (QueuedDirectories.Add(Directory))
{
Enqueue(() => ScanDirectoryTree(DirectoryItem.GetItemByDirectoryReference(Directory)));
}
}
}
/// <summary>
/// Scans a project directory, adding tasks for subdirectories
/// </summary>
/// <param name="ProjectDirectory">The project directory to search</param>
static void ScanProjectDirectory(DirectoryItem ProjectDirectory)
{
foreach (DirectoryReference ExtensionDir in UnrealBuildTool.GetExtensionDirs(UnrealBuildTool.EngineDirectory))
{
DirectoryItem BaseDirectory = DirectoryItem.GetItemByDirectoryReference(ExtensionDir);
BaseDirectory.CacheDirectories();
DirectoryItem BasePluginsDirectory = DirectoryItem.Combine(ProjectDirectory, "Plugins");
Enqueue(() => ScanPluginFolder(BasePluginsDirectory));
DirectoryItem BaseSourceDirectory = DirectoryItem.Combine(ProjectDirectory, "Source");
Enqueue(() => ScanDirectoryTree(BaseSourceDirectory));
}
}
/// <summary>
/// Scans a plugin parent directory, adding tasks for subdirectories
/// </summary>
/// <param name="Directory">The directory which may contain plugin directories</param>
static void ScanPluginFolder(DirectoryItem Directory)
{
foreach (DirectoryItem SubDirectory in Directory.EnumerateDirectories())
{
if (SubDirectory.EnumerateFiles().Any(x => x.HasExtension(".uplugin")))
{
Enqueue(() => ScanDirectoryTree(DirectoryItem.Combine(SubDirectory, "Source")));
}
else
{
Enqueue(() => ScanPluginFolder(SubDirectory));
}
}
}
/// <summary>
/// Finds or creates a directory item from its location
/// </summary>
/// <param name="Location">Path to the directory</param>
/// <returns>The directory item for this location</returns>
public static DirectoryItem GetItemByDirectoryReference(DirectoryReference Location)
{
DirectoryItem Result;
if (!LocationToItem.TryGetValue(Location, out Result))
{
DirectoryItem NewItem = new DirectoryItem(Location, new DirectoryInfo(Location.FullName));
if (LocationToItem.TryAdd(Location, NewItem))
{
Result = NewItem;
}
else
{
Result = LocationToItem[Location];
}
}
return(Result);
}
/// <summary>
/// Attempts to get a sub-directory by name
/// </summary>
/// <param name="Name">Name of the directory</param>
/// <param name="OutDirectory">If successful receives the matching directory item with this name</param>
/// <returns>True if the file exists, false otherwise</returns>
public bool TryGetDirectory(string Name, out DirectoryItem OutDirectory)
{
if (Name.Length > 0 && Name[0] == '.')
{
if (Name.Length == 1)
{
OutDirectory = this;
return(true);
}
else if (Name.Length == 2 && Name[1] == '.')
{
OutDirectory = GetParentDirectoryItem();
return(OutDirectory != null);
}
}
CacheDirectories();
return(Directories.TryGetValue(Name, out OutDirectory));
}
/// <summary>
/// Finds all the source files under a directory that contain reflection markup
/// </summary>
/// <param name="Directory">The directory to search</param>
/// <param name="MetadataCache">Cache of source file metadata</param>
/// <param name="ExcludedFolderNames">Set of folder names to ignore when recursing the directory tree</param>
/// <param name="FilesWithMarkup">Receives the set of files which contain reflection markup</param>
/// <param name="Queue">Queue to add sub-tasks to</param>
static void FindFilesWithMarkup(DirectoryItem Directory, SourceFileMetadataCache MetadataCache, ReadOnlyHashSet <string> ExcludedFolderNames, ConcurrentBag <FileItem> FilesWithMarkup, ThreadPoolWorkQueue Queue)
{
// Search through all the subfolders
foreach (DirectoryItem SubDirectory in Directory.EnumerateDirectories())
{
if (!ExcludedFolderNames.Contains(SubDirectory.Name))
{
Queue.Enqueue(() => FindFilesWithMarkup(SubDirectory, MetadataCache, ExcludedFolderNames, FilesWithMarkup, Queue));
}
}
// Check for all the headers in this folder
foreach (FileItem File in Directory.EnumerateFiles())
{
if (File.HasExtension(".h") && MetadataCache.ContainsReflectionMarkup(File))
{
FilesWithMarkup.Add(File);
}
}
}
/// <summary>
/// Search through a directory tree for any rules files
/// </summary>
/// <param name="Directory">The root directory to search from</param>
/// <param name="Cache">Receives all the discovered rules files</param>
/// <param name="Queue">Queue for adding additional tasks to</param>
private static void FindAllRulesFilesRecursively(DirectoryItem Directory, RulesFileCache Cache, ThreadPoolWorkQueue Queue)
{
// Scan all the files in this directory
bool bSearchSubFolders = true;
foreach (FileItem File in Directory.EnumerateFiles())
{
if (File.HasExtension(".build.cs"))
{
lock (Cache.ModuleRules)
{
Cache.ModuleRules.Add(File.Location);
}
bSearchSubFolders = false;
}
else if (File.HasExtension(".target.cs"))
{
lock (Cache.TargetRules)
{
Cache.TargetRules.Add(File.Location);
}
}
else if (File.HasExtension(".automation.csproj"))
{
lock (Cache.AutomationModules)
{
Cache.AutomationModules.Add(File.Location);
}
bSearchSubFolders = false;
}
}
// If we didn't find anything to stop the search, search all the subdirectories too
if (bSearchSubFolders)
{
foreach (DirectoryItem SubDirectory in Directory.EnumerateDirectories())
{
Queue.Enqueue(() => FindAllRulesFilesRecursively(SubDirectory, Cache, Queue));
}
}
}
/// <summary>
/// Scans the engine directory, adding tasks for subdirectories
/// </summary>
static void ScanEngineDirectory()
{
DirectoryItem EngineDirectory = DirectoryItem.GetItemByDirectoryReference(UnrealBuildTool.EngineDirectory);
EngineDirectory.CacheDirectories();
DirectoryItem EnginePluginsDirectory = DirectoryItem.Combine(EngineDirectory, "Plugins");
Enqueue(() => ScanPluginFolder(EnginePluginsDirectory));
DirectoryItem EngineRuntimeDirectory = DirectoryItem.GetItemByDirectoryReference(UnrealBuildTool.EngineSourceRuntimeDirectory);
Enqueue(() => ScanDirectoryTree(EngineRuntimeDirectory));
DirectoryItem EngineDeveloperDirectory = DirectoryItem.GetItemByDirectoryReference(UnrealBuildTool.EngineSourceDeveloperDirectory);
Enqueue(() => ScanDirectoryTree(EngineDeveloperDirectory));
DirectoryItem EngineEditorDirectory = DirectoryItem.GetItemByDirectoryReference(UnrealBuildTool.EngineSourceEditorDirectory);
Enqueue(() => ScanDirectoryTree(EngineEditorDirectory));
}
/// <summary>
/// Scans the engine directory, adding tasks for subdirectories
/// </summary>
static void ScanEngineDirectory()
{
foreach (DirectoryReference ExtensionDir in UnrealBuildTool.GetExtensionDirs(UnrealBuildTool.EngineDirectory))
{
DirectoryItem BaseDirectory = DirectoryItem.GetItemByDirectoryReference(ExtensionDir);
BaseDirectory.CacheDirectories();
DirectoryItem BasePluginsDirectory = DirectoryItem.Combine(BaseDirectory, "Plugins");
Enqueue(() => ScanPluginFolder(BasePluginsDirectory));
DirectoryItem BaseSourceDirectory = DirectoryItem.Combine(BaseDirectory, "Source");
BaseSourceDirectory.CacheDirectories();
DirectoryItem BaseSourceRuntimeDirectory = DirectoryItem.Combine(BaseSourceDirectory, "Runtime");
Enqueue(() => ScanDirectoryTree(BaseSourceRuntimeDirectory));
DirectoryItem BaseSourceDeveloperDirectory = DirectoryItem.Combine(BaseSourceDirectory, "Developer");
Enqueue(() => ScanDirectoryTree(BaseSourceDeveloperDirectory));
DirectoryItem BaseSourceEditorDirectory = DirectoryItem.Combine(BaseSourceDirectory, "Editor");
Enqueue(() => ScanDirectoryTree(BaseSourceEditorDirectory));
}
}
/// <summary>
/// Find paths to all the plugins under a given parent directory (recursively)
/// </summary>
/// <param name="ParentDirectory">Parent directory to look in. Plugins will be found in any *subfolders* of this directory.</param>
/// <param name="FileNames">List of filenames. Will have all the discovered .uplugin files appended to it.</param>
/// <param name="Queue">Queue for tasks to be executed</param>
static void EnumeratePluginsInternal(DirectoryItem ParentDirectory, List <FileReference> FileNames, ThreadPoolWorkQueue Queue)
{
foreach (DirectoryItem ChildDirectory in ParentDirectory.EnumerateDirectories())
{
bool bSearchSubDirectories = true;
foreach (FileItem PluginFile in ChildDirectory.EnumerateFiles())
{
if (PluginFile.HasExtension(".uplugin"))
{
lock (FileNames)
{
FileNames.Add(PluginFile.Location);
}
bSearchSubDirectories = false;
}
}
if (bSearchSubDirectories)
{
Queue.Enqueue(() => EnumeratePluginsInternal(ChildDirectory, FileNames, Queue));
}
}
}
/// <summary>
/// Determines if a directory, or any subdirectory of it, contains new source files
/// </summary>
/// <param name="Directory">Directory to search through</param>
/// <param name="ExcludedFolderNames">Set of directory names to exclude</param>
/// <returns>True if the directory contains any source files</returns>
static bool ContainsSourceFiles(DirectoryItem Directory, ReadOnlyHashSet <string> ExcludedFolderNames)
{
// Check this directory isn't ignored
if (!ExcludedFolderNames.Contains(Directory.Name))
{
// Check for any source files in this actual directory
FileItem[] SourceFiles = UEBuildModuleCPP.GetSourceFiles(Directory);
if (SourceFiles.Length > 0)
{
return(true);
}
// Check for any source files in a subdirectory
foreach (DirectoryItem SubDirectory in Directory.EnumerateDirectories())
{
if (ContainsSourceFiles(SubDirectory, ExcludedFolderNames))
{
return(true);
}
}
}
return(false);
}
/// <summary>
/// Find paths to all the plugins under a given parent directory (recursively)
/// </summary>
/// <param name="ParentDirectory">Parent directory to look in. Plugins will be found in any *subfolders* of this directory.</param>
public static IEnumerable <FileReference> EnumeratePlugins(DirectoryReference ParentDirectory)
{
List <FileReference> FileNames;
if (!PluginFileCache.TryGetValue(ParentDirectory, out FileNames))
{
FileNames = new List <FileReference>();
DirectoryItem ParentDirectoryItem = DirectoryItem.GetItemByDirectoryReference(ParentDirectory);
if (ParentDirectoryItem.Exists)
{
using (ThreadPoolWorkQueue Queue = new ThreadPoolWorkQueue())
{
EnumeratePluginsInternal(ParentDirectoryItem, FileNames, Queue);
}
}
// Sort the filenames to ensure that the plugin order is deterministic; otherwise response files will change with each build.
FileNames = FileNames.OrderBy(x => x.FullName, StringComparer.OrdinalIgnoreCase).ToList();
PluginFileCache.Add(ParentDirectory, FileNames);
}
return(FileNames);
}
static public bool DirectoryExists(DirectoryReference Directory)
{
return(DirectoryItem.GetItemByDirectoryReference(Directory).Exists);
}
static public IEnumerable <DirectoryReference> EnumerateDirectories(DirectoryReference Directory)
{
return(DirectoryItem.GetItemByDirectoryReference(Directory).EnumerateDirectories().Select(x => x.Location));
}
static public void InvalidateCachedDirectory(DirectoryReference Directory)
{
DirectoryItem.GetItemByDirectoryReference(Directory).ResetCachedInfo();
}
/// <summary>
/// Checks if the makefile is valid for the current set of source files. This is done separately to the Load() method to allow pre-build steps to modify source files.
/// </summary>
/// <param name="Makefile">The makefile that has been loaded</param>
/// <param name="ProjectFile">Path to the project file</param>
/// <param name="Platform">The platform being built</param>
/// <param name="WorkingSet">The current working set of source files</param>
/// <param name="ReasonNotLoaded">If the makefile is not valid, is set to a message describing why</param>
/// <returns>True if the makefile is valid, false otherwise</returns>
public static bool IsValidForSourceFiles(TargetMakefile Makefile, FileReference ProjectFile, UnrealTargetPlatform Platform, ISourceFileWorkingSet WorkingSet, out string ReasonNotLoaded)
{
using (Timeline.ScopeEvent("TargetMakefile.IsValidForSourceFiles()"))
{
// Get the list of excluded folder names for this platform
ReadOnlyHashSet <string> ExcludedFolderNames = UEBuildPlatform.GetBuildPlatform(Platform).GetExcludedFolderNames();
// Check if any source files have been added or removed
foreach (KeyValuePair <DirectoryItem, FileItem[]> Pair in Makefile.DirectoryToSourceFiles)
{
DirectoryItem InputDirectory = Pair.Key;
if (!InputDirectory.Exists || InputDirectory.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
FileItem[] SourceFiles = UEBuildModuleCPP.GetSourceFiles(InputDirectory);
if (SourceFiles.Length < Pair.Value.Length)
{
ReasonNotLoaded = "source file removed";
return(false);
}
else if (SourceFiles.Length > Pair.Value.Length)
{
ReasonNotLoaded = "source file added";
return(false);
}
else if (SourceFiles.Intersect(Pair.Value).Count() != SourceFiles.Length)
{
ReasonNotLoaded = "source file modified";
return(false);
}
foreach (DirectoryItem Directory in InputDirectory.EnumerateDirectories())
{
if (!Makefile.DirectoryToSourceFiles.ContainsKey(Directory) && ContainsSourceFiles(Directory, ExcludedFolderNames))
{
ReasonNotLoaded = "directory added";
return(false);
}
}
}
}
// Check if any external dependencies have changed. These comparisons are done against the makefile creation time.
foreach (FileItem ExternalDependency in Makefile.ExternalDependencies)
{
if (!ExternalDependency.Exists)
{
Log.TraceLog("{0} has been deleted since makefile was built.", ExternalDependency.Location);
ReasonNotLoaded = string.Format("{0} deleted", ExternalDependency.Location.GetFileName());
return(false);
}
if (ExternalDependency.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
Log.TraceLog("{0} has been modified since makefile was built.", ExternalDependency.Location);
ReasonNotLoaded = string.Format("{0} modified", ExternalDependency.Location.GetFileName());
return(false);
}
}
// Check if any internal dependencies has changed. These comparisons are done against the makefile modified time.
foreach (FileItem InternalDependency in Makefile.InternalDependencies)
{
if (!InternalDependency.Exists)
{
Log.TraceLog("{0} has been deleted since makefile was written.", InternalDependency.Location);
ReasonNotLoaded = string.Format("{0} deleted", InternalDependency.Location.GetFileName());
return(false);
}
if (InternalDependency.LastWriteTimeUtc > Makefile.ModifiedTimeUtc)
{
Log.TraceLog("{0} has been modified since makefile was written.", InternalDependency.Location);
ReasonNotLoaded = string.Format("{0} modified", InternalDependency.Location.GetFileName());
return(false);
}
}
// Check that no new plugins have been added
foreach (FileReference PluginFile in Plugins.EnumeratePlugins(ProjectFile))
{
FileItem PluginFileItem = FileItem.GetItemByFileReference(PluginFile);
if (!Makefile.PluginFiles.Contains(PluginFileItem))
{
Log.TraceLog("{0} has been added", PluginFile.GetFileName());
ReasonNotLoaded = string.Format("{0} has been added", PluginFile.GetFileName());
return(false);
}
}
// Load the metadata cache
SourceFileMetadataCache MetadataCache = SourceFileMetadataCache.CreateHierarchy(ProjectFile);
// Find the set of files that contain reflection markup
ConcurrentBag <FileItem> NewFilesWithMarkupBag = new ConcurrentBag <FileItem>();
using (ThreadPoolWorkQueue Queue = new ThreadPoolWorkQueue())
{
foreach (DirectoryItem SourceDirectory in Makefile.SourceDirectories)
{
Queue.Enqueue(() => FindFilesWithMarkup(SourceDirectory, MetadataCache, ExcludedFolderNames, NewFilesWithMarkupBag, Queue));
}
}
// Check whether the list has changed
List <FileItem> PrevFilesWithMarkup = Makefile.UObjectModuleHeaders.Where(x => !x.bUsePrecompiled).SelectMany(x => x.HeaderFiles).ToList();
List <FileItem> NextFilesWithMarkup = NewFilesWithMarkupBag.ToList();
if (NextFilesWithMarkup.Count != PrevFilesWithMarkup.Count || NextFilesWithMarkup.Intersect(PrevFilesWithMarkup).Count() != PrevFilesWithMarkup.Count)
{
ReasonNotLoaded = "UHT files changed";
return(false);
}
// If adaptive unity build is enabled, do a check to see if there are any source files that became part of the
// working set since the Makefile was created (or, source files were removed from the working set.) If anything
// changed, then we'll force a new Makefile to be created so that we have fresh unity build blobs. We always
// want to make sure that source files in the working set are excluded from those unity blobs (for fastest possible
// iteration times.)
// Check if any source files in the working set no longer belong in it
foreach (FileItem SourceFile in Makefile.WorkingSet)
{
if (!WorkingSet.Contains(SourceFile) && SourceFile.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
Log.TraceLog("{0} was part of source working set and now is not; invalidating makefile", SourceFile.AbsolutePath);
ReasonNotLoaded = string.Format("working set of source files changed");
return(false);
}
}
// Check if any source files that are eligible for being in the working set have been modified
foreach (FileItem SourceFile in Makefile.CandidatesForWorkingSet)
{
if (WorkingSet.Contains(SourceFile) && SourceFile.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
Log.TraceLog("{0} was part of source working set and now is not", SourceFile.AbsolutePath);
ReasonNotLoaded = string.Format("working set of source files changed");
return(false);
}
}
}
ReasonNotLoaded = null;
return(true);
}
/// <inheritdoc/>
public void AddSourceDir(DirectoryItem SourceDir)
{
SourceDirectories.Add(SourceDir);
}
/// <inheritdoc/>
public void AddSourceFiles(DirectoryItem SourceDir, FileItem[] SourceFiles)
{
DirectoryToSourceFiles[SourceDir] = SourceFiles;
}
/// <summary>
/// Write a directory item to a binary archive
/// </summary>
/// <param name="Writer">Writer to serialize data to</param>
/// <param name="DirectoryItem">Directory item to write</param>
public static void WriteDirectoryItem(this BinaryArchiveWriter Writer, DirectoryItem DirectoryItem)
{
Writer.WriteObjectReference <DirectoryItem>(DirectoryItem, () => Writer.WriteDirectoryReference(DirectoryItem.Location));
}
/// <summary>
/// Read a directory item from a binary archive
/// </summary>
/// <param name="Reader">Reader to serialize data from</param>
/// <returns>Instance of the serialized directory item</returns>
public static DirectoryItem ReadDirectoryItem(this BinaryArchiveReader Reader)
{
return(Reader.ReadObjectReference <DirectoryItem>(() => DirectoryItem.GetItemByDirectoryReference(Reader.ReadDirectoryReference())));
}
/// <summary>
/// Checks if the makefile is valid for the current set of source files. This is done separately to the Load() method to allow pre-build steps to modify source files.
/// </summary>
/// <param name="Makefile">The makefile that has been loaded</param>
/// <param name="ProjectFile">Path to the project file</param>
/// <param name="WorkingSet">The current working set of source files</param>
/// <param name="ReasonNotLoaded">If the makefile is not valid, is set to a message describing why</param>
/// <returns>True if the makefile is valid, false otherwise</returns>
public static bool IsValidForSourceFiles(TargetMakefile Makefile, FileReference ProjectFile, ISourceFileWorkingSet WorkingSet, out string ReasonNotLoaded)
{
using (Timeline.ScopeEvent("TargetMakefile.IsValidForSourceFiles()"))
{
// Check if any source files have been added or removed
foreach (KeyValuePair <DirectoryItem, FileItem[]> Pair in Makefile.DirectoryToSourceFiles)
{
DirectoryItem InputDirectory = Pair.Key;
if (!InputDirectory.Exists || InputDirectory.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
FileItem[] SourceFiles = UEBuildModuleCPP.GetSourceFiles(InputDirectory);
if (SourceFiles.Length < Pair.Value.Length)
{
ReasonNotLoaded = "source file removed";
return(false);
}
else if (SourceFiles.Length > Pair.Value.Length)
{
ReasonNotLoaded = "source file added";
return(false);
}
else if (SourceFiles.Intersect(Pair.Value).Count() != SourceFiles.Length)
{
ReasonNotLoaded = "source file modified";
return(false);
}
}
}
// Check if any of the additional dependencies has changed
foreach (FileItem AdditionalDependency in Makefile.AdditionalDependencies)
{
if (!AdditionalDependency.Exists)
{
Log.TraceLog("{0} has been deleted since makefile was built.", AdditionalDependency.Location);
ReasonNotLoaded = string.Format("{0} deleted", AdditionalDependency.Location.GetFileName());
return(false);
}
if (AdditionalDependency.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
Log.TraceLog("{0} has been modified since makefile was built.", AdditionalDependency.Location);
ReasonNotLoaded = string.Format("{0} modified", AdditionalDependency.Location.GetFileName());
return(false);
}
}
// Check that no new plugins have been added
foreach (FileReference PluginFile in Plugins.EnumeratePlugins(ProjectFile))
{
FileItem PluginFileItem = FileItem.GetItemByFileReference(PluginFile);
if (!Makefile.PluginFiles.Contains(PluginFileItem))
{
Log.TraceLog("{0} has been added", PluginFile.GetFileName());
ReasonNotLoaded = string.Format("{0} has been added", PluginFile.GetFileName());
return(false);
}
}
// We do a check to see if any modules' headers have changed which have
// acquired or lost UHT types. If so, which should be rare,
// we'll just invalidate the entire makefile and force it to be rebuilt.
// Get all H files in processed modules newer than the makefile itself
HashSet <FileItem> HFilesNewerThanMakefile = new HashSet <FileItem>();
foreach (UHTModuleHeaderInfo ModuleHeaderInfo in Makefile.UObjectModuleHeaders)
{
foreach (FileItem HeaderFile in ModuleHeaderInfo.SourceFolder.EnumerateFiles())
{
if (HeaderFile.HasExtension(".h") && HeaderFile.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
HFilesNewerThanMakefile.Add(HeaderFile);
}
}
}
// Get all H files in all modules processed in the last makefile build
HashSet <FileItem> AllUHTHeaders = new HashSet <FileItem>(Makefile.UObjectModuleHeaders.SelectMany(x => x.HeaderFiles));
// Check whether any headers have been deleted. If they have, we need to regenerate the makefile since the module might now be empty. If we don't,
// and the file has been moved to a different module, we may include stale generated headers.
foreach (FileItem HeaderFile in AllUHTHeaders)
{
if (!HeaderFile.Exists)
{
Log.TraceLog("File processed by UHT was deleted ({0}); invalidating makefile", HeaderFile);
ReasonNotLoaded = string.Format("UHT file was deleted");
return(false);
}
}
// Makefile is invalid if:
// * There are any newer files which contain no UHT data, but were previously in the makefile
// * There are any newer files contain data which needs processing by UHT, but weren't not previously in the makefile
SourceFileMetadataCache MetadataCache = SourceFileMetadataCache.CreateHierarchy(ProjectFile);
foreach (FileItem HeaderFile in HFilesNewerThanMakefile)
{
bool bContainsUHTData = MetadataCache.ContainsReflectionMarkup(HeaderFile);
bool bWasProcessed = AllUHTHeaders.Contains(HeaderFile);
if (bContainsUHTData != bWasProcessed)
{
Log.TraceLog("{0} {1} contain UHT types and now {2} , ignoring it", HeaderFile, bWasProcessed ? "used to" : "didn't", bWasProcessed ? "doesn't" : "does");
ReasonNotLoaded = string.Format("new files with reflected types");
return(false);
}
}
// If adaptive unity build is enabled, do a check to see if there are any source files that became part of the
// working set since the Makefile was created (or, source files were removed from the working set.) If anything
// changed, then we'll force a new Makefile to be created so that we have fresh unity build blobs. We always
// want to make sure that source files in the working set are excluded from those unity blobs (for fastest possible
// iteration times.)
// Check if any source files in the working set no longer belong in it
foreach (FileItem SourceFile in Makefile.WorkingSet)
{
if (!WorkingSet.Contains(SourceFile) && SourceFile.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
Log.TraceLog("{0} was part of source working set and now is not; invalidating makefile", SourceFile.AbsolutePath);
ReasonNotLoaded = string.Format("working set of source files changed");
return(false);
}
}
// Check if any source files that are eligible for being in the working set have been modified
foreach (FileItem SourceFile in Makefile.CandidatesForWorkingSet)
{
if (WorkingSet.Contains(SourceFile) && SourceFile.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
Log.TraceLog("{0} was part of source working set and now is not", SourceFile.AbsolutePath);
ReasonNotLoaded = string.Format("working set of source files changed");
return(false);
}
}
}
ReasonNotLoaded = null;
return(true);
}
/// <summary>
/// Gets a new directory item by combining the existing directory item with the given path fragments
/// </summary>
/// <param name="BaseDirectory">Base directory to append path fragments to</param>
/// <param name="Fragments">The path fragments to append</param>
/// <returns>Directory item corresponding to the combined path</returns>
public static DirectoryItem Combine(DirectoryItem BaseDirectory, params string[] Fragments)
{
return(DirectoryItem.GetItemByDirectoryReference(DirectoryReference.Combine(BaseDirectory.Location, Fragments)));
}
/// <summary>
/// Creates the makefile for a target. If an existing, valid makefile already exists on disk, loads that instead.
/// </summary>
/// <param name="BuildConfiguration">The build configuration</param>
/// <param name="TargetDescriptor">Target being built</param>
/// <param name="WorkingSet">Set of source files which are part of the working set</param>
/// <returns>Makefile for the given target</returns>
static TargetMakefile CreateMakefile(BuildConfiguration BuildConfiguration, TargetDescriptor TargetDescriptor, ISourceFileWorkingSet WorkingSet)
{
// Get the path to the makefile for this target
FileReference MakefileLocation = null;
if (BuildConfiguration.bUseUBTMakefiles && TargetDescriptor.SpecificFilesToCompile.Count == 0)
{
MakefileLocation = TargetMakefile.GetLocation(TargetDescriptor.ProjectFile, TargetDescriptor.Name, TargetDescriptor.Platform, TargetDescriptor.Architecture, TargetDescriptor.Configuration);
}
// Try to load an existing makefile
TargetMakefile Makefile = null;
if (MakefileLocation != null)
{
using (Timeline.ScopeEvent("TargetMakefile.Load()"))
{
string ReasonNotLoaded;
Makefile = TargetMakefile.Load(MakefileLocation, TargetDescriptor.ProjectFile, TargetDescriptor.Platform, TargetDescriptor.AdditionalArguments.GetRawArray(), out ReasonNotLoaded);
if (Makefile == null)
{
Log.TraceInformation("Creating makefile for {0} ({1})", TargetDescriptor.Name, ReasonNotLoaded);
}
}
}
// If we have a makefile, execute the pre-build steps and check it's still valid
bool bHasRunPreBuildScripts = false;
if (Makefile != null)
{
// Execute the scripts. We have to invalidate all cached file info after doing so, because we don't know what may have changed.
if (Makefile.PreBuildScripts.Length > 0)
{
Utils.ExecuteCustomBuildSteps(Makefile.PreBuildScripts);
DirectoryItem.ResetAllCachedInfo_SLOW();
}
// Don't run the pre-build steps again, even if we invalidate the makefile.
bHasRunPreBuildScripts = true;
// Check that the makefile is still valid
string Reason;
if (!TargetMakefile.IsValidForSourceFiles(Makefile, TargetDescriptor.ProjectFile, TargetDescriptor.Platform, WorkingSet, out Reason))
{
Log.TraceInformation("Invalidating makefile for {0} ({1})", TargetDescriptor.Name, Reason);
Makefile = null;
}
}
// If we couldn't load a makefile, create a new one
if (Makefile == null)
{
// Create the target
UEBuildTarget Target;
using (Timeline.ScopeEvent("UEBuildTarget.Create()"))
{
Target = UEBuildTarget.Create(TargetDescriptor, BuildConfiguration.bSkipRulesCompile, BuildConfiguration.bUsePrecompiled);
}
// Create the pre-build scripts
FileReference[] PreBuildScripts = Target.CreatePreBuildScripts();
// Execute the pre-build scripts
if (!bHasRunPreBuildScripts)
{
Utils.ExecuteCustomBuildSteps(PreBuildScripts);
bHasRunPreBuildScripts = true;
}
// Build the target
using (Timeline.ScopeEvent("UEBuildTarget.Build()"))
{
const bool bIsAssemblingBuild = true;
Makefile = Target.Build(BuildConfiguration, WorkingSet, bIsAssemblingBuild, TargetDescriptor.SpecificFilesToCompile);
}
// Save the pre-build scripts onto the makefile
Makefile.PreBuildScripts = PreBuildScripts;
// Save the additional command line arguments
Makefile.AdditionalArguments = TargetDescriptor.AdditionalArguments.GetRawArray();
// Save the environment variables
foreach (System.Collections.DictionaryEntry EnvironmentVariable in Environment.GetEnvironmentVariables())
{
Makefile.EnvironmentVariables.Add(Tuple.Create((string)EnvironmentVariable.Key, (string)EnvironmentVariable.Value));
}
// Save the makefile for next time
if (MakefileLocation != null)
{
using (Timeline.ScopeEvent("TargetMakefile.Save()"))
{
Makefile.Save(MakefileLocation);
}
}
}
else
{
// Restore the environment variables
foreach (Tuple <string, string> EnvironmentVariable in Makefile.EnvironmentVariables)
{
Environment.SetEnvironmentVariable(EnvironmentVariable.Item1, EnvironmentVariable.Item2);
}
// If the target needs UHT to be run, we'll go ahead and do that now
if (Makefile.UObjectModules.Count > 0)
{
const bool bIsGatheringBuild = false;
const bool bIsAssemblingBuild = true;
FileReference ModuleInfoFileName = FileReference.Combine(Makefile.ProjectIntermediateDirectory, TargetDescriptor.Name + ".uhtmanifest");
ExternalExecution.ExecuteHeaderToolIfNecessary(BuildConfiguration, TargetDescriptor.ProjectFile, TargetDescriptor.Name, Makefile.TargetType, Makefile.bHasProjectScriptPlugin, UObjectModules: Makefile.UObjectModules, ModuleInfoFileName: ModuleInfoFileName, bIsGatheringBuild: bIsGatheringBuild, bIsAssemblingBuild: bIsAssemblingBuild, WorkingSet: WorkingSet);
}
}
return(Makefile);
}
