public static List <FileItem> FindAndCacheAllIncludedFiles(UEBuildTarget Target, FileItem SourceFile, IUEBuildPlatform BuildPlatform, CPPIncludeInfo CPPIncludeInfo, bool bOnlyCachedDependencies)
{
var Result = new List <FileItem>();
if (CPPIncludeInfo.IncludeFileSearchDictionary == null)
{
CPPIncludeInfo.IncludeFileSearchDictionary = new Dictionary <string, FileItem>();
}
bool bUseFlatCPPIncludeDependencyCache =
(BuildConfiguration.bUseExperimentalFastDependencyScan &&
(!BuildConfiguration.bUseExperimentalFastBuildIteration || UnrealBuildTool.IsAssemblingBuild));
if (bUseFlatCPPIncludeDependencyCache && bOnlyCachedDependencies)
{
var Dependencies = FlatCPPIncludeDependencyCache[Target].GetDependenciesForFile(SourceFile.AbsolutePath);
if (Dependencies != null)
{
foreach (string Dependency in Dependencies)
{
Result.Add(FileItem.GetItemByFullPath(Dependency)); // @todo fastubt: Make sure this is as fast as possible (convert to FileItem)
}
}
else
{
// Nothing cached for this file! It is new to us. This is the expected flow when our CPPIncludeDepencencyCache is missing.
}
}
else
{
// @todo fastubt: HeaderParser.h is missing from the include set for Module.UnrealHeaderTool.cpp (failed to find include using: FileItem DirectIncludeResolvedFile = CPPEnvironment.FindIncludedFile(DirectInclude.IncludeName, !BuildConfiguration.bCheckExternalHeadersForModification, IncludePathsToSearch, IncludeFileSearchDictionary );)
var IncludedFileList = new IncludedFilesSet();
CPPEnvironment.FindAndCacheAllIncludedFiles(Target, SourceFile, BuildPlatform, CPPIncludeInfo, ref IncludedFileList, bOnlyCachedDependencies: bOnlyCachedDependencies);
foreach (FileItem IncludedFile in IncludedFileList)
{
Result.Add(IncludedFile);
}
// Update cache
if (bUseFlatCPPIncludeDependencyCache && !bOnlyCachedDependencies)
{
var Dependencies = new List <string>();
foreach (var IncludedFile in Result)
{
Dependencies.Add(IncludedFile.AbsolutePath);
}
string PCHName = SourceFile.PrecompiledHeaderIncludeFilename;
FlatCPPIncludeDependencyCache[Target].SetDependenciesForFile(SourceFile.AbsolutePath, PCHName, Dependencies);
}
}
return(Result);
}
public FileItem FixDependencies(LinkEnvironment LinkEnvironment, FileItem Executable)
{
if (!LinkEnvironment.Config.bIsCrossReferenced)
{
return(null);
}
Log.TraceVerbose("Adding postlink step");
bool bUseCmdExe = BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Win64 || BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Win32;
string ShellBinary = bUseCmdExe ? "cmd.exe" : "/bin/sh";
string ExecuteSwitch = bUseCmdExe ? " /C" : ""; // avoid -c so scripts don't need +x
string ScriptName = bUseCmdExe ? "FixDependencies.bat" : "FixDependencies.sh";
FileItem FixDepsScript = FileItem.GetItemByFullPath(Path.Combine(LinkEnvironment.Config.LocalShadowDirectory, ScriptName));
Action PostLinkAction = new Action(ActionType.Link);
PostLinkAction.WorkingDirectory = Path.GetFullPath(".");
PostLinkAction.CommandPath = ShellBinary;
PostLinkAction.StatusDescription = string.Format("{0}", Path.GetFileName(Executable.AbsolutePath));
PostLinkAction.CommandDescription = "FixDeps";
PostLinkAction.bCanExecuteRemotely = false;
PostLinkAction.CommandArguments = ExecuteSwitch;
PostLinkAction.CommandArguments += bUseCmdExe ? " \"" : " -c '";
FileItem OutputFile = FileItem.GetItemByPath(Path.Combine(LinkEnvironment.Config.LocalShadowDirectory, Path.GetFileNameWithoutExtension(Executable.AbsolutePath) + ".link"));
// Make sure we don't run this script until the all executables and shared libraries
// have been built.
PostLinkAction.PrerequisiteItems.Add(Executable);
foreach (FileItem Dependency in BundleDependencies)
{
PostLinkAction.PrerequisiteItems.Add(Dependency);
}
PostLinkAction.CommandArguments += ShellBinary + ExecuteSwitch + " \"" + FixDepsScript.AbsolutePath + "\" && ";
string Touch = bUseCmdExe ? "echo \"\" >> \"{0}\" && copy /b \"{0}\" +,," : "touch \"{0}\"";
PostLinkAction.CommandArguments += String.Format(Touch, OutputFile.AbsolutePath);
PostLinkAction.CommandArguments += bUseCmdExe ? "\"" : "'";
System.Console.WriteLine("{0} {1}", PostLinkAction.CommandPath, PostLinkAction.CommandArguments);
PostLinkAction.ProducedItems.Add(OutputFile);
return(OutputFile);
}
/// <summary>
/// Gets everything that this file includes from our cache (direct and indirect!)
/// </summary>
/// <param name="AbsoluteFilePath">Path to the file</param>
/// <returns>The list of includes</returns>
public List <FileItem> GetDependenciesForFile(string AbsoluteFilePath)
{
FlatCPPIncludeDependencyInfo DependencyInfo;
if (DependencyMap.TryGetValue(AbsoluteFilePath.ToLowerInvariant(), out DependencyInfo))
{
// Update our transient cache of FileItems for each of the included files
if (DependencyInfo.IncludeFileItems == null)
{
DependencyInfo.IncludeFileItems = new List <FileItem>(DependencyInfo.Includes.Count);
foreach (string Dependency in DependencyInfo.Includes)
{
DependencyInfo.IncludeFileItems.Add(FileItem.GetItemByFullPath(Dependency));
}
}
return(DependencyInfo.IncludeFileItems);
}
return(null);
}
/**
* Finds the header file that is referred to by a partial include filename.
* @param RelativeIncludePath path relative to the project
* @param bSkipExternalHeader true to skip processing of headers in external path
* @param SourceFilesDirectory - The folder containing the source files we're generating a PCH for
*/
public static FileItem FindIncludedFile(string RelativeIncludePath, bool bSkipExternalHeader, List <string> IncludePathsToSearch, Dictionary <string, FileItem> IncludeFileSearchDictionary)
{
FileItem Result = null;
if (IncludePathsToSearch == null)
{
throw new BuildException("Was not expecting IncludePathsToSearch to be empty for file '{0}'!", RelativeIncludePath);
}
++TotalFindIncludedFileCalls;
// Only search for the include file if the result hasn't been cached.
string InvariantPath = RelativeIncludePath.ToLowerInvariant();
if (!IncludeFileSearchDictionary.TryGetValue(InvariantPath, out Result))
{
int SearchAttempts = 0;
if (Path.IsPathRooted(RelativeIncludePath))
{
if (DirectoryLookupCache.FileExists(RelativeIncludePath))
{
Result = FileItem.GetItemByFullPath(RelativeIncludePath);
}
++SearchAttempts;
}
else
{
// Find the first include path that the included file exists in.
foreach (string IncludePath in IncludePathsToSearch)
{
++SearchAttempts;
string RelativeFilePath = "";
try
{
RelativeFilePath = Path.Combine(IncludePath, RelativeIncludePath);
}
catch (ArgumentException Exception)
{
throw new BuildException(Exception, "Failed to combine null or invalid include paths.");
}
string FullFilePath = null;
try
{
FullFilePath = Path.GetFullPath(RelativeFilePath);
}
catch (Exception Exception)
{
throw new BuildException(Exception, "Failed to get full path for include: \"{0}\"", RelativeFilePath);
}
if (DirectoryLookupCache.FileExists(FullFilePath))
{
Result = FileItem.GetItemByFullPath(FullFilePath);
break;
}
}
}
IncludePathSearchAttempts += SearchAttempts;
if (BuildConfiguration.bPrintPerformanceInfo)
{
// More than two search attempts indicates:
// - Include path was not relative to the directory that the including file was in
// - Include path was not relative to the project's base
if (SearchAttempts > 2)
{
Trace.TraceInformation(" Cache miss: " + RelativeIncludePath + " found after " + SearchAttempts.ToString() + " attempts: " + (Result != null ? Result.AbsolutePath : "NOT FOUND!"));
}
}
// Cache the result of the include path search.
IncludeFileSearchDictionary.Add(InvariantPath, Result);
}
// @todo ubtmake: The old UBT tried to skip 'external' (STABLE) headers here. But it didn't work. We might want to do this though! Skip system headers and source/thirdparty headers!
if (Result != null)
{
Log.TraceVerbose("Resolved included file \"{0}\" to: {1}", RelativeIncludePath, Result.AbsolutePath);
}
else
{
Log.TraceVerbose("Couldn't resolve included file \"{0}\"", RelativeIncludePath);
}
return(Result);
}
/// <summary>
/// Finds the files directly or indirectly included by the given C++ file.
/// </summary>
/// <param name="CPPFile">C++ file to get the dependencies for.</param>
/// <param name="Result">List of CPPFile dependencies.</param>
/// <returns>false if CPPFile is still being processed further down the callstack, true otherwise.</returns>
public static bool FindAndCacheAllIncludedFiles(UEBuildTarget Target, FileItem CPPFile, IUEBuildPlatform BuildPlatform, CPPIncludeInfo CPPIncludeInfo, ref IncludedFilesSet Result, bool bOnlyCachedDependencies)
{
IncludedFilesSet IncludedFileList;
var IncludedFilesMap = bOnlyCachedDependencies ? OnlyCachedIncludedFilesMap : ExhaustiveIncludedFilesMap;
if (!IncludedFilesMap.TryGetValue(CPPFile, out IncludedFileList))
{
var TimerStartTime = DateTime.UtcNow;
IncludedFileList = new IncludedFilesSet();
// Add an uninitialized entry for the include file to avoid infinitely recursing on include file loops.
IncludedFilesMap.Add(CPPFile, IncludedFileList);
// Gather a list of names of files directly included by this C++ file.
bool HasUObjects;
List <DependencyInclude> DirectIncludes = GetDirectIncludeDependencies(Target, CPPFile, BuildPlatform, bOnlyCachedDependencies: bOnlyCachedDependencies, HasUObjects: out HasUObjects);
// Build a list of the unique set of files that are included by this file.
var DirectlyIncludedFiles = new HashSet <FileItem>();
// require a for loop here because we need to keep track of the index in the list.
for (int DirectlyIncludedFileNameIndex = 0; DirectlyIncludedFileNameIndex < DirectIncludes.Count; ++DirectlyIncludedFileNameIndex)
{
// Resolve the included file name to an actual file.
DependencyInclude DirectInclude = DirectIncludes[DirectlyIncludedFileNameIndex];
if (DirectInclude.IncludeResolvedName == null ||
// ignore any preexisting resolve cache if we are not configured to use it.
!BuildConfiguration.bUseIncludeDependencyResolveCache ||
// if we are testing the resolve cache, we force UBT to resolve every time to look for conflicts
BuildConfiguration.bTestIncludeDependencyResolveCache
)
{
++TotalDirectIncludeResolveCacheMisses;
// search the include paths to resolve the file
FileItem DirectIncludeResolvedFile = CPPEnvironment.FindIncludedFile(DirectInclude.IncludeName, !BuildConfiguration.bCheckExternalHeadersForModification, CPPIncludeInfo.GetIncludesPathsToSearch(CPPFile), CPPIncludeInfo.IncludeFileSearchDictionary);
if (DirectIncludeResolvedFile != null)
{
DirectlyIncludedFiles.Add(DirectIncludeResolvedFile);
}
IncludeDependencyCache[Target].CacheResolvedIncludeFullPath(CPPFile, DirectlyIncludedFileNameIndex, DirectIncludeResolvedFile != null ? DirectIncludeResolvedFile.AbsolutePath : "");
}
else
{
// we might have cached an attempt to resolve the file, but couldn't actually find the file (system headers, etc).
if (DirectInclude.IncludeResolvedName != string.Empty)
{
DirectlyIncludedFiles.Add(FileItem.GetItemByFullPath(DirectInclude.IncludeResolvedName));
}
}
}
TotalDirectIncludeResolves += DirectIncludes.Count;
// Convert the dictionary of files included by this file into a list.
foreach (var DirectlyIncludedFile in DirectlyIncludedFiles)
{
// Add the file we're directly including
IncludedFileList.Add(DirectlyIncludedFile);
// Also add all of the indirectly included files!
if (FindAndCacheAllIncludedFiles(Target, DirectlyIncludedFile, BuildPlatform, CPPIncludeInfo, ref IncludedFileList, bOnlyCachedDependencies:bOnlyCachedDependencies) == false)
{
// DirectlyIncludedFile is a circular dependency which is still being processed
// further down the callstack. Add this file to its circular dependencies list
// so that it can update its dependencies later.
IncludedFilesSet DirectlyIncludedFileIncludedFileList;
if (IncludedFilesMap.TryGetValue(DirectlyIncludedFile, out DirectlyIncludedFileIncludedFileList))
{
DirectlyIncludedFileIncludedFileList.CircularDependencies.Add(CPPFile);
}
}
}
// All dependencies have been processed by now so update all circular dependencies
// with the full list.
foreach (var CircularDependency in IncludedFileList.CircularDependencies)
{
IncludedFilesSet CircularDependencyIncludedFiles = IncludedFilesMap[CircularDependency];
foreach (FileItem IncludedFile in IncludedFileList)
{
CircularDependencyIncludedFiles.Add(IncludedFile);
}
}
// No need to keep this around anymore.
IncludedFileList.CircularDependencies.Clear();
// Done collecting files.
IncludedFileList.bIsInitialized = true;
var TimerDuration = DateTime.UtcNow - TimerStartTime;
TotalTimeSpentGettingIncludes += TimerDuration.TotalSeconds;
}
if (IncludedFileList.bIsInitialized)
{
// Copy the list of files included by this file into the result list.
foreach (FileItem IncludedFile in IncludedFileList)
{
// If the result list doesn't contain this file yet, add the file and the files it includes.
// NOTE: For some reason in .NET 4, Add() is over twice as fast as calling UnionWith() on the set
Result.Add(IncludedFile);
}
return(true);
}
else
{
// The IncludedFileList.bIsInitialized was false because we added a dummy entry further down the call stack. We're already processing
// the include list for this header elsewhere in the stack frame, so we don't need to add anything here.
return(false);
}
}
