public List <FileItem> FindAndCacheAllIncludedFiles(FileItem SourceFile, CppIncludePaths IncludePaths, bool bOnlyCachedDependencies) { List <FileItem> Result = null; if (IncludePaths.IncludeFileSearchDictionary == null) { IncludePaths.IncludeFileSearchDictionary = new Dictionary <string, FileItem>(); } if (bOnlyCachedDependencies && bUseFlatCPPIncludeDependencyCache) { Result = FlatCPPIncludeDependencyCache.GetDependenciesForFile(SourceFile.Reference); if (Result == null) { // Nothing cached for this file! It is new to us. This is the expected flow when our CPPIncludeDepencencyCache is missing. } } else { // @todo ubtmake: HeaderParser.h is missing from the include set for Module.UnrealHeaderTool.cpp (failed to find include using: FileItem DirectIncludeResolvedFile = CPPEnvironment.FindIncludedFile(DirectInclude.IncludeName, !BuildConfiguration.bCheckExternalHeadersForModification, IncludePathsToSearch, IncludeFileSearchDictionary );) // If we're doing an exhaustive include scan, make sure that we have our include dependency cache loaded and ready if (!bOnlyCachedDependencies) { if (IncludeDependencyCache == null) { IncludeDependencyCache = DependencyCache.Create(DependencyCacheFile); } } Result = new List <FileItem>(); IncludedFilesSet IncludedFileList = new IncludedFilesSet(); FindAndCacheAllIncludedFiles(SourceFile, IncludePaths, ref IncludedFileList, bOnlyCachedDependencies: bOnlyCachedDependencies); foreach (FileItem IncludedFile in IncludedFileList) { Result.Add(IncludedFile); } // Update cache if (bUseFlatCPPIncludeDependencyCache && !bOnlyCachedDependencies) { List <FileReference> Dependencies = new List <FileReference>(); foreach (FileItem IncludedFile in Result) { Dependencies.Add(IncludedFile.Reference); } FileReference PCHName = SourceFile.PrecompiledHeaderIncludeFilename; FlatCPPIncludeDependencyCache.SetDependenciesForFile(SourceFile.Reference, PCHName, Dependencies); } } return(Result); }
public FileItem CachePCHUsageForCPPFile(FileItem CPPFile, CppIncludePaths IncludePaths, CppPlatform Platform) { // @todo ubtmake: We don't really need to scan every file looking for PCH headers, just need one. The rest is just for error checking. // @todo ubtmake: We don't need all of the direct includes either. We just need the first, unless we want to check for errors. List <DependencyInclude> DirectIncludeFilenames = GetDirectIncludeDependencies(CPPFile, bOnlyCachedDependencies: false); if (UnrealBuildTool.bPrintDebugInfo) { Log.TraceVerbose("Found direct includes for {0}: {1}", Path.GetFileName(CPPFile.AbsolutePath), string.Join(", ", DirectIncludeFilenames.Select(F => F.IncludeName))); } if (DirectIncludeFilenames.Count == 0) { return(null); } DependencyInclude FirstInclude = DirectIncludeFilenames[0]; // Resolve the PCH header to an absolute path. // Check NullOrEmpty here because if the file could not be resolved we need to throw an exception if (FirstInclude.IncludeResolvedNameIfSuccessful != null && // ignore any preexisting resolve cache if we are not configured to use it. bUseIncludeDependencyResolveCache && // if we are testing the resolve cache, we force UBT to resolve every time to look for conflicts !bTestIncludeDependencyResolveCache) { CPPFile.PrecompiledHeaderIncludeFilename = FirstInclude.IncludeResolvedNameIfSuccessful; return(FileItem.GetItemByFileReference(CPPFile.PrecompiledHeaderIncludeFilename)); } // search the include paths to resolve the file. string FirstIncludeName = FirstInclude.IncludeName; UEBuildPlatform BuildPlatform = UEBuildPlatform.GetBuildPlatformForCPPTargetPlatform(Platform); // convert back from relative to host path if needed if (!BuildPlatform.UseAbsolutePathsInUnityFiles()) { FirstIncludeName = RemoteExports.UnconvertPath(FirstIncludeName); } FileItem PrecompiledHeaderIncludeFile = CPPHeaders.FindIncludedFile(CPPFile.Reference, FirstIncludeName, IncludePaths); if (PrecompiledHeaderIncludeFile == null) { FirstIncludeName = RemoteExports.UnconvertPath(FirstInclude.IncludeName); throw new BuildException("The first include statement in source file '{0}' is trying to include the file '{1}' as the precompiled header, but that file could not be located in any of the module's include search paths.", CPPFile.AbsolutePath, FirstIncludeName); } IncludeDependencyCache.CacheResolvedIncludeFullPath(CPPFile, 0, PrecompiledHeaderIncludeFile.Reference, bUseIncludeDependencyResolveCache, bTestIncludeDependencyResolveCache); CPPFile.PrecompiledHeaderIncludeFilename = PrecompiledHeaderIncludeFile.Reference; return(PrecompiledHeaderIncludeFile); }
/// <summary> /// Copy constructor. /// </summary> /// <param name="Other">Environment to copy settings from</param> public CppCompileEnvironment(CppCompileEnvironment Other) { Platform = Other.Platform; Configuration = Other.Configuration; Architecture = Other.Architecture; OutputDirectory = Other.OutputDirectory; PCHOutputDirectory = Other.PCHOutputDirectory; LocalShadowDirectory = Other.LocalShadowDirectory; PrecompiledHeaderIncludeFilename = Other.PrecompiledHeaderIncludeFilename; PrecompiledHeaderAction = Other.PrecompiledHeaderAction; bUseRTTI = Other.bUseRTTI; bUseInlining = Other.bUseInlining; bUseAVX = Other.bUseAVX; bFasterWithoutUnity = Other.bFasterWithoutUnity; MinSourceFilesForUnityBuildOverride = Other.MinSourceFilesForUnityBuildOverride; MinFilesUsingPrecompiledHeaderOverride = Other.MinFilesUsingPrecompiledHeaderOverride; bBuildLocallyWithSNDBS = Other.bBuildLocallyWithSNDBS; bEnableExceptions = Other.bEnableExceptions; bEnableObjCExceptions = Other.bEnableObjCExceptions; bShadowVariableWarningsAsErrors = Other.bShadowVariableWarningsAsErrors; bEnableShadowVariableWarnings = Other.bEnableShadowVariableWarnings; bUndefinedIdentifierWarningsAsErrors = Other.bUndefinedIdentifierWarningsAsErrors; bEnableUndefinedIdentifierWarnings = Other.bEnableUndefinedIdentifierWarnings; bOptimizeCode = Other.bOptimizeCode; bOptimizeForSize = Other.bOptimizeForSize; bCreateDebugInfo = Other.bCreateDebugInfo; bIsBuildingLibrary = Other.bIsBuildingLibrary; bIsBuildingDLL = Other.bIsBuildingDLL; bUseStaticCRT = Other.bUseStaticCRT; bUseDebugCRT = Other.bUseDebugCRT; bOmitFramePointers = Other.bOmitFramePointers; bEnableOSX109Support = Other.bEnableOSX109Support; bUsePDBFiles = Other.bUsePDBFiles; bSupportEditAndContinue = Other.bSupportEditAndContinue; bUseIncrementalLinking = Other.bUseIncrementalLinking; bAllowLTCG = Other.bAllowLTCG; bPGOOptimize = Other.bPGOOptimize; bPGOProfile = Other.bPGOProfile; PGOFilenamePrefix = Other.PGOFilenamePrefix; PGODirectory = Other.PGODirectory; bPrintTimingInfo = Other.bPrintTimingInfo; bAllowRemotelyCompiledPCHs = Other.bAllowRemotelyCompiledPCHs; IncludePaths = new CppIncludePaths(Other.IncludePaths); ForceIncludeFiles.AddRange(Other.ForceIncludeFiles); Definitions.AddRange(Other.Definitions); AdditionalArguments = Other.AdditionalArguments; AdditionalFrameworks.AddRange(Other.AdditionalFrameworks); PrecompiledHeaderFile = Other.PrecompiledHeaderFile; Headers = Other.Headers; bHackHeaderGenerator = Other.bHackHeaderGenerator; bHideSymbolsByDefault = Other.bHideSymbolsByDefault; }
public List <FileItem> FindAndCacheAllIncludedFiles(FileItem SourceFile, CppIncludePaths IncludePaths, bool bOnlyCachedDependencies) { List <FileItem> Result = null; if (IncludePaths.IncludeFileSearchDictionary == null) { IncludePaths.IncludeFileSearchDictionary = new Dictionary <string, FileItem>(); } if (bOnlyCachedDependencies && bUseFlatCPPIncludeDependencyCache) { Result = FlatCPPIncludeDependencyCache.GetDependenciesForFile(SourceFile.Location); if (Result == null) { // Nothing cached for this file! It is new to us. This is the expected flow when our CPPIncludeDepencencyCache is missing. } } else { // If we're doing an exhaustive include scan, make sure that we have our include dependency cache loaded and ready if (!bOnlyCachedDependencies) { if (IncludeDependencyCache == null) { IncludeDependencyCache = DependencyCache.Create(DependencyCacheFile); } } // Get the headers Result = FindAndCacheIncludedFiles(SourceFile, IncludePaths, bOnlyCachedDependencies); // Update cache if (bUseFlatCPPIncludeDependencyCache && !bOnlyCachedDependencies) { List <FileReference> Dependencies = new List <FileReference>(); foreach (FileItem IncludedFile in Result) { Dependencies.Add(IncludedFile.Location); } FileReference PCHName = SourceFile.PrecompiledHeaderIncludeFilename; FlatCPPIncludeDependencyCache.SetDependenciesForFile(SourceFile.Location, PCHName, Dependencies); } } return(Result); }
/// <summary> /// Get a set of directly included files from the given source file, resolving their include paths to FileItem instances. /// </summary> /// <param name="SourceFile">The file to check.</param> /// <param name="IncludePaths">Include paths to search.</param> /// <param name="bOnlyCachedDependencies">Whether to just return cached dependencies, or update the cache with new results.</param> /// <returns>Set of files that are included</returns> private HashSet <FileItem> GetDirectlyIncludedFiles(FileItem SourceFile, CppIncludePaths IncludePaths, bool bOnlyCachedDependencies) { // Gather a list of names of files directly included by this C++ file. List <DependencyInclude> DirectIncludes = GetDirectIncludeDependencies(SourceFile, bOnlyCachedDependencies: bOnlyCachedDependencies); // Build a list of the unique set of files that are included by this file. HashSet <FileItem> DirectlyIncludedFiles = new HashSet <FileItem>(); // require a for loop here because we need to keep track of the index in the list. for (int DirectlyIncludedFileNameIndex = 0; DirectlyIncludedFileNameIndex < DirectIncludes.Count; ++DirectlyIncludedFileNameIndex) { // Resolve the included file name to an actual file. DependencyInclude DirectInclude = DirectIncludes[DirectlyIncludedFileNameIndex]; if (!DirectInclude.HasAttemptedResolve || // ignore any preexisting resolve cache if we are not configured to use it. !bUseIncludeDependencyResolveCache || // if we are testing the resolve cache, we force UBT to resolve every time to look for conflicts bTestIncludeDependencyResolveCache ) { ++TotalDirectIncludeResolveCacheMisses; // search the include paths to resolve the file FileItem DirectIncludeResolvedFile = CPPHeaders.FindIncludedFile(SourceFile.Location, DirectInclude.IncludeName, IncludePaths); if (DirectIncludeResolvedFile != null) { DirectlyIncludedFiles.Add(DirectIncludeResolvedFile); } IncludeDependencyCache.CacheResolvedIncludeFullPath(SourceFile, DirectlyIncludedFileNameIndex, DirectIncludeResolvedFile != null ? DirectIncludeResolvedFile.Location : null, bUseIncludeDependencyResolveCache, bTestIncludeDependencyResolveCache); } else { // we might have cached an attempt to resolve the file, but couldn't actually find the file (system headers, etc). if (DirectInclude.IncludeResolvedNameIfSuccessful != null) { DirectlyIncludedFiles.Add(FileItem.GetItemByFileReference(DirectInclude.IncludeResolvedNameIfSuccessful)); } } } TotalDirectIncludeResolves += DirectIncludes.Count; return(DirectlyIncludedFiles); }
/// <summary> /// Add all the files included by a source file to a set, using a cache. /// </summary> /// <param name="SourceFile">The file to check.</param> /// <param name="IncludePaths">Include paths to search.</param> /// <param name="bOnlyCachedDependencies">Whether to just return cached dependencies, or update the cache with new results.</param> private List <FileItem> FindAndCacheIncludedFiles(FileItem SourceFile, CppIncludePaths IncludePaths, bool bOnlyCachedDependencies) { // Get the map of files to their list of includes Dictionary <FileItem, List <FileItem> > FileToIncludedFiles = bOnlyCachedDependencies ? OnlyCachedIncludedFilesMap : ExhaustiveIncludedFilesMap; // Check if the included files is in the cache. If not, we'll create it. List <FileItem> IncludedFiles; if (!FileToIncludedFiles.TryGetValue(SourceFile, out IncludedFiles)) { HashSet <FileItem> VisitedFiles = new HashSet <FileItem>(); VisitedFiles.Add(SourceFile); HashSet <FileItem> IncludedFilesSet = new HashSet <FileItem>(); FindAndCacheIncludedFilesInner(SourceFile, IncludedFilesSet, IncludePaths, bOnlyCachedDependencies, VisitedFiles); IncludedFiles = IncludedFilesSet.ToList(); FileToIncludedFiles.Add(SourceFile, IncludedFiles); } return(IncludedFiles); }
/// <summary> /// Copy constructor /// </summary> /// <param name="Other">Duplicate another instance's settings</param> public CppIncludePaths(CppIncludePaths Other) { UserIncludePaths = new HashSet <DirectoryReference>(Other.UserIncludePaths); SystemIncludePaths = new HashSet <DirectoryReference>(Other.SystemIncludePaths); bCheckSystemHeadersForModification = Other.bCheckSystemHeadersForModification; }
/// <summary> /// Add all the files included by a source file to a set, using a cache. /// </summary> /// <param name="SourceFile">The file to check.</param> /// <param name="IncludedFiles">Set of included files to add to</param> /// <param name="IncludePaths">Include paths to search.</param> /// <param name="bOnlyCachedDependencies">Whether to just return cached dependencies, or update the cache with new results.</param> /// <param name="VisitedFiles">Set of files that have already been visited. Used to prevent infinite loops between circularly dependent headers.</param> private void FindAndCacheIncludedFilesInner(FileItem SourceFile, HashSet <FileItem> IncludedFiles, CppIncludePaths IncludePaths, bool bOnlyCachedDependencies, HashSet <FileItem> VisitedFiles) { HashSet <FileItem> DirectlyIncludedFiles = GetDirectlyIncludedFiles(SourceFile, IncludePaths, bOnlyCachedDependencies); foreach (FileItem DirectlyIncludedFile in DirectlyIncludedFiles) { if (IncludedFiles.Add(DirectlyIncludedFile)) { // Get the map of files to their list of includes Dictionary <FileItem, List <FileItem> > FileToIncludedFiles = bOnlyCachedDependencies ? OnlyCachedIncludedFilesMap : ExhaustiveIncludedFilesMap; // Recursively add the files included by this file List <FileItem> InnerFiles; if (FileToIncludedFiles.TryGetValue(DirectlyIncludedFile, out InnerFiles)) { // We already have the include paths cached; just add them directly. IncludedFiles.UnionWith(InnerFiles); } else if (VisitedFiles.Add(DirectlyIncludedFile)) { // We don't have include paths cached, and this isn't a recursive call. Create a new set and add it to the cache. HashSet <FileItem> InnerFilesSet = new HashSet <FileItem>(); FindAndCacheIncludedFilesInner(DirectlyIncludedFile, InnerFilesSet, IncludePaths, bOnlyCachedDependencies, VisitedFiles); FileToIncludedFiles.Add(DirectlyIncludedFile, InnerFilesSet.ToList()); IncludedFiles.UnionWith(InnerFilesSet); } else { // We're already building a list of include paths for this file up the stack. Just recurse through it this time. FindAndCacheIncludedFilesInner(DirectlyIncludedFile, IncludedFiles, IncludePaths, bOnlyCachedDependencies, VisitedFiles); } } } }
/// <summary> /// Finds the header file that is referred to by a partial include filename. /// </summary> /// <param name="FromFile">The file containing the include directory</param> /// <param name="RelativeIncludePath">path relative to the project</param> /// <param name="IncludePaths">Include paths to search</param> public static FileItem FindIncludedFile(FileReference FromFile, string RelativeIncludePath, CppIncludePaths IncludePaths) { FileItem Result = null; ++TotalFindIncludedFileCalls; // Only search for the include file if the result hasn't been cached. string InvariantPath = RelativeIncludePath.ToLowerInvariant(); if (!IncludePaths.IncludeFileSearchDictionary.TryGetValue(InvariantPath, out Result)) { int SearchAttempts = 0; if (Path.IsPathRooted(RelativeIncludePath)) { FileReference Reference = FileReference.Combine(UnrealBuildTool.EngineSourceDirectory, RelativeIncludePath); if (DirectoryLookupCache.FileExists(Reference)) { Result = FileItem.GetItemByFileReference(Reference); } ++SearchAttempts; } else { // Find the first include path that the included file exists in. List <DirectoryReference> IncludePathsToSearch = IncludePaths.GetPathsToSearch(FromFile); foreach (DirectoryReference IncludePath in IncludePathsToSearch) { ++SearchAttempts; FileReference FullFilePath; try { FullFilePath = FileReference.Combine(IncludePath, RelativeIncludePath); } catch (ArgumentException Exception) { throw new BuildException(Exception, "Failed to combine null or invalid include paths."); } if (DirectoryLookupCache.FileExists(FullFilePath)) { Result = FileItem.GetItemByFileReference(FullFilePath); break; } } } IncludePathSearchAttempts += SearchAttempts; if (UnrealBuildTool.bPrintPerformanceInfo) { // More than two search attempts indicates: // - Include path was not relative to the directory that the including file was in // - Include path was not relative to the project's base if (SearchAttempts > 2) { Log.TraceVerbose(" Cache miss: " + RelativeIncludePath + " found after " + SearchAttempts.ToString() + " attempts: " + (Result != null ? Result.AbsolutePath : "NOT FOUND!")); } } // Cache the result of the include path search. IncludePaths.IncludeFileSearchDictionary.Add(InvariantPath, Result); } // @todo ubtmake: The old UBT tried to skip 'external' (STABLE) headers here. But it didn't work. We might want to do this though! Skip system headers and source/thirdparty headers! if (Result != null) { Log.TraceVerbose("Resolved included file \"{0}\" to: {1}", RelativeIncludePath, Result.AbsolutePath); } else { Log.TraceVerbose("Couldn't resolve included file \"{0}\"", RelativeIncludePath); } return(Result); }
/// <summary> /// Copy constructor /// </summary> /// <param name="Other">Duplicate another instance's settings</param> public CppIncludePaths(CppIncludePaths Other) { UserIncludePaths = new HashSet <string>(Other.UserIncludePaths); SystemIncludePaths = new HashSet <string>(Other.SystemIncludePaths); bCheckSystemHeadersForModification = Other.bCheckSystemHeadersForModification; }
/// <summary> /// Finds the files directly or indirectly included by the given C++ file. /// </summary> /// <param name="CPPFile">C++ file to get the dependencies for.</param> /// <param name="IncludePaths"></param> /// <param name="bOnlyCachedDependencies"></param> /// <param name="Result">List of CPPFile dependencies.</param> /// <returns>false if CPPFile is still being processed further down the callstack, true otherwise.</returns> public bool FindAndCacheAllIncludedFiles(FileItem CPPFile, CppIncludePaths IncludePaths, ref IncludedFilesSet Result, bool bOnlyCachedDependencies) { IncludedFilesSet IncludedFileList; Dictionary <FileItem, IncludedFilesSet> IncludedFilesMap = bOnlyCachedDependencies ? OnlyCachedIncludedFilesMap : ExhaustiveIncludedFilesMap; if (!IncludedFilesMap.TryGetValue(CPPFile, out IncludedFileList)) { DateTime TimerStartTime = DateTime.UtcNow; IncludedFileList = new IncludedFilesSet(); // Add an uninitialized entry for the include file to avoid infinitely recursing on include file loops. IncludedFilesMap.Add(CPPFile, IncludedFileList); // Gather a list of names of files directly included by this C++ file. List <DependencyInclude> DirectIncludes = GetDirectIncludeDependencies(CPPFile, bOnlyCachedDependencies: bOnlyCachedDependencies); // Build a list of the unique set of files that are included by this file. HashSet <FileItem> DirectlyIncludedFiles = new HashSet <FileItem>(); // require a for loop here because we need to keep track of the index in the list. for (int DirectlyIncludedFileNameIndex = 0; DirectlyIncludedFileNameIndex < DirectIncludes.Count; ++DirectlyIncludedFileNameIndex) { // Resolve the included file name to an actual file. DependencyInclude DirectInclude = DirectIncludes[DirectlyIncludedFileNameIndex]; if (!DirectInclude.HasAttemptedResolve || // ignore any preexisting resolve cache if we are not configured to use it. !bUseIncludeDependencyResolveCache || // if we are testing the resolve cache, we force UBT to resolve every time to look for conflicts bTestIncludeDependencyResolveCache ) { ++TotalDirectIncludeResolveCacheMisses; // search the include paths to resolve the file FileItem DirectIncludeResolvedFile = CPPHeaders.FindIncludedFile(CPPFile.Reference, DirectInclude.IncludeName, IncludePaths); if (DirectIncludeResolvedFile != null) { DirectlyIncludedFiles.Add(DirectIncludeResolvedFile); } IncludeDependencyCache.CacheResolvedIncludeFullPath(CPPFile, DirectlyIncludedFileNameIndex, DirectIncludeResolvedFile != null ? DirectIncludeResolvedFile.Reference : null, bUseIncludeDependencyResolveCache, bTestIncludeDependencyResolveCache); } else { // we might have cached an attempt to resolve the file, but couldn't actually find the file (system headers, etc). if (DirectInclude.IncludeResolvedNameIfSuccessful != null) { DirectlyIncludedFiles.Add(FileItem.GetItemByFileReference(DirectInclude.IncludeResolvedNameIfSuccessful)); } } } TotalDirectIncludeResolves += DirectIncludes.Count; // Convert the dictionary of files included by this file into a list. foreach (FileItem DirectlyIncludedFile in DirectlyIncludedFiles) { // Add the file we're directly including IncludedFileList.Add(DirectlyIncludedFile); // Also add all of the indirectly included files! if (FindAndCacheAllIncludedFiles(DirectlyIncludedFile, IncludePaths, ref IncludedFileList, bOnlyCachedDependencies: bOnlyCachedDependencies) == false) { // DirectlyIncludedFile is a circular dependency which is still being processed // further down the callstack. Add this file to its circular dependencies list // so that it can update its dependencies later. IncludedFilesSet DirectlyIncludedFileIncludedFileList; if (IncludedFilesMap.TryGetValue(DirectlyIncludedFile, out DirectlyIncludedFileIncludedFileList)) { DirectlyIncludedFileIncludedFileList.CircularDependencies.Add(CPPFile); } } } // All dependencies have been processed by now so update all circular dependencies // with the full list. foreach (FileItem CircularDependency in IncludedFileList.CircularDependencies) { IncludedFilesSet CircularDependencyIncludedFiles = IncludedFilesMap[CircularDependency]; foreach (FileItem IncludedFile in IncludedFileList) { CircularDependencyIncludedFiles.Add(IncludedFile); } } // No need to keep this around anymore. IncludedFileList.CircularDependencies.Clear(); // Done collecting files. IncludedFileList.bIsInitialized = true; TimeSpan TimerDuration = DateTime.UtcNow - TimerStartTime; TotalTimeSpentGettingIncludes += TimerDuration.TotalSeconds; } if (IncludedFileList.bIsInitialized) { // Copy the list of files included by this file into the result list. foreach (FileItem IncludedFile in IncludedFileList) { // If the result list doesn't contain this file yet, add the file and the files it includes. // NOTE: For some reason in .NET 4, Add() is over twice as fast as calling UnionWith() on the set Result.Add(IncludedFile); } return(true); } else { // The IncludedFileList.bIsInitialized was false because we added a dummy entry further down the call stack. We're already processing // the include list for this header elsewhere in the stack frame, so we don't need to add anything here. return(false); } }