private void ExportModuleCpp(UEBuildModuleCPP ModuleCPP, CppCompileEnvironment ModuleCompileEnvironment, JsonWriter Writer)
{
Writer.WriteValue("GeneratedCodeDirectory", ModuleCPP.GeneratedCodeDirectory != null ? ModuleCPP.GeneratedCodeDirectory.FullName : string.Empty);
ToolchainInfo ModuleToolchainInfo = GenerateToolchainInfo(ModuleCompileEnvironment);
if (!ModuleToolchainInfo.Equals(RootToolchainInfo))
{
Writer.WriteObjectStart("ToolchainInfo");
foreach (Tuple <string, object> Field in ModuleToolchainInfo.GetDiff(RootToolchainInfo))
{
WriteField(ModuleCPP.Name, Writer, Field);
}
Writer.WriteObjectEnd();
}
if (ModuleCompileEnvironment.PrecompiledHeaderIncludeFilename != null)
{
string CorrectFilePathPch;
if (ExtractWrappedIncludeFile(ModuleCompileEnvironment.PrecompiledHeaderIncludeFilename, out CorrectFilePathPch))
{
Writer.WriteValue("SharedPCHFilePath", CorrectFilePathPch);
}
}
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="Module">The module with a valid shared PCH</param>
/// <param name="BaseCompileEnvironment">The compile environment to use</param>
/// <param name="HeaderFile">The header file to generate a PCH from</param>
/// <param name="OutputDir">Output directory for instances of this PCH</param>
public PrecompiledHeaderTemplate(UEBuildModuleCPP Module, CppCompileEnvironment BaseCompileEnvironment, FileItem HeaderFile, DirectoryReference OutputDir)
{
this.Module = Module;
this.BaseCompileEnvironment = BaseCompileEnvironment;
this.HeaderFile = HeaderFile;
this.OutputDir = OutputDir;
}
private List <FileItem> CreateOutputFilesForUCA(LinkEnvironment BinaryLinkEnvironment, ActionGraph ActionGraph)
{
List <FileItem> OutputFiles = new List <FileItem>();
string ModuleName = Modules.Select(Module => Module.Name).First(Name => Name.CompareTo(BuildConfiguration.UCAModuleToAnalyze) == 0);
UEBuildModuleCPP ModuleCPP = (UEBuildModuleCPP)Target.GetModuleByName(ModuleName);
FileItem ModulePrivatePCH = ModuleCPP.ProcessedDependencies.UniquePCHHeaderFile;
string IntermediatePath = Path.Combine(Target.ProjectIntermediateDirectory.FullName, ModuleName);
FileReference OutputFileName = Target.OutputPath;
FileItem OutputFile = FileItem.GetItemByFileReference(OutputFileName);
Action LinkAction = ActionGraph.Add(ActionType.Compile);
LinkAction.WorkingDirectory = UnrealBuildTool.EngineSourceDirectory.FullName;
LinkAction.CommandPath = System.IO.Path.Combine(LinkAction.WorkingDirectory, @"..", @"Binaries", @"Win32", @"UnrealCodeAnalyzer.exe");
LinkAction.ProducedItems.Add(OutputFile);
LinkAction.PrerequisiteItems.AddRange(BinaryLinkEnvironment.InputFiles);
LinkAction.CommandArguments = @"-AnalyzePCHFile -PCHFile=""" + ModulePrivatePCH.AbsolutePath + @""" -OutputFile=""" + OutputFileName + @""" -HeaderDataPath=""" + IntermediatePath + @""" -UsageThreshold " + BuildConfiguration.UCAUsageThreshold.ToString(CultureInfo.InvariantCulture);
foreach (string IncludeSearchPath in ModuleCPP.IncludeSearchPaths)
{
LinkAction.CommandArguments += @" /I""" + LinkAction.WorkingDirectory + @"\" + IncludeSearchPath + @"""";
}
OutputFiles.Add(OutputFile);
return(OutputFiles);
}
/// <summary>
/// Write a Target to a JSON writer. Is array is empty, don't write anything
/// </summary>
/// <param name="Target"></param>
/// <param name="Writer">Writer for the array data</param>
private static void ExportTarget(UEBuildTarget Target, JsonWriter Writer)
{
Writer.WriteObjectStart();
Writer.WriteValue("Name", Target.TargetName);
Writer.WriteValue("Configuration", Target.Configuration.ToString());
Writer.WriteValue("Platform", Target.Platform.ToString());
Writer.WriteValue("TargetFile", Target.TargetRulesFile.FullName);
if (Target.ProjectFile != null)
{
Writer.WriteValue("ProjectFile", Target.ProjectFile.FullName);
}
ExportEnvironmentToJson(Target, Writer);
if (Target.Binaries.Any())
{
Writer.WriteArrayStart("Binaries");
foreach (UEBuildBinary Binary in Target.Binaries)
{
Writer.WriteObjectStart();
ExportBinary(Binary, Writer);
Writer.WriteObjectEnd();
}
Writer.WriteArrayEnd();
}
CppCompileEnvironment GlobalCompileEnvironment = Target.CreateCompileEnvironmentForProjectFiles();
HashSet <string> ModuleNames = new HashSet <string>();
Writer.WriteObjectStart("Modules");
foreach (UEBuildBinary Binary in Target.Binaries)
{
CppCompileEnvironment BinaryCompileEnvironment = Binary.CreateBinaryCompileEnvironment(GlobalCompileEnvironment);
foreach (UEBuildModule Module in Binary.Modules)
{
if (ModuleNames.Add(Module.Name))
{
Writer.WriteObjectStart(Module.Name);
ExportModule(Module, Binary.OutputDir, Target.GetExecutableDir(), Writer);
UEBuildModuleCPP ModuleCpp = Module as UEBuildModuleCPP;
if (ModuleCpp != null)
{
CppCompileEnvironment ModuleCompileEnvironment = ModuleCpp.CreateCompileEnvironmentForIntellisense(Target.Rules, BinaryCompileEnvironment);
ExportModuleCpp(ModuleCpp, ModuleCompileEnvironment, Writer);
}
Writer.WriteObjectEnd();
}
}
}
Writer.WriteObjectEnd();
ExportPluginsFromTarget(Target, Writer);
Writer.WriteObjectEnd();
}
private static void ExportModuleCpp(UEBuildModuleCPP ModuleCPP, CppCompileEnvironment ModuleCompileEnvironment, JsonWriter Writer)
{
Writer.WriteValue("GeneratedCodeDirectory", ModuleCPP.GeneratedCodeDirectory != null ? ModuleCPP.GeneratedCodeDirectory.FullName : string.Empty);
if (ModuleCompileEnvironment.PrecompiledHeaderIncludeFilename != null)
{
string CorrectFilePathPch;
if (ExtractWrappedIncludeFile(ModuleCompileEnvironment.PrecompiledHeaderIncludeFilename, out CorrectFilePathPch))
{
Writer.WriteValue("SharedPCHFilePath", CorrectFilePathPch);
}
}
}
/// <summary>
/// Adds information about a module to this project file
/// </summary>
/// <param name="Module">The module to add</param>
/// <param name="CompileEnvironment">Compile environment for this module</param>
public virtual void AddModule(UEBuildModuleCPP Module, CppCompileEnvironment CompileEnvironment)
{
AddIntelliSensePreprocessorDefinitions(CompileEnvironment.Definitions);
AddIntelliSenseIncludePaths(SystemIncludePaths, CompileEnvironment.SystemIncludePaths);
AddIntelliSenseIncludePaths(UserIncludePaths, CompileEnvironment.UserIncludePaths);
foreach (DirectoryReference BaseDir in Module.ModuleDirectories)
{
AddIntelliSenseIncludePaths(BaseDir, BaseDirToSystemIncludePaths, CompileEnvironment.SystemIncludePaths);
AddIntelliSenseIncludePaths(BaseDir, BaseDirToUserIncludePaths, CompileEnvironment.UserIncludePaths);
}
SetIntelliSenseCppVersion(Module.Rules.CppStandard);
}
/// <summary>
/// Create an instance of the class with the given configuration data
/// </summary>
/// <param name="Type"></param>
/// <param name="OutputFilePaths"></param>
/// <param name="IntermediateDirectory"></param>
/// <param name="bAllowExports"></param>
/// <param name="PrimaryModule"></param>
/// <param name="bUsePrecompiled"></param>
public UEBuildBinary(
UEBuildBinaryType Type,
IEnumerable <FileReference> OutputFilePaths,
DirectoryReference IntermediateDirectory,
bool bAllowExports,
UEBuildModuleCPP PrimaryModule,
bool bUsePrecompiled
)
{
this.Type = Type;
this.OutputFilePaths = new List <FileReference>(OutputFilePaths);
this.IntermediateDirectory = IntermediateDirectory;
this.bAllowExports = bAllowExports;
this.PrimaryModule = PrimaryModule;
this.bUsePrecompiled = bUsePrecompiled;
Modules.Add(PrimaryModule);
}
/// <summary>
/// Determines if a directory, or any subdirectory of it, contains new source files
/// </summary>
/// <param name="Directory">Directory to search through</param>
/// <param name="ExcludedFolderNames">Set of directory names to exclude</param>
/// <returns>True if the directory contains any source files</returns>
static bool ContainsSourceFiles(DirectoryItem Directory, ReadOnlyHashSet <string> ExcludedFolderNames)
{
// Check this directory isn't ignored
if (!ExcludedFolderNames.Contains(Directory.Name))
{
// Check for any source files in this actual directory
FileItem[] SourceFiles = UEBuildModuleCPP.GetSourceFiles(Directory);
if (SourceFiles.Length > 0)
{
return(true);
}
// Check for any source files in a subdirectory
foreach (DirectoryItem SubDirectory in Directory.EnumerateDirectories())
{
if (ContainsSourceFiles(SubDirectory, ExcludedFolderNames))
{
return(true);
}
}
}
return(false);
}
/// <summary>
/// Given a set of C++ files, generates another set of C++ files that #include all the original
/// files, the goal being to compile the same code in fewer translation units.
/// The "unity" files are written to the CompileEnvironment's OutputDirectory.
/// </summary>
/// <param name="Target">The target we're building</param>
/// <param name="CPPFiles">The C++ files to #include.</param>
/// <param name="CompileEnvironment">The environment that is used to compile the C++ files.</param>
/// <param name="BaseName">Base name to use for the Unity files</param>
/// <returns>The "unity" C++ files.</returns>
public static List <FileItem> GenerateUnityCPPs(
UEBuildTarget Target,
List <FileItem> CPPFiles,
CPPEnvironment CompileEnvironment,
string BaseName
)
{
var ToolChain = UEToolChain.GetPlatformToolChain(CompileEnvironment.Config.Target.Platform);
var BuildPlatform = UEBuildPlatform.GetBuildPlatformForCPPTargetPlatform(CompileEnvironment.Config.Target.Platform);
// Figure out size of all input files combined. We use this to determine whether to use larger unity threshold or not.
long TotalBytesInCPPFiles = CPPFiles.Sum(F => F.Info.Length);
// We have an increased threshold for unity file size if, and only if, all files fit into the same unity file. This
// is beneficial when dealing with PCH files. The default PCH creation limit is X unity files so if we generate < X
// this could be fairly slow and we'd rather bump the limit a bit to group them all into the same unity file.
//
// Optimization only makes sense if PCH files are enabled.
bool bForceIntoSingleUnityFile = BuildConfiguration.bStressTestUnity || (TotalBytesInCPPFiles < BuildConfiguration.NumIncludedBytesPerUnityCPP * 2 && CompileEnvironment.ShouldUsePCHs());
// Build the list of unity files.
List <FileCollection> AllUnityFiles;
{
var CPPUnityFileBuilder = new UnityFileBuilder(bForceIntoSingleUnityFile ? -1 : BuildConfiguration.NumIncludedBytesPerUnityCPP);
foreach (var CPPFile in CPPFiles)
{
if (!bForceIntoSingleUnityFile && CPPFile.AbsolutePath.Contains(".GeneratedWrapper."))
{
CPPUnityFileBuilder.EndCurrentUnityFile();
CPPUnityFileBuilder.AddFile(CPPFile);
CPPUnityFileBuilder.EndCurrentUnityFile();
}
else
{
CPPUnityFileBuilder.AddFile(CPPFile);
}
// Now that the CPPFile is part of this unity file, we will no longer need to treat it like a root level prerequisite for our
// dependency cache, as it is now an "indirect include" from the unity file. We'll clear out the compile environment
// attached to this file. This prevents us from having to cache all of the indirect includes from these files inside our
// dependency cache, which speeds up iterative builds a lot!
CPPFile.CachedCPPIncludeInfo = null;
}
AllUnityFiles = CPPUnityFileBuilder.GetUnityFiles();
}
//THIS CHANGE WAS MADE TO FIX THE BUILD IN OUR BRANCH
//DO NOT MERGE THIS BACK TO MAIN
string PCHHeaderNameInCode = CPPFiles.Count > 0 ? CPPFiles[0].PCHHeaderNameInCode : "";
if (CompileEnvironment.Config.PrecompiledHeaderIncludeFilename != null)
{
PCHHeaderNameInCode = ToolChain.ConvertPath(CompileEnvironment.Config.PrecompiledHeaderIncludeFilename);
// Generated unity .cpp files always include the PCH using an absolute path, so we need to update
// our compile environment's PCH header name to use this instead of the text it pulled from the original
// C++ source files
CompileEnvironment.Config.PCHHeaderNameInCode = PCHHeaderNameInCode;
}
// Create a set of CPP files that combine smaller CPP files into larger compilation units, along with the corresponding
// actions to compile them.
int CurrentUnityFileCount = 0;
var UnityCPPFiles = new List <FileItem>();
foreach (var UnityFile in AllUnityFiles)
{
++CurrentUnityFileCount;
StringWriter OutputUnityCPPWriter = new StringWriter();
StringWriter OutputUnityCPPWriterExtra = null;
// add an extra file for UBT to get the #include dependencies from
if (BuildPlatform.RequiresExtraUnityCPPWriter() == true)
{
OutputUnityCPPWriterExtra = new StringWriter();
}
OutputUnityCPPWriter.WriteLine("// This file is automatically generated at compile-time to include some subset of the user-created cpp files.");
// Explicitly include the precompiled header first, since Visual C++ expects the first top-level #include to be the header file
// that was used to create the PCH.
if (CompileEnvironment.Config.PrecompiledHeaderIncludeFilename != null)
{
OutputUnityCPPWriter.WriteLine("#include \"{0}\"", PCHHeaderNameInCode);
if (OutputUnityCPPWriterExtra != null)
{
OutputUnityCPPWriterExtra.WriteLine("#include \"{0}\"", PCHHeaderNameInCode);
}
}
// Add source files to the unity file
foreach (var CPPFile in UnityFile.Files)
{
OutputUnityCPPWriter.WriteLine("#include \"{0}\"", ToolChain.ConvertPath(CPPFile.AbsolutePath));
if (OutputUnityCPPWriterExtra != null)
{
OutputUnityCPPWriterExtra.WriteLine("#include \"{0}\"", CPPFile.AbsolutePath);
}
}
// Determine unity file path name
string UnityCPPFilePath;
if (AllUnityFiles.Count > 1)
{
UnityCPPFilePath = string.Format("Module.{0}.{1}_of_{2}.cpp", BaseName, CurrentUnityFileCount, AllUnityFiles.Count);
}
else
{
UnityCPPFilePath = string.Format("Module.{0}.cpp", BaseName);
}
UnityCPPFilePath = Path.Combine(CompileEnvironment.Config.OutputDirectory, UnityCPPFilePath);
// Write the unity file to the intermediate folder.
FileItem UnityCPPFile = FileItem.CreateIntermediateTextFile(UnityCPPFilePath, OutputUnityCPPWriter.ToString());
if (OutputUnityCPPWriterExtra != null)
{
FileItem.CreateIntermediateTextFile(UnityCPPFilePath + ".ex", OutputUnityCPPWriterExtra.ToString());
}
UnityCPPFile.RelativeCost = UnityFile.TotalLength;
UnityCPPFile.PCHHeaderNameInCode = PCHHeaderNameInCode;
UnityCPPFiles.Add(UnityCPPFile);
// Cache information about the unity .cpp dependencies
// @todo ubtmake urgent: Fails when building remotely for Mac because unity .cpp has an include for a PCH on the REMOTE machine
UEBuildModuleCPP.CachePCHUsageForModuleSourceFile(Target, CompileEnvironment, UnityCPPFile);
}
return(UnityCPPFiles);
}
/// <summary>
/// Given a set of C++ files, generates another set of C++ files that #include all the original
/// files, the goal being to compile the same code in fewer translation units.
/// The "unity" files are written to the CompileEnvironment's OutputDirectory.
/// </summary>
/// <param name="Target">The target we're building</param>
/// <param name="CPPFiles">The C++ files to #include.</param>
/// <param name="CompileEnvironment">The environment that is used to compile the C++ files.</param>
/// <param name="BaseName">Base name to use for the Unity files</param>
/// <returns>The "unity" C++ files.</returns>
public static List <FileItem> GenerateUnityCPPs(
UEToolChain ToolChain,
UEBuildTarget Target,
List <FileItem> CPPFiles,
CPPEnvironment CompileEnvironment,
string BaseName
)
{
List <FileItem> NewCPPFiles = new List <FileItem>();
UEBuildPlatform BuildPlatform = UEBuildPlatform.GetBuildPlatformForCPPTargetPlatform(CompileEnvironment.Config.Platform);
// Figure out size of all input files combined. We use this to determine whether to use larger unity threshold or not.
long TotalBytesInCPPFiles = CPPFiles.Sum(F => F.Info.Length);
// We have an increased threshold for unity file size if, and only if, all files fit into the same unity file. This
// is beneficial when dealing with PCH files. The default PCH creation limit is X unity files so if we generate < X
// this could be fairly slow and we'd rather bump the limit a bit to group them all into the same unity file.
// When enabled, UnrealBuildTool will try to determine source files that you are actively iteratively changing, and break those files
// out of their unity blobs so that you can compile them as individual translation units, much faster than recompiling the entire
// unity blob each time.
bool bUseAdaptiveUnityBuild = BuildConfiguration.bUseAdaptiveUnityBuild && !BuildConfiguration.bStressTestUnity;
// Optimization only makes sense if PCH files are enabled.
bool bForceIntoSingleUnityFile = BuildConfiguration.bStressTestUnity || (TotalBytesInCPPFiles < BuildConfiguration.NumIncludedBytesPerUnityCPP * 2 && CompileEnvironment.ShouldUsePCHs());
// Build the list of unity files.
List <FileCollection> AllUnityFiles;
{
// Sort the incoming file paths alphabetically, so there will be consistency in unity blobs across multiple machines.
// Note that we're relying on this not only sorting files within each directory, but also the directories
// themselves, so the whole list of file paths is the same across computers.
List <FileItem> SortedCPPFiles = CPPFiles.GetRange(0, CPPFiles.Count);
{
// Case-insensitive file path compare, because you never know what is going on with local file systems
Comparison <FileItem> FileItemComparer = (FileA, FileB) => { return(FileA.AbsolutePath.ToLowerInvariant().CompareTo(FileB.AbsolutePath.ToLowerInvariant())); };
SortedCPPFiles.Sort(FileItemComparer);
}
// Figure out whether we REALLY want to use adaptive unity for this module. If nearly every file in the module appears in the working
// set, we'll just go ahead and let unity build do its thing.
if (bUseAdaptiveUnityBuild)
{
int CandidateWorkingSetSourceFileCount = 0;
int WorkingSetSourceFileCount = 0;
foreach (FileItem CPPFile in SortedCPPFiles)
{
// Don't include writable source files into unity blobs
if (!CPPFile.Reference.IsUnderDirectory(Target.EngineIntermediateDirectory) &&
!CPPFile.Reference.IsUnderDirectory(Target.ProjectIntermediateDirectory))
{
++CandidateWorkingSetSourceFileCount;
if (UnrealBuildTool.ShouldSourceFileBePartOfWorkingSet(CPPFile.AbsolutePath))
{
++WorkingSetSourceFileCount;
// Mark this file as part of the working set. This will be saved into the UBT Makefile so that
// the assembler can automatically invalidate the Makefile when the working set changes (allowing this
// code to run again, to build up new unity blobs.)
SourceFileWorkingSet.Add(CPPFile);
}
}
}
if (WorkingSetSourceFileCount >= CandidateWorkingSetSourceFileCount)
{
// Every single file in the module appears in the working set, so don't bother using adaptive unity for this
// module. Otherwise it would make full builds really slow.
bUseAdaptiveUnityBuild = false;
}
}
UnityFileBuilder CPPUnityFileBuilder = new UnityFileBuilder(bForceIntoSingleUnityFile ? -1 : BuildConfiguration.NumIncludedBytesPerUnityCPP);
StringBuilder AdaptiveUnityBuildInfoString = new StringBuilder();
foreach (FileItem CPPFile in SortedCPPFiles)
{
if (!bForceIntoSingleUnityFile && CPPFile.AbsolutePath.IndexOf(".GeneratedWrapper.", StringComparison.InvariantCultureIgnoreCase) != -1)
{
NewCPPFiles.Add(CPPFile);
}
// When adaptive unity is enabled, go ahead and exclude any source files that we're actively working with
if (bUseAdaptiveUnityBuild && SourceFileWorkingSet.Contains(CPPFile))
{
// Just compile this file normally, not as part of the unity blob
NewCPPFiles.Add(CPPFile);
// Let the unity file builder know about the file, so that we can retain the existing size of the unity blobs.
// This won't actually make the source file part of the unity blob, but it will keep track of how big the
// file is so that other existing unity blobs from the same module won't be invalidated. This prevents much
// longer compile times the first time you build after your working file set changes.
CPPUnityFileBuilder.AddVirtualFile(CPPFile);
string CPPFileName = Path.GetFileName(CPPFile.AbsolutePath);
if (AdaptiveUnityBuildInfoString.Length == 0)
{
AdaptiveUnityBuildInfoString.Append(String.Format("[Adaptive unity build] Excluded from {0} unity file: {1}", BaseName, CPPFileName));
}
else
{
AdaptiveUnityBuildInfoString.Append(", " + CPPFileName);
}
}
else
{
// If adaptive unity build is enabled for this module, add this source file to the set that will invalidate the makefile
if (bUseAdaptiveUnityBuild)
{
CandidateSourceFilesForWorkingSet.Add(CPPFile);
}
// Compile this file as part of the unity blob
CPPUnityFileBuilder.AddFile(CPPFile);
// Now that the CPPFile is part of this unity file, we will no longer need to treat it like a root level prerequisite for our
// dependency cache, as it is now an "indirect include" from the unity file. We'll clear out the compile environment
// attached to this file. This prevents us from having to cache all of the indirect includes from these files inside our
// dependency cache, which speeds up iterative builds a lot!
CPPFile.CachedCPPIncludeInfo = null;
}
}
if (AdaptiveUnityBuildInfoString.Length > 0)
{
Log.TraceInformation(AdaptiveUnityBuildInfoString.ToString());
}
AllUnityFiles = CPPUnityFileBuilder.GetUnityFiles();
}
// Create a set of CPP files that combine smaller CPP files into larger compilation units, along with the corresponding
// actions to compile them.
int CurrentUnityFileCount = 0;
foreach (FileCollection UnityFile in AllUnityFiles)
{
++CurrentUnityFileCount;
StringWriter OutputUnityCPPWriter = new StringWriter();
StringWriter OutputUnityCPPWriterExtra = null;
// add an extra file for UBT to get the #include dependencies from
if (BuildPlatform.RequiresExtraUnityCPPWriter() == true)
{
OutputUnityCPPWriterExtra = new StringWriter();
}
OutputUnityCPPWriter.WriteLine("// This file is automatically generated at compile-time to include some subset of the user-created cpp files.");
// Add source files to the unity file
foreach (FileItem CPPFile in UnityFile.Files)
{
OutputUnityCPPWriter.WriteLine("#include \"{0}\"", ToolChain.ConvertPath(CPPFile.AbsolutePath));
if (OutputUnityCPPWriterExtra != null)
{
OutputUnityCPPWriterExtra.WriteLine("#include \"{0}\"", CPPFile.AbsolutePath);
}
}
// Determine unity file path name
string UnityCPPFileName;
if (AllUnityFiles.Count > 1)
{
UnityCPPFileName = string.Format("{0}{1}.{2}_of_{3}.cpp", ModulePrefix, BaseName, CurrentUnityFileCount, AllUnityFiles.Count);
}
else
{
UnityCPPFileName = string.Format("{0}{1}.cpp", ModulePrefix, BaseName);
}
FileReference UnityCPPFilePath = FileReference.Combine(CompileEnvironment.Config.OutputDirectory, UnityCPPFileName);
// Write the unity file to the intermediate folder.
FileItem UnityCPPFile = FileItem.CreateIntermediateTextFile(UnityCPPFilePath, OutputUnityCPPWriter.ToString());
if (OutputUnityCPPWriterExtra != null)
{
FileItem.CreateIntermediateTextFile(UnityCPPFilePath + ".ex", OutputUnityCPPWriterExtra.ToString());
}
UnityCPPFile.RelativeCost = UnityFile.TotalLength;
NewCPPFiles.Add(UnityCPPFile);
// Cache information about the unity .cpp dependencies
// @todo ubtmake urgent: Fails when building remotely for Mac because unity .cpp has an include for a PCH on the REMOTE machine
UEBuildModuleCPP.CachePCHUsageForModuleSourceFile(CompileEnvironment, UnityCPPFile);
}
return(NewCPPFiles);
}
/// <summary>
/// Checks if the makefile is valid for the current set of source files. This is done separately to the Load() method to allow pre-build steps to modify source files.
/// </summary>
/// <param name="Makefile">The makefile that has been loaded</param>
/// <param name="ProjectFile">Path to the project file</param>
/// <param name="WorkingSet">The current working set of source files</param>
/// <param name="ReasonNotLoaded">If the makefile is not valid, is set to a message describing why</param>
/// <returns>True if the makefile is valid, false otherwise</returns>
public static bool IsValidForSourceFiles(TargetMakefile Makefile, FileReference ProjectFile, ISourceFileWorkingSet WorkingSet, out string ReasonNotLoaded)
{
using (Timeline.ScopeEvent("TargetMakefile.IsValidForSourceFiles()"))
{
// Check if any source files have been added or removed
foreach (KeyValuePair <DirectoryItem, FileItem[]> Pair in Makefile.DirectoryToSourceFiles)
{
DirectoryItem InputDirectory = Pair.Key;
if (!InputDirectory.Exists || InputDirectory.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
FileItem[] SourceFiles = UEBuildModuleCPP.GetSourceFiles(InputDirectory);
if (SourceFiles.Length < Pair.Value.Length)
{
ReasonNotLoaded = "source file removed";
return(false);
}
else if (SourceFiles.Length > Pair.Value.Length)
{
ReasonNotLoaded = "source file added";
return(false);
}
else if (SourceFiles.Intersect(Pair.Value).Count() != SourceFiles.Length)
{
ReasonNotLoaded = "source file modified";
return(false);
}
}
}
// Check if any of the additional dependencies has changed
foreach (FileItem AdditionalDependency in Makefile.AdditionalDependencies)
{
if (!AdditionalDependency.Exists)
{
Log.TraceLog("{0} has been deleted since makefile was built.", AdditionalDependency.Location);
ReasonNotLoaded = string.Format("{0} deleted", AdditionalDependency.Location.GetFileName());
return(false);
}
if (AdditionalDependency.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
Log.TraceLog("{0} has been modified since makefile was built.", AdditionalDependency.Location);
ReasonNotLoaded = string.Format("{0} modified", AdditionalDependency.Location.GetFileName());
return(false);
}
}
// Check that no new plugins have been added
foreach (FileReference PluginFile in Plugins.EnumeratePlugins(ProjectFile))
{
FileItem PluginFileItem = FileItem.GetItemByFileReference(PluginFile);
if (!Makefile.PluginFiles.Contains(PluginFileItem))
{
Log.TraceLog("{0} has been added", PluginFile.GetFileName());
ReasonNotLoaded = string.Format("{0} has been added", PluginFile.GetFileName());
return(false);
}
}
// We do a check to see if any modules' headers have changed which have
// acquired or lost UHT types. If so, which should be rare,
// we'll just invalidate the entire makefile and force it to be rebuilt.
// Get all H files in processed modules newer than the makefile itself
HashSet <FileItem> HFilesNewerThanMakefile = new HashSet <FileItem>();
foreach (UHTModuleHeaderInfo ModuleHeaderInfo in Makefile.UObjectModuleHeaders)
{
foreach (FileItem HeaderFile in ModuleHeaderInfo.SourceFolder.EnumerateFiles())
{
if (HeaderFile.HasExtension(".h") && HeaderFile.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
HFilesNewerThanMakefile.Add(HeaderFile);
}
}
}
// Get all H files in all modules processed in the last makefile build
HashSet <FileItem> AllUHTHeaders = new HashSet <FileItem>(Makefile.UObjectModuleHeaders.SelectMany(x => x.HeaderFiles));
// Check whether any headers have been deleted. If they have, we need to regenerate the makefile since the module might now be empty. If we don't,
// and the file has been moved to a different module, we may include stale generated headers.
foreach (FileItem HeaderFile in AllUHTHeaders)
{
if (!HeaderFile.Exists)
{
Log.TraceLog("File processed by UHT was deleted ({0}); invalidating makefile", HeaderFile);
ReasonNotLoaded = string.Format("UHT file was deleted");
return(false);
}
}
// Makefile is invalid if:
// * There are any newer files which contain no UHT data, but were previously in the makefile
// * There are any newer files contain data which needs processing by UHT, but weren't not previously in the makefile
SourceFileMetadataCache MetadataCache = SourceFileMetadataCache.CreateHierarchy(ProjectFile);
foreach (FileItem HeaderFile in HFilesNewerThanMakefile)
{
bool bContainsUHTData = MetadataCache.ContainsReflectionMarkup(HeaderFile);
bool bWasProcessed = AllUHTHeaders.Contains(HeaderFile);
if (bContainsUHTData != bWasProcessed)
{
Log.TraceLog("{0} {1} contain UHT types and now {2} , ignoring it", HeaderFile, bWasProcessed ? "used to" : "didn't", bWasProcessed ? "doesn't" : "does");
ReasonNotLoaded = string.Format("new files with reflected types");
return(false);
}
}
// If adaptive unity build is enabled, do a check to see if there are any source files that became part of the
// working set since the Makefile was created (or, source files were removed from the working set.) If anything
// changed, then we'll force a new Makefile to be created so that we have fresh unity build blobs. We always
// want to make sure that source files in the working set are excluded from those unity blobs (for fastest possible
// iteration times.)
// Check if any source files in the working set no longer belong in it
foreach (FileItem SourceFile in Makefile.WorkingSet)
{
if (!WorkingSet.Contains(SourceFile) && SourceFile.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
Log.TraceLog("{0} was part of source working set and now is not; invalidating makefile", SourceFile.AbsolutePath);
ReasonNotLoaded = string.Format("working set of source files changed");
return(false);
}
}
// Check if any source files that are eligible for being in the working set have been modified
foreach (FileItem SourceFile in Makefile.CandidatesForWorkingSet)
{
if (WorkingSet.Contains(SourceFile) && SourceFile.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
Log.TraceLog("{0} was part of source working set and now is not", SourceFile.AbsolutePath);
ReasonNotLoaded = string.Format("working set of source files changed");
return(false);
}
}
}
ReasonNotLoaded = null;
return(true);
}
/// <summary>
/// Checks if the makefile is valid for the current set of source files. This is done separately to the Load() method to allow pre-build steps to modify source files.
/// </summary>
/// <param name="Makefile">The makefile that has been loaded</param>
/// <param name="ProjectFile">Path to the project file</param>
/// <param name="Platform">The platform being built</param>
/// <param name="WorkingSet">The current working set of source files</param>
/// <param name="ReasonNotLoaded">If the makefile is not valid, is set to a message describing why</param>
/// <returns>True if the makefile is valid, false otherwise</returns>
public static bool IsValidForSourceFiles(TargetMakefile Makefile, FileReference ProjectFile, UnrealTargetPlatform Platform, ISourceFileWorkingSet WorkingSet, out string ReasonNotLoaded)
{
using (Timeline.ScopeEvent("TargetMakefile.IsValidForSourceFiles()"))
{
// Get the list of excluded folder names for this platform
ReadOnlyHashSet <string> ExcludedFolderNames = UEBuildPlatform.GetBuildPlatform(Platform).GetExcludedFolderNames();
// Check if any source files have been added or removed
foreach (KeyValuePair <DirectoryItem, FileItem[]> Pair in Makefile.DirectoryToSourceFiles)
{
DirectoryItem InputDirectory = Pair.Key;
if (!InputDirectory.Exists || InputDirectory.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
FileItem[] SourceFiles = UEBuildModuleCPP.GetSourceFiles(InputDirectory);
if (SourceFiles.Length < Pair.Value.Length)
{
ReasonNotLoaded = "source file removed";
return(false);
}
else if (SourceFiles.Length > Pair.Value.Length)
{
ReasonNotLoaded = "source file added";
return(false);
}
else if (SourceFiles.Intersect(Pair.Value).Count() != SourceFiles.Length)
{
ReasonNotLoaded = "source file modified";
return(false);
}
foreach (DirectoryItem Directory in InputDirectory.EnumerateDirectories())
{
if (!Makefile.DirectoryToSourceFiles.ContainsKey(Directory) && ContainsSourceFiles(Directory, ExcludedFolderNames))
{
ReasonNotLoaded = "directory added";
return(false);
}
}
}
}
// Check if any external dependencies have changed. These comparisons are done against the makefile creation time.
foreach (FileItem ExternalDependency in Makefile.ExternalDependencies)
{
if (!ExternalDependency.Exists)
{
Log.TraceLog("{0} has been deleted since makefile was built.", ExternalDependency.Location);
ReasonNotLoaded = string.Format("{0} deleted", ExternalDependency.Location.GetFileName());
return(false);
}
if (ExternalDependency.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
Log.TraceLog("{0} has been modified since makefile was built.", ExternalDependency.Location);
ReasonNotLoaded = string.Format("{0} modified", ExternalDependency.Location.GetFileName());
return(false);
}
}
// Check if any internal dependencies has changed. These comparisons are done against the makefile modified time.
foreach (FileItem InternalDependency in Makefile.InternalDependencies)
{
if (!InternalDependency.Exists)
{
Log.TraceLog("{0} has been deleted since makefile was written.", InternalDependency.Location);
ReasonNotLoaded = string.Format("{0} deleted", InternalDependency.Location.GetFileName());
return(false);
}
if (InternalDependency.LastWriteTimeUtc > Makefile.ModifiedTimeUtc)
{
Log.TraceLog("{0} has been modified since makefile was written.", InternalDependency.Location);
ReasonNotLoaded = string.Format("{0} modified", InternalDependency.Location.GetFileName());
return(false);
}
}
// Check that no new plugins have been added
foreach (FileReference PluginFile in Plugins.EnumeratePlugins(ProjectFile))
{
FileItem PluginFileItem = FileItem.GetItemByFileReference(PluginFile);
if (!Makefile.PluginFiles.Contains(PluginFileItem))
{
Log.TraceLog("{0} has been added", PluginFile.GetFileName());
ReasonNotLoaded = string.Format("{0} has been added", PluginFile.GetFileName());
return(false);
}
}
// Load the metadata cache
SourceFileMetadataCache MetadataCache = SourceFileMetadataCache.CreateHierarchy(ProjectFile);
// Find the set of files that contain reflection markup
ConcurrentBag <FileItem> NewFilesWithMarkupBag = new ConcurrentBag <FileItem>();
using (ThreadPoolWorkQueue Queue = new ThreadPoolWorkQueue())
{
foreach (DirectoryItem SourceDirectory in Makefile.SourceDirectories)
{
Queue.Enqueue(() => FindFilesWithMarkup(SourceDirectory, MetadataCache, ExcludedFolderNames, NewFilesWithMarkupBag, Queue));
}
}
// Check whether the list has changed
List <FileItem> PrevFilesWithMarkup = Makefile.UObjectModuleHeaders.Where(x => !x.bUsePrecompiled).SelectMany(x => x.HeaderFiles).ToList();
List <FileItem> NextFilesWithMarkup = NewFilesWithMarkupBag.ToList();
if (NextFilesWithMarkup.Count != PrevFilesWithMarkup.Count || NextFilesWithMarkup.Intersect(PrevFilesWithMarkup).Count() != PrevFilesWithMarkup.Count)
{
ReasonNotLoaded = "UHT files changed";
return(false);
}
// If adaptive unity build is enabled, do a check to see if there are any source files that became part of the
// working set since the Makefile was created (or, source files were removed from the working set.) If anything
// changed, then we'll force a new Makefile to be created so that we have fresh unity build blobs. We always
// want to make sure that source files in the working set are excluded from those unity blobs (for fastest possible
// iteration times.)
// Check if any source files in the working set no longer belong in it
foreach (FileItem SourceFile in Makefile.WorkingSet)
{
if (!WorkingSet.Contains(SourceFile) && SourceFile.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
Log.TraceLog("{0} was part of source working set and now is not; invalidating makefile", SourceFile.AbsolutePath);
ReasonNotLoaded = string.Format("working set of source files changed");
return(false);
}
}
// Check if any source files that are eligible for being in the working set have been modified
foreach (FileItem SourceFile in Makefile.CandidatesForWorkingSet)
{
if (WorkingSet.Contains(SourceFile) && SourceFile.LastWriteTimeUtc > Makefile.CreateTimeUtc)
{
Log.TraceLog("{0} was part of source working set and now is not", SourceFile.AbsolutePath);
ReasonNotLoaded = string.Format("working set of source files changed");
return(false);
}
}
}
ReasonNotLoaded = null;
return(true);
}
/// <summary>
/// Write a Module to a JSON writer. If array is empty, don't write anything
/// </summary>
/// <param name="BinaryOutputDir"></param>
/// <param name="TargetOutputDir"></param>
/// <param name="Writer">Writer for the array data</param>
/// <param name="Module"></param>
private static void ExportModule(UEBuildModule Module, DirectoryReference BinaryOutputDir, DirectoryReference TargetOutputDir, JsonWriter Writer)
{
Writer.WriteValue("Name", Module.Name);
Writer.WriteValue("Directory", Module.ModuleDirectory.FullName);
Writer.WriteValue("Rules", Module.RulesFile.FullName);
Writer.WriteValue("PCHUsage", Module.Rules.PCHUsage.ToString());
UEBuildModuleCPP ModuleCPP = Module as UEBuildModuleCPP;
if (ModuleCPP != null)
{
Writer.WriteValue("GeneratedCodeDirectory", ModuleCPP.GeneratedCodeDirectory != null ? ModuleCPP.GeneratedCodeDirectory.FullName : string.Empty);
}
if (Module.Rules.PrivatePCHHeaderFile != null)
{
Writer.WriteValue("PrivatePCH", FileReference.Combine(Module.ModuleDirectory, Module.Rules.PrivatePCHHeaderFile).FullName);
}
if (Module.Rules.SharedPCHHeaderFile != null)
{
Writer.WriteValue("SharedPCH", FileReference.Combine(Module.ModuleDirectory, Module.Rules.SharedPCHHeaderFile).FullName);
}
ExportJsonModuleArray(Writer, "PublicDependencyModules", Module.PublicDependencyModules);
ExportJsonModuleArray(Writer, "PublicIncludePathModules", Module.PublicIncludePathModules);
ExportJsonModuleArray(Writer, "PrivateDependencyModules", Module.PrivateDependencyModules);
ExportJsonModuleArray(Writer, "PrivateIncludePathModules", Module.PrivateIncludePathModules);
ExportJsonModuleArray(Writer, "DynamicallyLoadedModules", Module.DynamicallyLoadedModules);
ExportJsonStringArray(Writer, "PublicSystemIncludePaths", Module.PublicSystemIncludePaths.Select(x => x.FullName));
ExportJsonStringArray(Writer, "PublicIncludePaths", Module.PublicIncludePaths.Select(x => x.FullName));
ExportJsonStringArray(Writer, "LegacyPublicIncludePaths", Module.LegacyPublicIncludePaths.Select(x => x.FullName));
ExportJsonStringArray(Writer, "PrivateIncludePaths", Module.PrivateIncludePaths.Select(x => x.FullName));
ExportJsonStringArray(Writer, "PublicLibraryPaths", Module.PublicSystemLibraryPaths.Select(x => x.FullName));
ExportJsonStringArray(Writer, "PublicAdditionalLibraries", Module.PublicAdditionalLibraries);
ExportJsonStringArray(Writer, "PublicFrameworks", Module.PublicFrameworks);
ExportJsonStringArray(Writer, "PublicWeakFrameworks", Module.PublicWeakFrameworks);
ExportJsonStringArray(Writer, "PublicDelayLoadDLLs", Module.PublicDelayLoadDLLs);
ExportJsonStringArray(Writer, "PublicDefinitions", Module.PublicDefinitions);
ExportJsonStringArray(Writer, "PrivateDefinitions", Module.Rules.PrivateDefinitions);
ExportJsonStringArray(Writer, "ProjectDefinitions", /* TODO: Add method ShouldAddProjectDefinitions */ !Module.Rules.bTreatAsEngineModule ? Module.Rules.Target.ProjectDefinitions : new string[0]);
ExportJsonStringArray(Writer, "ApiDefinitions", Module.GetEmptyApiMacros());
Writer.WriteValue("ShouldAddLegacyPublicIncludePaths", Module.Rules.bLegacyPublicIncludePaths);
if (Module.Rules.CircularlyReferencedDependentModules.Any())
{
Writer.WriteArrayStart("CircularlyReferencedModules");
foreach (string ModuleName in Module.Rules.CircularlyReferencedDependentModules)
{
Writer.WriteValue(ModuleName);
}
Writer.WriteArrayEnd();
}
if (Module.Rules.RuntimeDependencies.Inner.Any())
{
// We don't use info from RuntimeDependencies for code analyzes (at the moment)
// So we're OK with skipping some values if they are not presented
Writer.WriteArrayStart("RuntimeDependencies");
foreach (ModuleRules.RuntimeDependency RuntimeDependency in Module.Rules.RuntimeDependencies.Inner)
{
Writer.WriteObjectStart();
try
{
Writer.WriteValue("Path",
Module.ExpandPathVariables(RuntimeDependency.Path, BinaryOutputDir, TargetOutputDir));
}
catch (BuildException buildException)
{
Log.TraceVerbose("Value {0} for module {1} will not be stored. Reason: {2}", "Path", Module.Name, buildException);
}
if (RuntimeDependency.SourcePath != null)
{
try
{
Writer.WriteValue("SourcePath",
Module.ExpandPathVariables(RuntimeDependency.SourcePath, BinaryOutputDir,
TargetOutputDir));
}
catch (BuildException buildException)
{
Log.TraceVerbose("Value {0} for module {1} will not be stored. Reason: {2}", "SourcePath", Module.Name, buildException);
}
}
Writer.WriteValue("Type", RuntimeDependency.Type.ToString());
Writer.WriteObjectEnd();
}
Writer.WriteArrayEnd();
}
}
