/// <summary>
/// Given a set of C++ files, generates another set of C++ files that #include all the original
/// files, the goal being to compile the same code in fewer translation units.
/// The "unity" files are written to the CompileEnvironment's OutputDirectory.
/// </summary>
/// <param name="Target">The target we're building</param>
/// <param name="CPPFiles">The C++ files to #include.</param>
/// <param name="CompileEnvironment">The environment that is used to compile the C++ files.</param>
/// <param name="BaseName">Base name to use for the Unity files</param>
/// <returns>The "unity" C++ files.</returns>
public static List <FileItem> GenerateUnityCPPs(
UEBuildTarget Target,
List <FileItem> CPPFiles,
CPPEnvironment CompileEnvironment,
string BaseName
)
{
var ToolChain = UEToolChain.GetPlatformToolChain(CompileEnvironment.Config.Target.Platform);
var BuildPlatform = UEBuildPlatform.GetBuildPlatformForCPPTargetPlatform(CompileEnvironment.Config.Target.Platform);
// Figure out size of all input files combined. We use this to determine whether to use larger unity threshold or not.
long TotalBytesInCPPFiles = CPPFiles.Sum(F => F.Info.Length);
// We have an increased threshold for unity file size if, and only if, all files fit into the same unity file. This
// is beneficial when dealing with PCH files. The default PCH creation limit is X unity files so if we generate < X
// this could be fairly slow and we'd rather bump the limit a bit to group them all into the same unity file.
//
// Optimization only makes sense if PCH files are enabled.
bool bForceIntoSingleUnityFile = BuildConfiguration.bStressTestUnity || (TotalBytesInCPPFiles < BuildConfiguration.NumIncludedBytesPerUnityCPP * 2 && CompileEnvironment.ShouldUsePCHs());
// Build the list of unity files.
List <FileCollection> AllUnityFiles;
{
var CPPUnityFileBuilder = new UnityFileBuilder(bForceIntoSingleUnityFile ? -1 : BuildConfiguration.NumIncludedBytesPerUnityCPP);
foreach (var CPPFile in CPPFiles)
{
if (!bForceIntoSingleUnityFile && CPPFile.AbsolutePath.Contains(".GeneratedWrapper."))
{
CPPUnityFileBuilder.EndCurrentUnityFile();
CPPUnityFileBuilder.AddFile(CPPFile);
CPPUnityFileBuilder.EndCurrentUnityFile();
}
else
{
CPPUnityFileBuilder.AddFile(CPPFile);
}
// Now that the CPPFile is part of this unity file, we will no longer need to treat it like a root level prerequisite for our
// dependency cache, as it is now an "indirect include" from the unity file. We'll clear out the compile environment
// attached to this file. This prevents us from having to cache all of the indirect includes from these files inside our
// dependency cache, which speeds up iterative builds a lot!
CPPFile.CachedCPPIncludeInfo = null;
}
AllUnityFiles = CPPUnityFileBuilder.GetUnityFiles();
}
//THIS CHANGE WAS MADE TO FIX THE BUILD IN OUR BRANCH
//DO NOT MERGE THIS BACK TO MAIN
string PCHHeaderNameInCode = CPPFiles.Count > 0 ? CPPFiles[0].PCHHeaderNameInCode : "";
if (CompileEnvironment.Config.PrecompiledHeaderIncludeFilename != null)
{
PCHHeaderNameInCode = ToolChain.ConvertPath(CompileEnvironment.Config.PrecompiledHeaderIncludeFilename);
// Generated unity .cpp files always include the PCH using an absolute path, so we need to update
// our compile environment's PCH header name to use this instead of the text it pulled from the original
// C++ source files
CompileEnvironment.Config.PCHHeaderNameInCode = PCHHeaderNameInCode;
}
// Create a set of CPP files that combine smaller CPP files into larger compilation units, along with the corresponding
// actions to compile them.
int CurrentUnityFileCount = 0;
var UnityCPPFiles = new List <FileItem>();
foreach (var UnityFile in AllUnityFiles)
{
++CurrentUnityFileCount;
StringWriter OutputUnityCPPWriter = new StringWriter();
StringWriter OutputUnityCPPWriterExtra = null;
// add an extra file for UBT to get the #include dependencies from
if (BuildPlatform.RequiresExtraUnityCPPWriter() == true)
{
OutputUnityCPPWriterExtra = new StringWriter();
}
OutputUnityCPPWriter.WriteLine("// This file is automatically generated at compile-time to include some subset of the user-created cpp files.");
// Explicitly include the precompiled header first, since Visual C++ expects the first top-level #include to be the header file
// that was used to create the PCH.
if (CompileEnvironment.Config.PrecompiledHeaderIncludeFilename != null)
{
OutputUnityCPPWriter.WriteLine("#include \"{0}\"", PCHHeaderNameInCode);
if (OutputUnityCPPWriterExtra != null)
{
OutputUnityCPPWriterExtra.WriteLine("#include \"{0}\"", PCHHeaderNameInCode);
}
}
// Add source files to the unity file
foreach (var CPPFile in UnityFile.Files)
{
OutputUnityCPPWriter.WriteLine("#include \"{0}\"", ToolChain.ConvertPath(CPPFile.AbsolutePath));
if (OutputUnityCPPWriterExtra != null)
{
OutputUnityCPPWriterExtra.WriteLine("#include \"{0}\"", CPPFile.AbsolutePath);
}
}
// Determine unity file path name
string UnityCPPFilePath;
if (AllUnityFiles.Count > 1)
{
UnityCPPFilePath = string.Format("Module.{0}.{1}_of_{2}.cpp", BaseName, CurrentUnityFileCount, AllUnityFiles.Count);
}
else
{
UnityCPPFilePath = string.Format("Module.{0}.cpp", BaseName);
}
UnityCPPFilePath = Path.Combine(CompileEnvironment.Config.OutputDirectory, UnityCPPFilePath);
// Write the unity file to the intermediate folder.
FileItem UnityCPPFile = FileItem.CreateIntermediateTextFile(UnityCPPFilePath, OutputUnityCPPWriter.ToString());
if (OutputUnityCPPWriterExtra != null)
{
FileItem.CreateIntermediateTextFile(UnityCPPFilePath + ".ex", OutputUnityCPPWriterExtra.ToString());
}
UnityCPPFile.RelativeCost = UnityFile.TotalLength;
UnityCPPFile.PCHHeaderNameInCode = PCHHeaderNameInCode;
UnityCPPFiles.Add(UnityCPPFile);
// Cache information about the unity .cpp dependencies
// @todo ubtmake urgent: Fails when building remotely for Mac because unity .cpp has an include for a PCH on the REMOTE machine
UEBuildModuleCPP.CachePCHUsageForModuleSourceFile(Target, CompileEnvironment, UnityCPPFile);
}
return(UnityCPPFiles);
}
/// <summary>
/// Given a set of C++ files, generates another set of C++ files that #include all the original
/// files, the goal being to compile the same code in fewer translation units.
/// The "unity" files are written to the CompileEnvironment's OutputDirectory.
/// </summary>
/// <param name="Target">The target we're building</param>
/// <param name="CPPFiles">The C++ files to #include.</param>
/// <param name="CompileEnvironment">The environment that is used to compile the C++ files.</param>
/// <param name="BaseName">Base name to use for the Unity files</param>
/// <returns>The "unity" C++ files.</returns>
public static List<FileItem> GenerateUnityCPPs(
UEBuildTarget Target,
List<FileItem> CPPFiles,
CPPEnvironment CompileEnvironment,
string BaseName
)
{
var ToolChain = UEToolChain.GetPlatformToolChain(CompileEnvironment.Config.Target.Platform);
var BuildPlatform = UEBuildPlatform.GetBuildPlatformForCPPTargetPlatform(CompileEnvironment.Config.Target.Platform);
// Figure out size of all input files combined. We use this to determine whether to use larger unity threshold or not.
long TotalBytesInCPPFiles = CPPFiles.Sum(F => F.Info.Length);
// We have an increased threshold for unity file size if, and only if, all files fit into the same unity file. This
// is beneficial when dealing with PCH files. The default PCH creation limit is X unity files so if we generate < X
// this could be fairly slow and we'd rather bump the limit a bit to group them all into the same unity file.
//
// Optimization only makes sense if PCH files are enabled.
bool bForceIntoSingleUnityFile = BuildConfiguration.bStressTestUnity || (TotalBytesInCPPFiles < BuildConfiguration.NumIncludedBytesPerUnityCPP * 2 && CompileEnvironment.ShouldUsePCHs());
// Build the list of unity files.
List<FileCollection> AllUnityFiles;
{
var CPPUnityFileBuilder = new UnityFileBuilder(bForceIntoSingleUnityFile ? -1 : BuildConfiguration.NumIncludedBytesPerUnityCPP);
foreach( var CPPFile in CPPFiles )
{
if (!bForceIntoSingleUnityFile && CPPFile.AbsolutePath.Contains(".GeneratedWrapper."))
{
CPPUnityFileBuilder.EndCurrentUnityFile();
CPPUnityFileBuilder.AddFile(CPPFile);
CPPUnityFileBuilder.EndCurrentUnityFile();
}
else
{
CPPUnityFileBuilder.AddFile(CPPFile);
}
// Now that the CPPFile is part of this unity file, we will no longer need to treat it like a root level prerequisite for our
// dependency cache, as it is now an "indirect include" from the unity file. We'll clear out the compile environment
// attached to this file. This prevents us from having to cache all of the indirect includes from these files inside our
// dependency cache, which speeds up iterative builds a lot!
CPPFile.CachedCPPIncludeInfo = null;
}
AllUnityFiles = CPPUnityFileBuilder.GetUnityFiles();
}
//THIS CHANGE WAS MADE TO FIX THE BUILD IN OUR BRANCH
//DO NOT MERGE THIS BACK TO MAIN
string PCHHeaderNameInCode = CPPFiles.Count > 0 ? CPPFiles[0].PCHHeaderNameInCode : "";
if( CompileEnvironment.Config.PrecompiledHeaderIncludeFilename != null )
{
PCHHeaderNameInCode = ToolChain.ConvertPath( CompileEnvironment.Config.PrecompiledHeaderIncludeFilename );
// Generated unity .cpp files always include the PCH using an absolute path, so we need to update
// our compile environment's PCH header name to use this instead of the text it pulled from the original
// C++ source files
CompileEnvironment.Config.PCHHeaderNameInCode = PCHHeaderNameInCode;
}
// Create a set of CPP files that combine smaller CPP files into larger compilation units, along with the corresponding
// actions to compile them.
int CurrentUnityFileCount = 0;
var UnityCPPFiles = new List<FileItem>();
foreach( var UnityFile in AllUnityFiles )
{
++CurrentUnityFileCount;
StringWriter OutputUnityCPPWriter = new StringWriter();
StringWriter OutputUnityCPPWriterExtra = null;
// add an extra file for UBT to get the #include dependencies from
if (BuildPlatform.RequiresExtraUnityCPPWriter() == true)
{
OutputUnityCPPWriterExtra = new StringWriter();
}
OutputUnityCPPWriter.WriteLine("// This file is automatically generated at compile-time to include some subset of the user-created cpp files.");
// Explicitly include the precompiled header first, since Visual C++ expects the first top-level #include to be the header file
// that was used to create the PCH.
if (CompileEnvironment.Config.PrecompiledHeaderIncludeFilename != null)
{
OutputUnityCPPWriter.WriteLine("#include \"{0}\"", PCHHeaderNameInCode);
if (OutputUnityCPPWriterExtra != null)
{
OutputUnityCPPWriterExtra.WriteLine("#include \"{0}\"", PCHHeaderNameInCode);
}
}
// Add source files to the unity file
foreach( var CPPFile in UnityFile.Files )
{
OutputUnityCPPWriter.WriteLine("#include \"{0}\"", ToolChain.ConvertPath(CPPFile.AbsolutePath));
if (OutputUnityCPPWriterExtra != null)
{
OutputUnityCPPWriterExtra.WriteLine("#include \"{0}\"", CPPFile.AbsolutePath);
}
}
// Determine unity file path name
string UnityCPPFilePath;
if (AllUnityFiles.Count > 1)
{
UnityCPPFilePath = string.Format("Module.{0}.{1}_of_{2}.cpp", BaseName, CurrentUnityFileCount, AllUnityFiles.Count);
}
else
{
UnityCPPFilePath = string.Format("Module.{0}.cpp", BaseName);
}
UnityCPPFilePath = Path.Combine(CompileEnvironment.Config.OutputDirectory, UnityCPPFilePath);
// Write the unity file to the intermediate folder.
FileItem UnityCPPFile = FileItem.CreateIntermediateTextFile(UnityCPPFilePath, OutputUnityCPPWriter.ToString());
if (OutputUnityCPPWriterExtra != null)
{
FileItem.CreateIntermediateTextFile(UnityCPPFilePath + ".ex", OutputUnityCPPWriterExtra.ToString());
}
UnityCPPFile.RelativeCost = UnityFile.TotalLength;
UnityCPPFile.PCHHeaderNameInCode = PCHHeaderNameInCode;
UnityCPPFiles.Add(UnityCPPFile);
// Cache information about the unity .cpp dependencies
// @todo ubtmake urgent: Fails when building remotely for Mac because unity .cpp has an include for a PCH on the REMOTE machine
UEBuildModuleCPP.CachePCHUsageForModuleSourceFile( Target, CompileEnvironment, UnityCPPFile );
}
return UnityCPPFiles;
}
/// <summary>
/// Given a set of C++ files, generates another set of C++ files that #include all the original
/// files, the goal being to compile the same code in fewer translation units.
/// The "unity" files are written to the CompileEnvironment's OutputDirectory.
/// </summary>
/// <param name="Target">The target we're building</param>
/// <param name="CPPFiles">The C++ files to #include.</param>
/// <param name="CompileEnvironment">The environment that is used to compile the C++ files.</param>
/// <param name="WorkingSet">Interface to query files which belong to the working set</param>
/// <param name="BaseName">Base name to use for the Unity files</param>
/// <param name="IntermediateDirectory">Intermediate directory for unity cpp files</param>
/// <param name="Makefile">The makefile being built</param>
/// <param name="SourceFileToUnityFile">Receives a mapping of source file to unity file</param>
/// <returns>The "unity" C++ files.</returns>
public static List <FileItem> GenerateUnityCPPs(
ReadOnlyTargetRules Target,
List <FileItem> CPPFiles,
CppCompileEnvironment CompileEnvironment,
ISourceFileWorkingSet WorkingSet,
string BaseName,
DirectoryReference IntermediateDirectory,
TargetMakefile Makefile,
Dictionary <FileItem, FileItem> SourceFileToUnityFile
)
{
List <FileItem> NewCPPFiles = new List <FileItem>();
UEBuildPlatform BuildPlatform = UEBuildPlatform.GetBuildPlatform(CompileEnvironment.Platform);
// Figure out size of all input files combined. We use this to determine whether to use larger unity threshold or not.
long TotalBytesInCPPFiles = CPPFiles.Sum(F => F.Length);
// We have an increased threshold for unity file size if, and only if, all files fit into the same unity file. This
// is beneficial when dealing with PCH files. The default PCH creation limit is X unity files so if we generate < X
// this could be fairly slow and we'd rather bump the limit a bit to group them all into the same unity file.
// When enabled, UnrealBuildTool will try to determine source files that you are actively iteratively changing, and break those files
// out of their unity blobs so that you can compile them as individual translation units, much faster than recompiling the entire
// unity blob each time.
bool bUseAdaptiveUnityBuild = Target.bUseAdaptiveUnityBuild && !Target.bStressTestUnity;
// Optimization only makes sense if PCH files are enabled.
bool bForceIntoSingleUnityFile = Target.bStressTestUnity || (TotalBytesInCPPFiles < Target.NumIncludedBytesPerUnityCPP * 2 && Target.bUsePCHFiles);
// Build the list of unity files.
List <FileCollection> AllUnityFiles;
{
// Sort the incoming file paths alphabetically, so there will be consistency in unity blobs across multiple machines.
// Note that we're relying on this not only sorting files within each directory, but also the directories
// themselves, so the whole list of file paths is the same across computers.
List <FileItem> SortedCPPFiles = CPPFiles.GetRange(0, CPPFiles.Count);
{
// Case-insensitive file path compare, because you never know what is going on with local file systems
Comparison <FileItem> FileItemComparer = (FileA, FileB) => { return(FileA.AbsolutePath.ToLowerInvariant().CompareTo(FileB.AbsolutePath.ToLowerInvariant())); };
SortedCPPFiles.Sort(FileItemComparer);
}
// Figure out whether we REALLY want to use adaptive unity for this module. If nearly every file in the module appears in the working
// set, we'll just go ahead and let unity build do its thing.
if (bUseAdaptiveUnityBuild)
{
int CandidateWorkingSetSourceFileCount = 0;
int WorkingSetSourceFileCount = 0;
foreach (FileItem CPPFile in SortedCPPFiles)
{
++CandidateWorkingSetSourceFileCount;
// Don't include writable source files into unity blobs
if (WorkingSet.Contains(CPPFile))
{
++WorkingSetSourceFileCount;
// Mark this file as part of the working set. This will be saved into the UBT Makefile so that
// the assembler can automatically invalidate the Makefile when the working set changes (allowing this
// code to run again, to build up new unity blobs.)
Makefile.WorkingSet.Add(CPPFile);
}
}
if (WorkingSetSourceFileCount >= CandidateWorkingSetSourceFileCount)
{
// Every single file in the module appears in the working set, so don't bother using adaptive unity for this
// module. Otherwise it would make full builds really slow.
bUseAdaptiveUnityBuild = false;
}
}
UnityFileBuilder CPPUnityFileBuilder = new UnityFileBuilder(bForceIntoSingleUnityFile ? -1 : Target.NumIncludedBytesPerUnityCPP);
StringBuilder AdaptiveUnityBuildInfoString = new StringBuilder();
foreach (FileItem CPPFile in SortedCPPFiles)
{
if (!bForceIntoSingleUnityFile && CPPFile.AbsolutePath.IndexOf(".GeneratedWrapper.", StringComparison.InvariantCultureIgnoreCase) != -1)
{
NewCPPFiles.Add(CPPFile);
}
// When adaptive unity is enabled, go ahead and exclude any source files that we're actively working with
if (bUseAdaptiveUnityBuild && Makefile.WorkingSet.Contains(CPPFile))
{
// Just compile this file normally, not as part of the unity blob
NewCPPFiles.Add(CPPFile);
// Let the unity file builder know about the file, so that we can retain the existing size of the unity blobs.
// This won't actually make the source file part of the unity blob, but it will keep track of how big the
// file is so that other existing unity blobs from the same module won't be invalidated. This prevents much
// longer compile times the first time you build after your working file set changes.
CPPUnityFileBuilder.AddVirtualFile(CPPFile);
string CPPFileName = Path.GetFileName(CPPFile.AbsolutePath);
if (AdaptiveUnityBuildInfoString.Length == 0)
{
AdaptiveUnityBuildInfoString.Append(String.Format("[Adaptive unity build] Excluded from {0} unity file: {1}", BaseName, CPPFileName));
}
else
{
AdaptiveUnityBuildInfoString.Append(", " + CPPFileName);
}
}
else
{
// If adaptive unity build is enabled for this module, add this source file to the set that will invalidate the makefile
if (bUseAdaptiveUnityBuild)
{
Makefile.CandidatesForWorkingSet.Add(CPPFile);
}
// Compile this file as part of the unity blob
CPPUnityFileBuilder.AddFile(CPPFile);
}
}
if (AdaptiveUnityBuildInfoString.Length > 0)
{
if (Target.bAdaptiveUnityCreatesDedicatedPCH)
{
AddUniqueDiagnostic(Makefile, "[Adaptive unity build] Creating dedicated PCH for each excluded file. Set bAdaptiveUnityCreatesDedicatedPCH to false in BuildConfiguration.xml to change this behavior.");
}
else if (Target.bAdaptiveUnityDisablesPCH)
{
AddUniqueDiagnostic(Makefile, "[Adaptive unity build] Disabling PCH for excluded files. Set bAdaptiveUnityDisablesPCH to false in BuildConfiguration.xml to change this behavior.");
}
if (Target.bAdaptiveUnityDisablesOptimizations)
{
AddUniqueDiagnostic(Makefile, "[Adaptive unity build] Disabling optimizations for excluded files. Set bAdaptiveUnityDisablesOptimizations to false in BuildConfiguration.xml to change this behavior.");
}
if (Target.bAdaptiveUnityEnablesEditAndContinue)
{
AddUniqueDiagnostic(Makefile, "[Adaptive unity build] Enabling Edit & Continue for excluded files. Set bAdaptiveUnityEnablesEditAndContinue to false in BuildConfiguration.xml to change this behavior.");
}
Makefile.Diagnostics.Add(AdaptiveUnityBuildInfoString.ToString());
}
AllUnityFiles = CPPUnityFileBuilder.GetUnityFiles();
}
// Create a set of CPP files that combine smaller CPP files into larger compilation units, along with the corresponding
// actions to compile them.
int CurrentUnityFileCount = 0;
foreach (FileCollection UnityFile in AllUnityFiles)
{
++CurrentUnityFileCount;
StringWriter OutputUnityCPPWriter = new StringWriter();
OutputUnityCPPWriter.WriteLine("// This file is automatically generated at compile-time to include some subset of the user-created cpp files.");
// Add source files to the unity file
foreach (FileItem CPPFile in UnityFile.Files)
{
OutputUnityCPPWriter.WriteLine("#include \"{0}\"", CPPFile.AbsolutePath.Replace('\\', '/'));
}
// Determine unity file path name
string UnityCPPFileName;
if (AllUnityFiles.Count > 1)
{
UnityCPPFileName = string.Format("{0}{1}.{2}_of_{3}.cpp", ModulePrefix, BaseName, CurrentUnityFileCount, AllUnityFiles.Count);
}
else
{
UnityCPPFileName = string.Format("{0}{1}.cpp", ModulePrefix, BaseName);
}
FileReference UnityCPPFilePath = FileReference.Combine(IntermediateDirectory, UnityCPPFileName);
// Write the unity file to the intermediate folder.
FileItem UnityCPPFile = FileItem.CreateIntermediateTextFile(UnityCPPFilePath, OutputUnityCPPWriter.ToString());
NewCPPFiles.Add(UnityCPPFile);
// Store the mapping of source files to unity files in the makefile
foreach (FileItem SourceFile in UnityFile.Files)
{
SourceFileToUnityFile[SourceFile] = UnityCPPFile;
}
foreach (FileItem SourceFile in UnityFile.VirtualFiles)
{
SourceFileToUnityFile[SourceFile] = UnityCPPFile;
}
}
return(NewCPPFiles);
}
/// <summary>
/// Given a set of C++ files, generates another set of C++ files that #include all the original
/// files, the goal being to compile the same code in fewer translation units.
/// The "unity" files are written to the CompileEnvironment's OutputDirectory.
/// </summary>
/// <param name="Target">The target we're building</param>
/// <param name="CPPFiles">The C++ files to #include.</param>
/// <param name="CompileEnvironment">The environment that is used to compile the C++ files.</param>
/// <param name="BaseName">Base name to use for the Unity files</param>
/// <returns>The "unity" C++ files.</returns>
public static List <FileItem> GenerateUnityCPPs(
UEToolChain ToolChain,
UEBuildTarget Target,
List <FileItem> CPPFiles,
CPPEnvironment CompileEnvironment,
string BaseName
)
{
List <FileItem> NewCPPFiles = new List <FileItem>();
UEBuildPlatform BuildPlatform = UEBuildPlatform.GetBuildPlatformForCPPTargetPlatform(CompileEnvironment.Config.Platform);
// Figure out size of all input files combined. We use this to determine whether to use larger unity threshold or not.
long TotalBytesInCPPFiles = CPPFiles.Sum(F => F.Info.Length);
// We have an increased threshold for unity file size if, and only if, all files fit into the same unity file. This
// is beneficial when dealing with PCH files. The default PCH creation limit is X unity files so if we generate < X
// this could be fairly slow and we'd rather bump the limit a bit to group them all into the same unity file.
// When enabled, UnrealBuildTool will try to determine source files that you are actively iteratively changing, and break those files
// out of their unity blobs so that you can compile them as individual translation units, much faster than recompiling the entire
// unity blob each time.
bool bUseAdaptiveUnityBuild = BuildConfiguration.bUseAdaptiveUnityBuild && !BuildConfiguration.bStressTestUnity;
// Optimization only makes sense if PCH files are enabled.
bool bForceIntoSingleUnityFile = BuildConfiguration.bStressTestUnity || (TotalBytesInCPPFiles < BuildConfiguration.NumIncludedBytesPerUnityCPP * 2 && CompileEnvironment.ShouldUsePCHs());
// Build the list of unity files.
List <FileCollection> AllUnityFiles;
{
// Sort the incoming file paths alphabetically, so there will be consistency in unity blobs across multiple machines.
// Note that we're relying on this not only sorting files within each directory, but also the directories
// themselves, so the whole list of file paths is the same across computers.
List <FileItem> SortedCPPFiles = CPPFiles.GetRange(0, CPPFiles.Count);
{
// Case-insensitive file path compare, because you never know what is going on with local file systems
Comparison <FileItem> FileItemComparer = (FileA, FileB) => { return(FileA.AbsolutePath.ToLowerInvariant().CompareTo(FileB.AbsolutePath.ToLowerInvariant())); };
SortedCPPFiles.Sort(FileItemComparer);
}
// Figure out whether we REALLY want to use adaptive unity for this module. If nearly every file in the module appears in the working
// set, we'll just go ahead and let unity build do its thing.
if (bUseAdaptiveUnityBuild)
{
int CandidateWorkingSetSourceFileCount = 0;
int WorkingSetSourceFileCount = 0;
foreach (FileItem CPPFile in SortedCPPFiles)
{
// Don't include writable source files into unity blobs
if (!CPPFile.Reference.IsUnderDirectory(Target.EngineIntermediateDirectory) &&
!CPPFile.Reference.IsUnderDirectory(Target.ProjectIntermediateDirectory))
{
++CandidateWorkingSetSourceFileCount;
if (UnrealBuildTool.ShouldSourceFileBePartOfWorkingSet(CPPFile.AbsolutePath))
{
++WorkingSetSourceFileCount;
// Mark this file as part of the working set. This will be saved into the UBT Makefile so that
// the assembler can automatically invalidate the Makefile when the working set changes (allowing this
// code to run again, to build up new unity blobs.)
SourceFileWorkingSet.Add(CPPFile);
}
}
}
if (WorkingSetSourceFileCount >= CandidateWorkingSetSourceFileCount)
{
// Every single file in the module appears in the working set, so don't bother using adaptive unity for this
// module. Otherwise it would make full builds really slow.
bUseAdaptiveUnityBuild = false;
}
}
UnityFileBuilder CPPUnityFileBuilder = new UnityFileBuilder(bForceIntoSingleUnityFile ? -1 : BuildConfiguration.NumIncludedBytesPerUnityCPP);
StringBuilder AdaptiveUnityBuildInfoString = new StringBuilder();
foreach (FileItem CPPFile in SortedCPPFiles)
{
if (!bForceIntoSingleUnityFile && CPPFile.AbsolutePath.IndexOf(".GeneratedWrapper.", StringComparison.InvariantCultureIgnoreCase) != -1)
{
NewCPPFiles.Add(CPPFile);
}
// When adaptive unity is enabled, go ahead and exclude any source files that we're actively working with
if (bUseAdaptiveUnityBuild && SourceFileWorkingSet.Contains(CPPFile))
{
// Just compile this file normally, not as part of the unity blob
NewCPPFiles.Add(CPPFile);
// Let the unity file builder know about the file, so that we can retain the existing size of the unity blobs.
// This won't actually make the source file part of the unity blob, but it will keep track of how big the
// file is so that other existing unity blobs from the same module won't be invalidated. This prevents much
// longer compile times the first time you build after your working file set changes.
CPPUnityFileBuilder.AddVirtualFile(CPPFile);
string CPPFileName = Path.GetFileName(CPPFile.AbsolutePath);
if (AdaptiveUnityBuildInfoString.Length == 0)
{
AdaptiveUnityBuildInfoString.Append(String.Format("[Adaptive unity build] Excluded from {0} unity file: {1}", BaseName, CPPFileName));
}
else
{
AdaptiveUnityBuildInfoString.Append(", " + CPPFileName);
}
}
else
{
// If adaptive unity build is enabled for this module, add this source file to the set that will invalidate the makefile
if (bUseAdaptiveUnityBuild)
{
CandidateSourceFilesForWorkingSet.Add(CPPFile);
}
// Compile this file as part of the unity blob
CPPUnityFileBuilder.AddFile(CPPFile);
// Now that the CPPFile is part of this unity file, we will no longer need to treat it like a root level prerequisite for our
// dependency cache, as it is now an "indirect include" from the unity file. We'll clear out the compile environment
// attached to this file. This prevents us from having to cache all of the indirect includes from these files inside our
// dependency cache, which speeds up iterative builds a lot!
CPPFile.CachedCPPIncludeInfo = null;
}
}
if (AdaptiveUnityBuildInfoString.Length > 0)
{
Log.TraceInformation(AdaptiveUnityBuildInfoString.ToString());
}
AllUnityFiles = CPPUnityFileBuilder.GetUnityFiles();
}
// Create a set of CPP files that combine smaller CPP files into larger compilation units, along with the corresponding
// actions to compile them.
int CurrentUnityFileCount = 0;
foreach (FileCollection UnityFile in AllUnityFiles)
{
++CurrentUnityFileCount;
StringWriter OutputUnityCPPWriter = new StringWriter();
StringWriter OutputUnityCPPWriterExtra = null;
// add an extra file for UBT to get the #include dependencies from
if (BuildPlatform.RequiresExtraUnityCPPWriter() == true)
{
OutputUnityCPPWriterExtra = new StringWriter();
}
OutputUnityCPPWriter.WriteLine("// This file is automatically generated at compile-time to include some subset of the user-created cpp files.");
// Add source files to the unity file
foreach (FileItem CPPFile in UnityFile.Files)
{
OutputUnityCPPWriter.WriteLine("#include \"{0}\"", ToolChain.ConvertPath(CPPFile.AbsolutePath));
if (OutputUnityCPPWriterExtra != null)
{
OutputUnityCPPWriterExtra.WriteLine("#include \"{0}\"", CPPFile.AbsolutePath);
}
}
// Determine unity file path name
string UnityCPPFileName;
if (AllUnityFiles.Count > 1)
{
UnityCPPFileName = string.Format("{0}{1}.{2}_of_{3}.cpp", ModulePrefix, BaseName, CurrentUnityFileCount, AllUnityFiles.Count);
}
else
{
UnityCPPFileName = string.Format("{0}{1}.cpp", ModulePrefix, BaseName);
}
FileReference UnityCPPFilePath = FileReference.Combine(CompileEnvironment.Config.OutputDirectory, UnityCPPFileName);
// Write the unity file to the intermediate folder.
FileItem UnityCPPFile = FileItem.CreateIntermediateTextFile(UnityCPPFilePath, OutputUnityCPPWriter.ToString());
if (OutputUnityCPPWriterExtra != null)
{
FileItem.CreateIntermediateTextFile(UnityCPPFilePath + ".ex", OutputUnityCPPWriterExtra.ToString());
}
UnityCPPFile.RelativeCost = UnityFile.TotalLength;
NewCPPFiles.Add(UnityCPPFile);
// Cache information about the unity .cpp dependencies
// @todo ubtmake urgent: Fails when building remotely for Mac because unity .cpp has an include for a PCH on the REMOTE machine
UEBuildModuleCPP.CachePCHUsageForModuleSourceFile(CompileEnvironment, UnityCPPFile);
}
return(NewCPPFiles);
}
/// <summary>
/// Given a set of C++ files, generates another set of C++ files that #include all the original
/// files, the goal being to compile the same code in fewer translation units.
/// The "unity" files are written to the CompileEnvironment's OutputDirectory.
/// </summary>
/// <param name="Target">The target we're building</param>
/// <param name="CPPFiles">The C++ files to #include.</param>
/// <param name="CompileEnvironment">The environment that is used to compile the C++ files.</param>
/// <param name="BaseName">Base name to use for the Unity files</param>
/// <returns>The "unity" C++ files.</returns>
public static List<FileItem> GenerateUnityCPPs(
UEToolChain ToolChain,
UEBuildTarget Target,
List<FileItem> CPPFiles,
CPPEnvironment CompileEnvironment,
string BaseName
)
{
List<FileItem> NewCPPFiles = new List<FileItem>();
UEBuildPlatform BuildPlatform = UEBuildPlatform.GetBuildPlatformForCPPTargetPlatform(CompileEnvironment.Config.Target.Platform);
// Figure out size of all input files combined. We use this to determine whether to use larger unity threshold or not.
long TotalBytesInCPPFiles = CPPFiles.Sum(F => F.Info.Length);
// We have an increased threshold for unity file size if, and only if, all files fit into the same unity file. This
// is beneficial when dealing with PCH files. The default PCH creation limit is X unity files so if we generate < X
// this could be fairly slow and we'd rather bump the limit a bit to group them all into the same unity file.
// When enabled, UnrealBuildTool will try to determine source files that you are actively iteratively changing, and break those files
// out of their unity blobs so that you can compile them as individual translation units, much faster than recompiling the entire
// unity blob each time.
bool bUseAdaptiveUnityBuild = BuildConfiguration.bUseAdaptiveUnityBuild && !BuildConfiguration.bStressTestUnity;
// Optimization only makes sense if PCH files are enabled.
bool bForceIntoSingleUnityFile = BuildConfiguration.bStressTestUnity || (TotalBytesInCPPFiles < BuildConfiguration.NumIncludedBytesPerUnityCPP * 2 && CompileEnvironment.ShouldUsePCHs());
// Build the list of unity files.
List<FileCollection> AllUnityFiles;
{
// Sort the incoming file paths alphabetically, so there will be consistency in unity blobs across multiple machines.
// Note that we're relying on this not only sorting files within each directory, but also the directories
// themselves, so the whole list of file paths is the same across computers.
List<FileItem> SortedCPPFiles = CPPFiles.GetRange(0, CPPFiles.Count);
{
// Case-insensitive file path compare, because you never know what is going on with local file systems
Comparison<FileItem> FileItemComparer = (FileA, FileB) => { return FileA.AbsolutePath.ToLowerInvariant().CompareTo(FileB.AbsolutePath.ToLowerInvariant()); };
SortedCPPFiles.Sort(FileItemComparer);
}
// Figure out whether we REALLY want to use adaptive unity for this module. If nearly every file in the module appears in the working
// set, we'll just go ahead and let unity build do its thing.
if (bUseAdaptiveUnityBuild)
{
int CandidateWorkingSetSourceFileCount = 0;
int WorkingSetSourceFileCount = 0;
foreach (FileItem CPPFile in SortedCPPFiles)
{
// Don't include writable source files into unity blobs
if (!CPPFile.Reference.IsUnderDirectory(Target.EngineIntermediateDirectory) &&
!CPPFile.Reference.IsUnderDirectory(Target.ProjectIntermediateDirectory))
{
++CandidateWorkingSetSourceFileCount;
if (UnrealBuildTool.ShouldSourceFileBePartOfWorkingSet(CPPFile.AbsolutePath))
{
++WorkingSetSourceFileCount;
// Mark this file as part of the working set. This will be saved into the UBT Makefile so that
// the assembler can automatically invalidate the Makefile when the working set changes (allowing this
// code to run again, to build up new unity blobs.)
SourceFileWorkingSet.Add(CPPFile);
}
}
}
if (WorkingSetSourceFileCount >= CandidateWorkingSetSourceFileCount)
{
// Every single file in the module appears in the working set, so don't bother using adaptive unity for this
// module. Otherwise it would make full builds really slow.
bUseAdaptiveUnityBuild = false;
}
}
UnityFileBuilder CPPUnityFileBuilder = new UnityFileBuilder(bForceIntoSingleUnityFile ? -1 : BuildConfiguration.NumIncludedBytesPerUnityCPP);
StringBuilder AdaptiveUnityBuildInfoString = new StringBuilder();
foreach (FileItem CPPFile in SortedCPPFiles)
{
if (!bForceIntoSingleUnityFile && CPPFile.AbsolutePath.IndexOf(".GeneratedWrapper.", StringComparison.InvariantCultureIgnoreCase) != -1)
{
NewCPPFiles.Add(CPPFile);
}
// When adaptive unity is enabled, go ahead and exclude any source files that we're actively working with
if (bUseAdaptiveUnityBuild && SourceFileWorkingSet.Contains(CPPFile))
{
// Just compile this file normally, not as part of the unity blob
NewCPPFiles.Add(CPPFile);
// Let the unity file builder know about the file, so that we can retain the existing size of the unity blobs.
// This won't actually make the source file part of the unity blob, but it will keep track of how big the
// file is so that other existing unity blobs from the same module won't be invalidated. This prevents much
// longer compile times the first time you build after your working file set changes.
CPPUnityFileBuilder.AddVirtualFile(CPPFile);
string CPPFileName = Path.GetFileName(CPPFile.AbsolutePath);
if (AdaptiveUnityBuildInfoString.Length == 0)
{
AdaptiveUnityBuildInfoString.Append(String.Format("[Adaptive unity build] Excluded from {0} unity file: {1}", BaseName, CPPFileName));
}
else
{
AdaptiveUnityBuildInfoString.Append(", " + CPPFileName);
}
}
else
{
// If adaptive unity build is enabled for this module, add this source file to the set that will invalidate the makefile
if(bUseAdaptiveUnityBuild)
{
CandidateSourceFilesForWorkingSet.Add(CPPFile);
}
// Compile this file as part of the unity blob
CPPUnityFileBuilder.AddFile(CPPFile);
// Now that the CPPFile is part of this unity file, we will no longer need to treat it like a root level prerequisite for our
// dependency cache, as it is now an "indirect include" from the unity file. We'll clear out the compile environment
// attached to this file. This prevents us from having to cache all of the indirect includes from these files inside our
// dependency cache, which speeds up iterative builds a lot!
CPPFile.CachedCPPIncludeInfo = null;
}
}
if (AdaptiveUnityBuildInfoString.Length > 0)
{
Log.TraceInformation(AdaptiveUnityBuildInfoString.ToString());
}
AllUnityFiles = CPPUnityFileBuilder.GetUnityFiles();
}
string PCHHeaderNameInCode = CPPFiles[0].PCHHeaderNameInCode;
if (CompileEnvironment.Config.PrecompiledHeaderIncludeFilename != null)
{
PCHHeaderNameInCode = ToolChain.ConvertPath(CompileEnvironment.Config.PrecompiledHeaderIncludeFilename.FullName);
// Generated unity .cpp files always include the PCH using an absolute path, so we need to update
// our compile environment's PCH header name to use this instead of the text it pulled from the original
// C++ source files
CompileEnvironment.Config.PCHHeaderNameInCode = PCHHeaderNameInCode;
}
// Create a set of CPP files that combine smaller CPP files into larger compilation units, along with the corresponding
// actions to compile them.
int CurrentUnityFileCount = 0;
foreach (FileCollection UnityFile in AllUnityFiles)
{
++CurrentUnityFileCount;
StringWriter OutputUnityCPPWriter = new StringWriter();
StringWriter OutputUnityCPPWriterExtra = null;
// add an extra file for UBT to get the #include dependencies from
if (BuildPlatform.RequiresExtraUnityCPPWriter() == true)
{
OutputUnityCPPWriterExtra = new StringWriter();
}
OutputUnityCPPWriter.WriteLine("// This file is automatically generated at compile-time to include some subset of the user-created cpp files.");
// Explicitly include the precompiled header first, since Visual C++ expects the first top-level #include to be the header file
// that was used to create the PCH.
if (CompileEnvironment.Config.PrecompiledHeaderIncludeFilename != null)
{
OutputUnityCPPWriter.WriteLine("#include \"{0}\"", PCHHeaderNameInCode);
if (OutputUnityCPPWriterExtra != null)
{
OutputUnityCPPWriterExtra.WriteLine("#include \"{0}\"", PCHHeaderNameInCode);
}
}
// Add source files to the unity file
foreach (FileItem CPPFile in UnityFile.Files)
{
OutputUnityCPPWriter.WriteLine("#include \"{0}\"", ToolChain.ConvertPath(CPPFile.AbsolutePath));
if (OutputUnityCPPWriterExtra != null)
{
OutputUnityCPPWriterExtra.WriteLine("#include \"{0}\"", CPPFile.AbsolutePath);
}
}
// Determine unity file path name
string UnityCPPFileName;
if (AllUnityFiles.Count > 1)
{
UnityCPPFileName = string.Format("Module.{0}.{1}_of_{2}.cpp", BaseName, CurrentUnityFileCount, AllUnityFiles.Count);
}
else
{
UnityCPPFileName = string.Format("Module.{0}.cpp", BaseName);
}
FileReference UnityCPPFilePath = FileReference.Combine(CompileEnvironment.Config.OutputDirectory, UnityCPPFileName);
// Write the unity file to the intermediate folder.
FileItem UnityCPPFile = FileItem.CreateIntermediateTextFile(UnityCPPFilePath, OutputUnityCPPWriter.ToString());
if (OutputUnityCPPWriterExtra != null)
{
FileItem.CreateIntermediateTextFile(UnityCPPFilePath + ".ex", OutputUnityCPPWriterExtra.ToString());
}
UnityCPPFile.RelativeCost = UnityFile.TotalLength;
UnityCPPFile.PCHHeaderNameInCode = PCHHeaderNameInCode;
NewCPPFiles.Add(UnityCPPFile);
// Cache information about the unity .cpp dependencies
// @todo ubtmake urgent: Fails when building remotely for Mac because unity .cpp has an include for a PCH on the REMOTE machine
UEBuildModuleCPP.CachePCHUsageForModuleSourceFile(Target, CompileEnvironment, UnityCPPFile);
}
return NewCPPFiles;
}