public override int Execute(CommandLineArguments Arguments)
{
FileReference ManifestFile = Arguments.GetFileReference("-ManifestFile=");
string[] ParsedFileNames = FileReference.ReadAllLines(ManifestFile);
// Load all the file timing data and summarize them for the aggregate.
FileTimingData = new TimingData()
{
Name = "Files", Type = TimingDataType.Summary
};
Parallel.ForEach(ParsedFileNames, new ParallelOptions()
{
MaxDegreeOfParallelism = 4
}, ParseTimingDataFile);
// Create aggregate summary. Duration is the duration of the files in the aggregate.
string AggregateName = Arguments.GetString("-Name=");
TimingData AggregateData = new TimingData()
{
Name = AggregateName, Type = TimingDataType.Aggregate
};
AggregateData.AddChild(FileTimingData);
// Group the includes, classes, and functions by name and sum them up then add to the aggregate.
GroupTimingDataOnName(AggregateData, "Include Timings", AggregateIncludes);
GroupTimingDataOnName(AggregateData, "Class Timings", AggregateClasses);
GroupTimingDataOnName(AggregateData, "Function Timings", AggregateFunctions);
// Write out aggregate summary.
string OutputFile = Path.Combine(ManifestFile.Directory.FullName, String.Format("{0}.timing.bin", AggregateName));
using (BinaryWriter Writer = new BinaryWriter(File.Open(OutputFile, FileMode.Create)))
{
// Write out the aggregate data.
Writer.Write(AggregateData);
// Write the look up table for the compressed binary blobs.
int Offset = 0;
Writer.Write(CompressedFiles.Count);
foreach (KeyValuePair <string, byte[]> CompressedFile in CompressedFiles)
{
Writer.Write(CompressedFile.Key);
Writer.Write(Offset);
Writer.Write(CompressedFile.Value.Length);
Writer.Write(DecompressedFileSizes[CompressedFile.Key]);
Offset += CompressedFile.Value.Length;
}
// Write the compressed binary blobs.
foreach (KeyValuePair <string, byte[]> CompressedFile in CompressedFiles)
{
Writer.Write(CompressedFile.Value);
}
}
return(0);
}
void ParseTimingDataToTracingFiles(FileReference InputFile)
{
string[] Lines = FileReference.ReadAllLines(InputFile);
for (int LineIdx = 0; LineIdx < Lines.Length;)
{
string Line = Lines[LineIdx];
if (Line.StartsWith("Include Headers:", StringComparison.Ordinal))
{
LineIdx = ParseIncludeHeadersToTraces(Lines, LineIdx + 1, InputFile.ChangeExtension(".json"));
}
else if (Line.StartsWith("Class Definitions:", StringComparison.Ordinal))
{
LineIdx = ParseDefinitions(Lines, LineIdx + 1, InputFile.ChangeExtension(".classes.txt"));
}
else if (Line.StartsWith("Function Definitions:", StringComparison.Ordinal))
{
LineIdx = ParseDefinitions(Lines, LineIdx + 1, InputFile.ChangeExtension(".functions.txt"));
}
else
{
LineIdx++;
}
}
}
/// <summary>
/// Execute the command
/// </summary>
/// <param name="Arguments">List of command line arguments</param>
/// <returns>Always zero, or throws an exception</returns>
public override int Execute(CommandLineArguments Arguments)
{
Arguments.ApplyTo(this);
Arguments.CheckAllArgumentsUsed();
Log.TraceInformation("{0}", OutputFile.GetFileName());
// Read the input files
string[] InputFileLines = FileReference.ReadAllLines(InputFileList);
FileReference[] InputFiles = InputFileLines.Select(x => x.Trim()).Where(x => x.Length > 0).Select(x => new FileReference(x)).ToArray();
// Create the combined output file, and print the diagnostics to the log
HashSet <string> UniqueItems = new HashSet <string>();
using (StreamWriter RawWriter = new StreamWriter(OutputFile.FullName))
{
foreach (FileReference InputFile in InputFiles)
{
string[] Lines = File.ReadAllLines(InputFile.FullName);
for (int LineIdx = 0; LineIdx < Lines.Length; LineIdx++)
{
string Line = Lines[LineIdx];
if (!String.IsNullOrWhiteSpace(Line) && UniqueItems.Add(Line))
{
bool bCanParse = false;
string[] Tokens = Line.Split(new string[] { "<#~>" }, StringSplitOptions.None);
if (Tokens.Length >= 9)
{
//string Trial = Tokens[1];
string LineNumberStr = Tokens[2];
string FileName = Tokens[3];
string WarningCode = Tokens[5];
string WarningMessage = Tokens[6];
string FalseAlarmStr = Tokens[7];
string LevelStr = Tokens[8];
int LineNumber;
bool bFalseAlarm;
int Level;
if (int.TryParse(LineNumberStr, out LineNumber) && bool.TryParse(FalseAlarmStr, out bFalseAlarm) && int.TryParse(LevelStr, out Level))
{
bCanParse = true;
// Ignore anything in ThirdParty folders
if (FileName.Replace('/', '\\').IndexOf("\\ThirdParty\\", StringComparison.InvariantCultureIgnoreCase) == -1)
{
// Output the line to the raw output file
RawWriter.WriteLine(Line);
// Output the line to the log
if (!bFalseAlarm && Level == 1)
{
Log.WriteLine(LogEventType.Warning, LogFormatOptions.NoSeverityPrefix, "{0}({1}): warning {2}: {3}", FileName, LineNumber, WarningCode, WarningMessage);
}
}
}
}
if (!bCanParse)
{
Log.WriteLine(LogEventType.Warning, LogFormatOptions.NoSeverityPrefix, "{0}({1}): warning: Unable to parse PVS output line '{2}' (tokens=|{3}|)", InputFile, LineIdx + 1, Line, String.Join("|", Tokens));
}
}
}
}
}
Log.TraceInformation("Written {0} {1} to {2}.", UniqueItems.Count, (UniqueItems.Count == 1)? "diagnostic" : "diagnostics", OutputFile.FullName);
return(0);
}
/// <summary>
/// Patches a set of actions for use with live coding. The new action list will output object files to a different location.
/// </summary>
/// <param name="Actions">Set of actions</param>
/// <param name="OriginalFileToPatchedFile">Dictionary that receives a map of original object file to patched object file</param>
public static void PatchActionGraphForLiveCoding(IEnumerable <Action> Actions, Dictionary <FileReference, FileReference> OriginalFileToPatchedFile)
{
foreach (Action Action in Actions)
{
if (Action.ActionType == ActionType.Compile)
{
if (!Action.CommandPath.GetFileName().Equals("cl-filter.exe", StringComparison.OrdinalIgnoreCase))
{
throw new BuildException("Unable to patch action graph - unexpected executable in compile action ({0})", Action.CommandPath);
}
List <string> Arguments = Utils.ParseArgumentList(Action.CommandArguments);
// Find the index of the cl-filter argument delimiter
int DelimiterIdx = Arguments.IndexOf("--");
if (DelimiterIdx == -1)
{
throw new BuildException("Unable to patch action graph - missing '--' delimiter to cl-filter");
}
// Fix the dependencies path
const string DependenciesPrefix = "-dependencies=";
int DependenciesIdx = 0;
for (;; DependenciesIdx++)
{
if (DependenciesIdx == DelimiterIdx)
{
throw new BuildException("Unable to patch action graph - missing '{0}' argument to cl-filter", DependenciesPrefix);
}
else if (Arguments[DependenciesIdx].StartsWith(DependenciesPrefix, StringComparison.OrdinalIgnoreCase))
{
break;
}
}
FileReference OldDependenciesFile = new FileReference(Arguments[DependenciesIdx].Substring(DependenciesPrefix.Length));
FileItem OldDependenciesFileItem = Action.ProducedItems.First(x => x.Location == OldDependenciesFile);
Action.ProducedItems.Remove(OldDependenciesFileItem);
FileReference NewDependenciesFile = OldDependenciesFile.ChangeExtension(".lc.response");
FileItem NewDependenciesFileItem = FileItem.GetItemByFileReference(NewDependenciesFile);
Action.ProducedItems.Add(NewDependenciesFileItem);
Arguments[DependenciesIdx] = DependenciesPrefix + NewDependenciesFile.FullName;
// Fix the response file
int ResponseFileIdx = DelimiterIdx + 1;
for (; ; ResponseFileIdx++)
{
if (ResponseFileIdx == Arguments.Count)
{
throw new BuildException("Unable to patch action graph - missing response file argument to cl-filter");
}
else if (Arguments[ResponseFileIdx].StartsWith("@", StringComparison.Ordinal))
{
break;
}
}
FileReference OldResponseFile = new FileReference(Arguments[ResponseFileIdx].Substring(1));
FileReference NewResponseFile = new FileReference(OldResponseFile.FullName + ".lc");
const string OutputFilePrefix = "/Fo";
string[] ResponseLines = FileReference.ReadAllLines(OldResponseFile);
for (int Idx = 0; Idx < ResponseLines.Length; Idx++)
{
string ResponseLine = ResponseLines[Idx];
if (ResponseLine.StartsWith(OutputFilePrefix, StringComparison.Ordinal))
{
FileReference OldOutputFile = new FileReference(ResponseLine.Substring(3).Trim('\"'));
FileItem OldOutputFileItem = Action.ProducedItems.First(x => x.Location == OldOutputFile);
Action.ProducedItems.Remove(OldOutputFileItem);
FileReference NewOutputFile = OldOutputFile.ChangeExtension(".lc.obj");
FileItem NewOutputFileItem = FileItem.GetItemByFileReference(NewOutputFile);
Action.ProducedItems.Add(NewOutputFileItem);
OriginalFileToPatchedFile[OldOutputFile] = NewOutputFile;
ResponseLines[Idx] = OutputFilePrefix + "\"" + NewOutputFile.FullName + "\"";
break;
}
}
FileReference.WriteAllLines(NewResponseFile, ResponseLines);
Arguments[ResponseFileIdx] = "@" + NewResponseFile.FullName;
// Update the final arguments
Action.CommandArguments = Utils.FormatCommandLine(Arguments);
}
}
}
/// <summary>
/// Upload all the files in the workspace for the current project
/// </summary>
void UploadWorkspace(DirectoryReference TempDir)
{
// Path to the scripts to be uploaded
FileReference ScriptPathsFileName = FileReference.Combine(UnrealBuildTool.EngineDirectory, "Build", "Rsync", "RsyncEngineScripts.txt");
// Read the list of scripts to be uploaded
List <string> ScriptPaths = new List <string>();
foreach (string Line in FileReference.ReadAllLines(ScriptPathsFileName))
{
string FileToUpload = Line.Trim();
if (FileToUpload.Length > 0 && FileToUpload[0] != ';')
{
ScriptPaths.Add(FileToUpload);
}
}
// Fixup the line endings
List <FileReference> TargetFiles = new List <FileReference>();
foreach (string ScriptPath in ScriptPaths)
{
FileReference SourceFile = FileReference.Combine(UnrealBuildTool.EngineDirectory, ScriptPath.TrimStart('/'));
if (!FileReference.Exists(SourceFile))
{
throw new BuildException("Missing script required for remote upload: {0}", SourceFile);
}
FileReference TargetFile = FileReference.Combine(TempDir, SourceFile.MakeRelativeTo(UnrealBuildTool.EngineDirectory));
if (!FileReference.Exists(TargetFile) || FileReference.GetLastWriteTimeUtc(TargetFile) < FileReference.GetLastWriteTimeUtc(SourceFile))
{
DirectoryReference.CreateDirectory(TargetFile.Directory);
string ScriptText = FileReference.ReadAllText(SourceFile);
FileReference.WriteAllText(TargetFile, ScriptText.Replace("\r\n", "\n"));
}
TargetFiles.Add(TargetFile);
}
// Write a file that protects all the scripts from being overridden by the standard engine filters
FileReference ScriptUploadList = FileReference.Combine(TempDir, "RsyncEngineScripts-Upload.txt");
using (StreamWriter Writer = new StreamWriter(ScriptUploadList.FullName))
{
foreach (string ScriptPath in ScriptPaths)
{
for (int SlashIdx = ScriptPath.IndexOf('/', 1); SlashIdx != -1; SlashIdx = ScriptPath.IndexOf('/', SlashIdx + 1))
{
Writer.WriteLine("+ {0}", ScriptPath.Substring(0, SlashIdx));
}
Writer.WriteLine("+ {0}", ScriptPath);
}
Writer.WriteLine("protect *");
}
// Write a file that protects all the scripts from being overridden by the standard engine filters
FileReference ScriptProtectList = FileReference.Combine(TempDir, "RsyncEngineScripts-Protect.txt");
using (StreamWriter Writer = new StreamWriter(ScriptProtectList.FullName))
{
foreach (string ScriptPath in ScriptPaths)
{
Writer.WriteLine("protect {0}", ScriptPath);
}
}
// Upload these files to the remote
List <FileReference> FilterLocations = new List <FileReference>();
FilterLocations.Add(ScriptUploadList);
UploadDirectory(TempDir, GetRemotePath(UnrealBuildTool.EngineDirectory), FilterLocations);
// Upload the engine files
List <FileReference> EngineFilters = new List <FileReference>();
EngineFilters.Add(ScriptProtectList);
EngineFilters.Add(FileReference.Combine(UnrealBuildTool.EngineDirectory, "Build", "Rsync", "RsyncEngine.txt"));
UploadDirectory(UnrealBuildTool.EngineDirectory, GetRemotePath(UnrealBuildTool.EngineDirectory), EngineFilters);
// Upload the project files
if (ProjectFile != null && !ProjectFile.IsUnderDirectory(UnrealBuildTool.EngineDirectory))
{
List <FileReference> ProjectFilters = new List <FileReference>();
FileReference CustomProjectFilter = FileReference.Combine(ProjectFile.Directory, "Build", "Rsync", "RsyncProject.txt");
if (FileReference.Exists(CustomProjectFilter))
{
ProjectFilters.Add(CustomProjectFilter);
}
ProjectFilters.Add(FileReference.Combine(UnrealBuildTool.EngineDirectory, "Build", "Rsync", "RsyncProject.txt"));
UploadDirectory(ProjectFile.Directory, GetRemotePath(ProjectFile.Directory), ProjectFilters);
}
}
/// <summary>
/// Parse crypto settings from INI file
/// </summary>
public static CryptoSettings ParseCryptoSettings(DirectoryReference InProjectDirectory, UnrealTargetPlatform InTargetPlatform)
{
CryptoSettings Settings = new CryptoSettings();
ConfigHierarchy Ini = ConfigCache.ReadHierarchy(ConfigHierarchyType.Engine, InProjectDirectory, InTargetPlatform);
Ini.GetBool("PlatformCrypto", "PlatformRequiresDataCrypto", out Settings.bDataCryptoRequired);
Ini.GetBool("PlatformCrypto", "PakSigningRequired", out Settings.PakSigningRequired);
Ini.GetBool("PlatformCrypto", "PakEncryptionRequired", out Settings.PakEncryptionRequired);
{
// Start by parsing the legacy encryption.ini settings
Ini = ConfigCache.ReadHierarchy(ConfigHierarchyType.Encryption, InProjectDirectory, InTargetPlatform);
Ini.GetBool("Core.Encryption", "SignPak", out Settings.bEnablePakSigning);
string[] SigningKeyStrings = new string[3];
Ini.GetString("Core.Encryption", "rsa.privateexp", out SigningKeyStrings[0]);
Ini.GetString("Core.Encryption", "rsa.modulus", out SigningKeyStrings[1]);
Ini.GetString("Core.Encryption", "rsa.publicexp", out SigningKeyStrings[2]);
if (String.IsNullOrEmpty(SigningKeyStrings[0]) || String.IsNullOrEmpty(SigningKeyStrings[1]) || String.IsNullOrEmpty(SigningKeyStrings[2]))
{
SigningKeyStrings = null;
}
else
{
Settings.SigningKey = new SigningKeyPair();
Settings.SigningKey.PrivateKey.Exponent = ParseHexStringToByteArray(ProcessSigningKeyInputStrings(SigningKeyStrings[0]), 64);
Settings.SigningKey.PrivateKey.Modulus = ParseHexStringToByteArray(ProcessSigningKeyInputStrings(SigningKeyStrings[1]), 64);
Settings.SigningKey.PublicKey.Exponent = ParseHexStringToByteArray(ProcessSigningKeyInputStrings(SigningKeyStrings[2]), 64);
Settings.SigningKey.PublicKey.Modulus = Settings.SigningKey.PrivateKey.Modulus;
if ((Settings.SigningKey.PrivateKey.Exponent.Length > 64) ||
(Settings.SigningKey.PrivateKey.Modulus.Length > 64) ||
(Settings.SigningKey.PublicKey.Exponent.Length > 64) ||
(Settings.SigningKey.PublicKey.Modulus.Length > 64))
{
throw new Exception(string.Format("[{0}] Signing keys parsed from encryption.ini are too long. They must be a maximum of 64 bytes long!", InProjectDirectory));
}
}
Ini.GetBool("Core.Encryption", "EncryptPak", out Settings.bEnablePakIndexEncryption);
Settings.bEnablePakFullAssetEncryption = false;
Settings.bEnablePakUAssetEncryption = false;
Settings.bEnablePakIniEncryption = Settings.bEnablePakIndexEncryption;
string EncryptionKeyString;
Ini.GetString("Core.Encryption", "aes.key", out EncryptionKeyString);
Settings.EncryptionKey = new EncryptionKey();
if (EncryptionKeyString.Length > 0)
{
if (EncryptionKeyString.Length < 32)
{
Log.WriteLine(LogEventType.Warning, "AES key parsed from encryption.ini is too short. It must be 32 bytes, so will be padded with 0s, giving sub-optimal security!");
}
else if (EncryptionKeyString.Length > 32)
{
Log.WriteLine(LogEventType.Warning, "AES key parsed from encryption.ini is too long. It must be 32 bytes, so will be truncated!");
}
Settings.EncryptionKey.Key = ParseAnsiStringToByteArray(EncryptionKeyString, 32);
}
}
Ini = ConfigCache.ReadHierarchy(ConfigHierarchyType.Crypto, InProjectDirectory, InTargetPlatform);
string SectionName = "/Script/CryptoKeys.CryptoKeysSettings";
ConfigHierarchySection CryptoSection = Ini.FindSection(SectionName);
// If we have new format crypto keys, read them in over the top of the legacy settings
if (CryptoSection != null && CryptoSection.KeyNames.Count() > 0)
{
Ini.GetBool(SectionName, "bEnablePakSigning", out Settings.bEnablePakSigning);
Ini.GetBool(SectionName, "bEncryptPakIniFiles", out Settings.bEnablePakIniEncryption);
Ini.GetBool(SectionName, "bEncryptPakIndex", out Settings.bEnablePakIndexEncryption);
Ini.GetBool(SectionName, "bEncryptUAssetFiles", out Settings.bEnablePakUAssetEncryption);
Ini.GetBool(SectionName, "bEncryptAllAssetFiles", out Settings.bEnablePakFullAssetEncryption);
// Parse encryption key
string EncryptionKeyString;
Ini.GetString(SectionName, "EncryptionKey", out EncryptionKeyString);
if (!string.IsNullOrEmpty(EncryptionKeyString))
{
Settings.EncryptionKey = new EncryptionKey();
Settings.EncryptionKey.Key = System.Convert.FromBase64String(EncryptionKeyString);
Settings.EncryptionKey.Guid = Guid.Empty.ToString();
Settings.EncryptionKey.Name = "Embedded";
}
// Parse secondary encryption keys
List <EncryptionKey> SecondaryEncryptionKeys = new List <EncryptionKey>();
List <string> SecondaryEncryptionKeyStrings;
if (Ini.GetArray(SectionName, "SecondaryEncryptionKeys", out SecondaryEncryptionKeyStrings))
{
foreach (string KeySource in SecondaryEncryptionKeyStrings)
{
EncryptionKey NewKey = new EncryptionKey();
SecondaryEncryptionKeys.Add(NewKey);
Regex Search = new Regex("\\(Guid=(?\'Guid\'.*),Name=\\\"(?\'Name\'.*)\\\",Key=\\\"(?\'Key\'.*)\\\"\\)");
Match Match = Search.Match(KeySource);
if (Match.Success)
{
foreach (string GroupName in Search.GetGroupNames())
{
string Value = Match.Groups[GroupName].Value;
if (GroupName == "Guid")
{
NewKey.Guid = Value;
}
else if (GroupName == "Name")
{
NewKey.Name = Value;
}
else if (GroupName == "Key")
{
NewKey.Key = System.Convert.FromBase64String(Value);
}
}
}
}
}
Settings.SecondaryEncryptionKeys = SecondaryEncryptionKeys.ToArray();
// Parse signing key
string PrivateExponent, PublicExponent, Modulus;
Ini.GetString(SectionName, "SigningPrivateExponent", out PrivateExponent);
Ini.GetString(SectionName, "SigningModulus", out Modulus);
Ini.GetString(SectionName, "SigningPublicExponent", out PublicExponent);
if (!String.IsNullOrEmpty(PrivateExponent) && !String.IsNullOrEmpty(PublicExponent) && !String.IsNullOrEmpty(Modulus))
{
Settings.SigningKey = new SigningKeyPair();
Settings.SigningKey.PublicKey.Exponent = System.Convert.FromBase64String(PublicExponent);
Settings.SigningKey.PublicKey.Modulus = System.Convert.FromBase64String(Modulus);
Settings.SigningKey.PrivateKey.Exponent = System.Convert.FromBase64String(PrivateExponent);
Settings.SigningKey.PrivateKey.Modulus = Settings.SigningKey.PublicKey.Modulus;
}
}
// Parse project dynamic keychain keys
if (InProjectDirectory != null)
{
ConfigHierarchy GameIni = ConfigCache.ReadHierarchy(ConfigHierarchyType.Game, InProjectDirectory, InTargetPlatform);
if (GameIni != null)
{
string Filename;
if (GameIni.GetString("ContentEncryption", "ProjectKeyChain", out Filename))
{
FileReference ProjectKeyChainFile = FileReference.Combine(InProjectDirectory, "Content", Filename);
if (FileReference.Exists(ProjectKeyChainFile))
{
List <EncryptionKey> EncryptionKeys = new List <EncryptionKey>();
if (Settings.SecondaryEncryptionKeys != null)
{
EncryptionKeys.AddRange(Settings.SecondaryEncryptionKeys);
}
string[] Lines = FileReference.ReadAllLines(ProjectKeyChainFile);
foreach (string Line in Lines)
{
string[] KeyParts = Line.Split(':');
if (KeyParts.Length == 4)
{
EncryptionKey NewKey = new EncryptionKey();
NewKey.Name = KeyParts[0];
NewKey.Guid = KeyParts[2];
NewKey.Key = System.Convert.FromBase64String(KeyParts[3]);
EncryptionKey ExistingKey = EncryptionKeys.Find((EncryptionKey OtherKey) => { return(OtherKey.Guid == NewKey.Guid); });
if (ExistingKey != null && !CompareKey(ExistingKey.Key, NewKey.Key))
{
throw new Exception("Found multiple encryption keys with the same guid but different AES keys while merging secondary keys from the project key-chain!");
}
EncryptionKeys.Add(NewKey);
}
}
Settings.SecondaryEncryptionKeys = EncryptionKeys.ToArray();
}
}
}
}
if (!Settings.bDataCryptoRequired)
{
CryptoSettings NewSettings = new CryptoSettings();
NewSettings.SecondaryEncryptionKeys = Settings.SecondaryEncryptionKeys;
Settings = NewSettings;
}
else
{
if (!Settings.PakSigningRequired)
{
Settings.bEnablePakSigning = false;
Settings.SigningKey = null;
}
if (!Settings.PakEncryptionRequired)
{
Settings.bEnablePakFullAssetEncryption = false;
Settings.bEnablePakIndexEncryption = false;
Settings.bEnablePakIniEncryption = false;
Settings.EncryptionKey = null;
Settings.SigningKey = null;
}
}
// Check if we have a valid signing key that is of the old short form
if (Settings.SigningKey != null && Settings.SigningKey.IsValid() && Settings.SigningKey.IsUnsecureLegacyKey())
{
Log.TraceWarningOnce("Project signing keys found in '{0}' are of the old insecure short format. Please regenerate them using the project crypto settings panel in the editor!", InProjectDirectory);
}
// Validate the settings we have read
if (Settings.bDataCryptoRequired && Settings.bEnablePakSigning && (Settings.SigningKey == null || !Settings.SigningKey.IsValid()))
{
Log.TraceWarningOnce("Pak signing is enabled, but no valid signing keys were found. Please generate a key in the editor project crypto settings. Signing will be disabled");
Settings.bEnablePakSigning = false;
}
if (Settings.bDataCryptoRequired && Settings.IsAnyEncryptionEnabled() && (Settings.EncryptionKey == null || !Settings.EncryptionKey.IsValid()))
{
Log.TraceWarningOnce("Pak encryption is enabled, but no valid encryption key was found. Please generate a key in the editor project crypto settings. Encryption will be disabled");
Settings.bEnablePakUAssetEncryption = false;
Settings.bEnablePakFullAssetEncryption = false;
Settings.bEnablePakIndexEncryption = false;
Settings.bEnablePakIniEncryption = false;
}
return(Settings);
}
/// <summary>
/// Determine what needs to be built for a target
/// </summary>
/// <param name="BuildConfiguration">The build configuration</param>
/// <param name="TargetDescriptor">Target being built</param>
/// <param name="Makefile">Makefile generated for this target</param>
/// <returns>Set of actions to execute</returns>
static HashSet <Action> GetActionsForTarget(BuildConfiguration BuildConfiguration, TargetDescriptor TargetDescriptor, TargetMakefile Makefile)
{
// Create the action graph
ActionGraph.Link(Makefile.Actions);
// Get the hot-reload mode
HotReloadMode HotReloadMode = TargetDescriptor.HotReloadMode;
if (HotReloadMode == HotReloadMode.Default)
{
if (TargetDescriptor.HotReloadModuleNameToSuffix.Count > 0 && TargetDescriptor.ForeignPlugin == null)
{
HotReloadMode = HotReloadMode.FromEditor;
}
else if (BuildConfiguration.bAllowHotReloadFromIDE && HotReload.ShouldDoHotReloadFromIDE(BuildConfiguration, TargetDescriptor))
{
HotReloadMode = HotReloadMode.FromIDE;
}
else
{
HotReloadMode = HotReloadMode.Disabled;
}
}
// Guard against a live coding session for this target being active
if (HotReloadMode != HotReloadMode.LiveCoding && TargetDescriptor.ForeignPlugin == null && HotReload.IsLiveCodingSessionActive(Makefile))
{
throw new BuildException("Unable to start regular build while Live Coding is active. Press Ctrl+Alt+F11 to trigger a Live Coding compile.");
}
// Get the root prerequisite actions
List <Action> PrerequisiteActions = GatherPrerequisiteActions(TargetDescriptor, Makefile);
// Get the path to the hot reload state file for this target
FileReference HotReloadStateFile = global::UnrealBuildTool.HotReloadState.GetLocation(TargetDescriptor.ProjectFile, TargetDescriptor.Name, TargetDescriptor.Platform, TargetDescriptor.Configuration, TargetDescriptor.Architecture);
// Apply the previous hot reload state
HotReloadState HotReloadState = null;
if (HotReloadMode == HotReloadMode.Disabled)
{
// Make sure we're not doing a partial build from the editor (eg. compiling a new plugin)
if (TargetDescriptor.ForeignPlugin == null && TargetDescriptor.SingleFileToCompile == null)
{
// Delete the previous state file
HotReload.DeleteTemporaryFiles(HotReloadStateFile);
}
}
else
{
// Read the previous state file and apply it to the action graph
if (FileReference.Exists(HotReloadStateFile))
{
HotReloadState = HotReloadState.Load(HotReloadStateFile);
}
else
{
HotReloadState = new HotReloadState();
}
// Apply the old state to the makefile
HotReload.ApplyState(HotReloadState, Makefile);
// If we want a specific suffix on any modules, apply that now. We'll track the outputs later, but the suffix has to be forced (and is always out of date if it doesn't exist).
HotReload.PatchActionGraphWithNames(PrerequisiteActions, TargetDescriptor.HotReloadModuleNameToSuffix, Makefile);
// Re-link the action graph
ActionGraph.Link(PrerequisiteActions);
}
// Create the dependencies cache
CppDependencyCache CppDependencies;
using (Timeline.ScopeEvent("Reading dependency cache"))
{
CppDependencies = CppDependencyCache.CreateHierarchy(TargetDescriptor.ProjectFile, TargetDescriptor.Name, TargetDescriptor.Platform, TargetDescriptor.Configuration, Makefile.TargetType, TargetDescriptor.Architecture);
}
// Create the action history
ActionHistory History;
using (Timeline.ScopeEvent("Reading action history"))
{
History = ActionHistory.CreateHierarchy(TargetDescriptor.ProjectFile, TargetDescriptor.Name, TargetDescriptor.Platform, Makefile.TargetType, TargetDescriptor.Architecture);
}
// Plan the actions to execute for the build. For single file compiles, always rebuild the source file regardless of whether it's out of date.
HashSet <Action> TargetActionsToExecute;
if (TargetDescriptor.SingleFileToCompile == null)
{
TargetActionsToExecute = ActionGraph.GetActionsToExecute(Makefile.Actions, PrerequisiteActions, CppDependencies, History, BuildConfiguration.bIgnoreOutdatedImportLibraries);
}
else
{
TargetActionsToExecute = new HashSet <Action>(PrerequisiteActions);
}
// Additional processing for hot reload
if (HotReloadMode == HotReloadMode.LiveCoding)
{
// Make sure we're not overwriting any lazy-loaded modules
if (TargetDescriptor.LiveCodingModules != null)
{
// Read the list of modules that we're allowed to build
string[] Lines = FileReference.ReadAllLines(TargetDescriptor.LiveCodingModules);
// Parse it out into a set of filenames
HashSet <string> AllowedOutputFileNames = new HashSet <string>(FileReference.Comparer);
foreach (string Line in Lines)
{
string TrimLine = Line.Trim();
if (TrimLine.Length > 0)
{
AllowedOutputFileNames.Add(Path.GetFileName(TrimLine));
}
}
// Find all the binaries that we're actually going to build
HashSet <FileReference> OutputFiles = new HashSet <FileReference>();
foreach (Action Action in TargetActionsToExecute)
{
if (Action.ActionType == ActionType.Link)
{
OutputFiles.UnionWith(Action.ProducedItems.Where(x => x.HasExtension(".exe") || x.HasExtension(".dll")).Select(x => x.Location));
}
}
// Find all the files that will be built that aren't allowed
List <FileReference> ProtectedOutputFiles = OutputFiles.Where(x => !AllowedOutputFileNames.Contains(x.GetFileName())).ToList();
if (ProtectedOutputFiles.Count > 0)
{
FileReference.WriteAllLines(new FileReference(TargetDescriptor.LiveCodingModules.FullName + ".out"), ProtectedOutputFiles.Select(x => x.ToString()));
foreach (FileReference ProtectedOutputFile in ProtectedOutputFiles)
{
Log.TraceInformation("Module {0} is not currently enabled for Live Coding", ProtectedOutputFile);
}
throw new CompilationResultException(CompilationResult.Canceled);
}
}
// Filter the prerequisite actions down to just the compile actions, then recompute all the actions to execute
PrerequisiteActions = new List <Action>(TargetActionsToExecute.Where(x => x.ActionType == ActionType.Compile));
TargetActionsToExecute = ActionGraph.GetActionsToExecute(Makefile.Actions, PrerequisiteActions, CppDependencies, History, BuildConfiguration.bIgnoreOutdatedImportLibraries);
// Update the action graph with these new paths
Dictionary <FileReference, FileReference> OriginalFileToPatchedFile = new Dictionary <FileReference, FileReference>();
HotReload.PatchActionGraphForLiveCoding(PrerequisiteActions, OriginalFileToPatchedFile);
// Get a new list of actions to execute now that the graph has been modified
TargetActionsToExecute = ActionGraph.GetActionsToExecute(Makefile.Actions, PrerequisiteActions, CppDependencies, History, BuildConfiguration.bIgnoreOutdatedImportLibraries);
// Output the Live Coding manifest
if (TargetDescriptor.LiveCodingManifest != null)
{
HotReload.WriteLiveCodingManifest(TargetDescriptor.LiveCodingManifest, Makefile.Actions, OriginalFileToPatchedFile);
}
}
else if (HotReloadMode == HotReloadMode.FromEditor || HotReloadMode == HotReloadMode.FromIDE)
{
// Patch action history for hot reload when running in assembler mode. In assembler mode, the suffix on the output file will be
// the same for every invocation on that makefile, but we need a new suffix each time.
// For all the hot-reloadable modules that may need a unique suffix appended, build a mapping from output item to all the output items in that module. We can't
// apply a suffix to one without applying a suffix to all of them.
Dictionary <FileItem, FileItem[]> HotReloadItemToDependentItems = new Dictionary <FileItem, FileItem[]>();
foreach (string HotReloadModuleName in Makefile.HotReloadModuleNames)
{
int ModuleSuffix;
if (!TargetDescriptor.HotReloadModuleNameToSuffix.TryGetValue(HotReloadModuleName, out ModuleSuffix) || ModuleSuffix == -1)
{
FileItem[] ModuleOutputItems;
if (Makefile.ModuleNameToOutputItems.TryGetValue(HotReloadModuleName, out ModuleOutputItems))
{
foreach (FileItem ModuleOutputItem in ModuleOutputItems)
{
HotReloadItemToDependentItems[ModuleOutputItem] = ModuleOutputItems;
}
}
}
}
// Expand the list of actions to execute to include everything that references any files with a new suffix. Unlike a regular build, we can't ignore
// dependencies on import libraries under the assumption that a header would change if the API changes, because the dependency will be on a different DLL.
HashSet <FileItem> FilesRequiringSuffix = new HashSet <FileItem>(TargetActionsToExecute.SelectMany(x => x.ProducedItems).Where(x => HotReloadItemToDependentItems.ContainsKey(x)));
for (int LastNumFilesWithNewSuffix = 0; FilesRequiringSuffix.Count > LastNumFilesWithNewSuffix;)
{
LastNumFilesWithNewSuffix = FilesRequiringSuffix.Count;
foreach (Action PrerequisiteAction in PrerequisiteActions)
{
if (!TargetActionsToExecute.Contains(PrerequisiteAction))
{
foreach (FileItem ProducedItem in PrerequisiteAction.ProducedItems)
{
FileItem[] DependentItems;
if (HotReloadItemToDependentItems.TryGetValue(ProducedItem, out DependentItems))
{
TargetActionsToExecute.Add(PrerequisiteAction);
FilesRequiringSuffix.UnionWith(DependentItems);
}
}
}
}
}
// Build a list of file mappings
Dictionary <FileReference, FileReference> OldLocationToNewLocation = new Dictionary <FileReference, FileReference>();
foreach (FileItem FileRequiringSuffix in FilesRequiringSuffix)
{
FileReference OldLocation = FileRequiringSuffix.Location;
FileReference NewLocation = HotReload.ReplaceSuffix(OldLocation, HotReloadState.NextSuffix);
OldLocationToNewLocation[OldLocation] = NewLocation;
}
// Update the action graph with these new paths
HotReload.PatchActionGraph(PrerequisiteActions, OldLocationToNewLocation);
// Get a new list of actions to execute now that the graph has been modified
TargetActionsToExecute = ActionGraph.GetActionsToExecute(Makefile.Actions, PrerequisiteActions, CppDependencies, History, BuildConfiguration.bIgnoreOutdatedImportLibraries);
// Build a mapping of all file items to their original
Dictionary <FileReference, FileReference> HotReloadFileToOriginalFile = new Dictionary <FileReference, FileReference>();
foreach (KeyValuePair <FileReference, FileReference> Pair in HotReloadState.OriginalFileToHotReloadFile)
{
HotReloadFileToOriginalFile[Pair.Value] = Pair.Key;
}
foreach (KeyValuePair <FileReference, FileReference> Pair in OldLocationToNewLocation)
{
FileReference OriginalLocation;
if (!HotReloadFileToOriginalFile.TryGetValue(Pair.Key, out OriginalLocation))
{
OriginalLocation = Pair.Key;
}
HotReloadFileToOriginalFile[Pair.Value] = OriginalLocation;
}
// Now filter out all the hot reload files and update the state
foreach (Action Action in TargetActionsToExecute)
{
foreach (FileItem ProducedItem in Action.ProducedItems)
{
FileReference OriginalLocation;
if (HotReloadFileToOriginalFile.TryGetValue(ProducedItem.Location, out OriginalLocation))
{
HotReloadState.OriginalFileToHotReloadFile[OriginalLocation] = ProducedItem.Location;
HotReloadState.TemporaryFiles.Add(ProducedItem.Location);
}
}
}
// Increment the suffix for the next iteration
if (TargetActionsToExecute.Count > 0)
{
HotReloadState.NextSuffix++;
}
// Save the new state
HotReloadState.Save(HotReloadStateFile);
// Prevent this target from deploying
Makefile.bDeployAfterCompile = false;
}
return(TargetActionsToExecute);
}
/// <summary>
/// Reads dependencies from the given file.
/// </summary>
/// <param name="InputFile">The file to read from</param>
/// <returns>List of included dependencies</returns>
static List <FileItem> ReadDependenciesFile(FileReference InputFile)
{
if (InputFile.HasExtension(".txt"))
{
string[] Lines = FileReference.ReadAllLines(InputFile);
HashSet <FileItem> DependencyItems = new HashSet <FileItem>();
foreach (string Line in Lines)
{
if (Line.Length > 0)
{
// Ignore *.tlh and *.tli files generated by the compiler from COM DLLs
if (!Line.EndsWith(".tlh", StringComparison.OrdinalIgnoreCase) && !Line.EndsWith(".tli", StringComparison.OrdinalIgnoreCase))
{
DependencyItems.Add(FileItem.GetItemByPath(Line));
}
}
}
return(DependencyItems.ToList());
}
else if (InputFile.HasExtension(".d"))
{
string Text = FileReference.ReadAllText(InputFile);
List <string> Tokens = new List <string>();
StringBuilder Token = new StringBuilder();
for (int Idx = 0; TryReadMakefileToken(Text, ref Idx, Token);)
{
Tokens.Add(Token.ToString());
}
int TokenIdx = 0;
while (TokenIdx < Tokens.Count && Tokens[TokenIdx] == "\n")
{
TokenIdx++;
}
if (TokenIdx + 1 >= Tokens.Count || Tokens[TokenIdx + 1] != ":")
{
throw new BuildException("Unable to parse dependency file");
}
TokenIdx += 2;
List <FileItem> NewDependencyFiles = new List <FileItem>();
for (; TokenIdx < Tokens.Count && Tokens[TokenIdx] != "\n"; TokenIdx++)
{
NewDependencyFiles.Add(FileItem.GetItemByPath(Tokens[TokenIdx]));
}
while (TokenIdx < Tokens.Count && Tokens[TokenIdx] == "\n")
{
TokenIdx++;
}
if (TokenIdx != Tokens.Count)
{
throw new BuildException("Unable to parse dependency file");
}
return(NewDependencyFiles);
}
else
{
throw new BuildException("Unknown dependency list file type: {0}", InputFile);
}
}
/// <summary>
/// Checks to see if the assembly needs compilation
/// </summary>
/// <param name="SourceFiles">Set of source files</param>
/// <param name="AssemblySourceListFilePath">File to use to cache source file names</param>
/// <param name="OutputAssemblyPath">Output path for the assembly</param>
/// <returns>True if the assembly needs to be built</returns>
private static bool RequiresCompilation(HashSet <FileReference> SourceFiles, FileReference AssemblySourceListFilePath, FileReference OutputAssemblyPath)
{
// Check to see if we already have a compiled assembly file on disk
FileItem OutputAssemblyInfo = FileItem.GetItemByFileReference(OutputAssemblyPath);
if (!OutputAssemblyInfo.Exists)
{
Log.TraceLog("Compiling {0}: Assembly does not exist", OutputAssemblyPath);
return(true);
}
// Check the time stamp of the UnrealBuildTool.exe file. If Unreal Build Tool was compiled more
// recently than the dynamically-compiled assembly, then we'll always recompile it. This is
// because Unreal Build Tool's code may have changed in such a way that invalidate these
// previously-compiled assembly files.
FileItem ExecutableItem = FileItem.GetItemByFileReference(UnrealBuildTool.GetUBTPath());
if (ExecutableItem.LastWriteTimeUtc > OutputAssemblyInfo.LastWriteTimeUtc)
{
Log.TraceLog("Compiling {0}: {1} is newer", OutputAssemblyPath, ExecutableItem.Name);
return(true);
}
// Make sure we have a manifest of source files used to compile the output assembly. If it doesn't exist
// for some reason (not an expected case) then we'll need to recompile.
FileItem AssemblySourceListFile = FileItem.GetItemByFileReference(AssemblySourceListFilePath);
if (!AssemblySourceListFile.Exists)
{
Log.TraceLog("Compiling {0}: Missing source file list ({1})", OutputAssemblyPath, AssemblySourceListFilePath);
return(true);
}
// Make sure the source files we're compiling are the same as the source files that were compiled
// for the assembly that we want to load
HashSet <FileItem> CurrentSourceFileItems = new HashSet <FileItem>();
foreach (string Line in FileReference.ReadAllLines(AssemblySourceListFile.Location))
{
CurrentSourceFileItems.Add(FileItem.GetItemByPath(Line));
}
// Get the new source files
HashSet <FileItem> SourceFileItems = new HashSet <FileItem>();
foreach (FileReference SourceFile in SourceFiles)
{
SourceFileItems.Add(FileItem.GetItemByFileReference(SourceFile));
}
// Check if there are any differences between the sets
foreach (FileItem CurrentSourceFileItem in CurrentSourceFileItems)
{
if (!SourceFileItems.Contains(CurrentSourceFileItem))
{
Log.TraceLog("Compiling {0}: Removed source file ({1})", OutputAssemblyPath, AssemblySourceListFilePath);
return(true);
}
}
foreach (FileItem SourceFileItem in SourceFileItems)
{
if (!CurrentSourceFileItems.Contains(SourceFileItem))
{
Log.TraceLog("Compiling {0}: Added source file ({1})", OutputAssemblyPath, AssemblySourceListFilePath);
return(true);
}
}
// Check if any of the timestamps are newer
foreach (FileItem SourceFileItem in SourceFileItems)
{
if (SourceFileItem.LastWriteTimeUtc > OutputAssemblyInfo.LastWriteTimeUtc)
{
Log.TraceLog("Compiling {0}: {1} is newer", OutputAssemblyPath, SourceFileItem);
return(true);
}
}
return(false);
}
public override int Execute(CommandLineArguments Arguments)
{
FileReference InputFile = Arguments.GetFileReference("-TimingFile=");
// If the tracing argument was passed, hand off to the logic to generate a JSON file compatible with
// chrome://tracing
if (Arguments.HasOption("-Tracing"))
{
ParseTimingDataToTracingFiles(InputFile);
return(0);
}
// Break the input file into the various sections for processing.
string[] AllLines = FileReference.ReadAllLines(InputFile);
List <string> Includes = new List <string>();
List <string> Classes = new List <string>();
List <string> Functions = new List <string>();
TimingDataType CurrentType = TimingDataType.None;
foreach (string Line in AllLines)
{
if (string.IsNullOrWhiteSpace(Line))
{
continue;
}
// Check for a change of type.
if (Line.StartsWith("Include Headers:", StringComparison.OrdinalIgnoreCase))
{
CurrentType = TimingDataType.Include;
continue;
}
else if (Line.StartsWith("Class Definitions:", StringComparison.OrdinalIgnoreCase))
{
CurrentType = TimingDataType.Class;
continue;
}
else if (Line.StartsWith("Function Definitions:", StringComparison.OrdinalIgnoreCase))
{
CurrentType = TimingDataType.Function;
continue;
}
// Skip the count line, we don't need it.
if (Regex.IsMatch(Line, @"^\tCount\:\s*\d*$"))
{
continue;
}
// If we didn't change types and this isn't the count line and it doesn't match the expected output,
// clear the current type and move on.
Match TimingDataMatch = Regex.Match(Line, TimingDataRegex);
if (!TimingDataMatch.Success)
{
CurrentType = TimingDataType.None;
continue;
}
// If we get to here this is a line we want to parse. Add it to the correct collection.
switch (CurrentType)
{
case TimingDataType.Include:
{
Includes.Add(Line);
break;
}
case TimingDataType.Class:
{
Classes.Add(Line);
break;
}
case TimingDataType.Function:
{
Functions.Add(Line);
break;
}
}
}
// Build the summary.
TimingData Summary = new TimingData()
{
Name = InputFile.FullName.Replace(".timing.txt", string.Empty), Type = TimingDataType.Summary
};
Summary.AddChild(SummarizeParsedTimingData("IncludeTimings", TimingDataType.Include, Includes));
Summary.AddChild(SummarizeParsedTimingData("ClassTimings", TimingDataType.Class, Classes));
Summary.AddChild(SummarizeParsedTimingData("FunctionTimings", TimingDataType.Function, Functions));
// Write out the timing binary file.
using (BinaryWriter Writer = new BinaryWriter(File.Open(InputFile.ChangeExtension(".timing.bin").FullName, FileMode.Create)))
{
Writer.Write(Summary);
}
return(0);
}