/// <summary>
/// Gets the path to a file created to indicate that a node is complete, under the given base directory.
/// </summary>
/// <param name="BaseDir">A local or shared temp storage root directory.</param>
/// <param name="NodeName">Name of the node to get the file for</param>
static FileReference GetCompleteMarkerFile(DirectoryReference BaseDir, string NodeName)
{
return(FileReference.Combine(GetDirectoryForNode(BaseDir, NodeName), "Complete"));
}
//#nv begin #Blast Linux build
void StageBootstrapExecutable(DeploymentContext SC, StageTarget Target, string ExeName, string TargetFile, string StagedRelativeTargetPath, string StagedArguments) //@third party code - NVSTUDIOS Set LD_LIBRARY_PATH
//nv end
{
// create a temp script file location
DirectoryReference IntermediateDir = DirectoryReference.Combine(SC.ProjectRoot, "Intermediate", "Staging");
FileReference IntermediateFile = FileReference.Combine(IntermediateDir, ExeName);
DirectoryReference.CreateDirectory(IntermediateDir);
// make sure slashes are good
StagedRelativeTargetPath = StagedRelativeTargetPath.Replace("\\", "/");
// make contents
StringBuilder Script = new StringBuilder();
string EOL = "\n";
Script.Append("#!/bin/sh" + EOL);
// allow running from symlinks
Script.AppendFormat("UE4_TRUE_SCRIPT_NAME=$(echo \\\"$0\\\" | xargs readlink -f)" + EOL);
Script.AppendFormat("UE4_PROJECT_ROOT=$(dirname \"$UE4_TRUE_SCRIPT_NAME\")" + EOL);
Script.AppendFormat("chmod +x \"$UE4_PROJECT_ROOT/{0}\"" + EOL, StagedRelativeTargetPath);
//#nv begin #Blast Linux build
//The Blast .so files are not loaded by dlopen so we we need to setup the search paths
//Really UE should be doing this for all dependent libraries, but they usually statically link
HashSet <string> LDLibraryPaths = new HashSet <string>();
DirectoryReference TargetFileDir = (new FileReference(TargetFile)).Directory;
DirectoryReference SourceEngineDir = SC.LocalRoot;
DirectoryReference SourceProjectDir = SC.ProjectRoot;
foreach (var RuntimeDependency in Target.Receipt.RuntimeDependencies)
{
foreach (FileReference File in CommandUtils.ResolveFilespec(CommandUtils.RootDirectory, RuntimeDependency.Path.FullName, new string[] { }))
{
if (FileReference.Exists(File) && File.GetExtension().Equals(".so", StringComparison.OrdinalIgnoreCase))
{
string FileRelativePath = null;
DirectoryReference SharedLibFolder = File.Directory;
if (SharedLibFolder.IsUnderDirectory(SourceProjectDir))
{
FileRelativePath = Path.Combine(SharedLibFolder.MakeRelativeTo(SourceProjectDir), SC.RelativeProjectRootForStage.ToString());
}
else if (SharedLibFolder.IsUnderDirectory(SourceEngineDir))
{
FileRelativePath = SharedLibFolder.MakeRelativeTo(SourceEngineDir);
}
if (FileRelativePath != null)
{
FileRelativePath = Path.Combine("$UE4_PROJECT_ROOT", FileRelativePath);
FileRelativePath = FileRelativePath.Replace("\\", "/");
//Escape spaces
FileRelativePath = FileRelativePath.Replace(" ", @"\ ");
LDLibraryPaths.Add(FileRelativePath);
}
}
}
}
if (LDLibraryPaths.Count > 0)
{
Script.AppendFormat("export LD_LIBRARY_PATH={0}" + EOL, string.Join(":", LDLibraryPaths));
}
//nv end
Script.AppendFormat("\"$UE4_PROJECT_ROOT/{0}\" {1} $@ " + EOL, StagedRelativeTargetPath, StagedArguments);
// write out the
FileReference.WriteAllText(IntermediateFile, Script.ToString());
if (Utils.IsRunningOnMono)
{
var Result = CommandUtils.Run("sh", string.Format("-c 'chmod +x \\\"{0}\\\"'", IntermediateFile));
if (Result.ExitCode != 0)
{
throw new AutomationException(string.Format("Failed to chmod \"{0}\"", IntermediateFile));
}
}
SC.StageFile(StagedFileType.NonUFS, IntermediateFile, new StagedFileReference(ExeName));
}
public override void GetTargetFile(string RemoteFilePath, string LocalFile, ProjectParams Params)
{
var SourceFile = FileReference.Combine(new DirectoryReference(Params.BaseStageDirectory), GetCookPlatform(Params.HasServerCookedTargets, Params.HasClientTargetDetected), RemoteFilePath);
CommandUtils.CopyFile(SourceFile.FullName, LocalFile);
}
/// <summary>
/// Finds all targets for the project.
/// </summary>
/// <param name="Properties">Project properties.</param>
/// <param name="ExtraSearchPaths">Additional search paths.</param>
private static void DetectTargetsForProject(ProjectProperties Properties, List <string> ExtraSearchPaths = null)
{
Properties.Targets = new Dictionary <TargetRules.TargetType, SingleTargetProperties>();
FileReference TargetsDllFilename;
string FullProjectPath = null;
var GameFolders = new List <DirectoryReference>();
var RulesFolder = new DirectoryReference(GetRulesAssemblyFolder());
if (Properties.RawProjectPath != null)
{
CommandUtils.LogVerbose("Looking for targets for project {0}", Properties.RawProjectPath);
TargetsDllFilename = FileReference.Combine(RulesFolder, String.Format("UATRules{0}.dll", Properties.RawProjectPath.GetHashCode()));
FullProjectPath = CommandUtils.GetDirectoryName(Properties.RawProjectPath.FullName);
GameFolders.Add(new DirectoryReference(FullProjectPath));
CommandUtils.LogVerbose("Searching for target rule files in {0}", FullProjectPath);
}
else
{
TargetsDllFilename = FileReference.Combine(RulesFolder, String.Format("UATRules{0}.dll", "_BaseEngine_"));
}
// the UBT code assumes a certain CWD, but artists don't have this CWD.
var SourceDir = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Source");
bool DirPushed = false;
if (CommandUtils.DirectoryExists_NoExceptions(SourceDir))
{
CommandUtils.PushDir(SourceDir);
DirPushed = true;
}
var ExtraSearchDirectories = (ExtraSearchPaths == null)? null : ExtraSearchPaths.Select(x => new DirectoryReference(x)).ToList();
var TargetScripts = RulesCompiler.FindAllRulesSourceFiles(RulesCompiler.RulesFileType.Target, GameFolders: GameFolders, ForeignPlugins: null, AdditionalSearchPaths: ExtraSearchDirectories);
if (DirPushed)
{
CommandUtils.PopDir();
}
if (!CommandUtils.IsNullOrEmpty(TargetScripts))
{
// We only care about project target script so filter out any scripts not in the project folder, or take them all if we are just doing engine stuff
var ProjectTargetScripts = new List <FileReference>();
foreach (var TargetScript in TargetScripts)
{
if (FullProjectPath == null || TargetScript.IsUnderDirectory(new DirectoryReference(FullProjectPath)))
{
ProjectTargetScripts.Add(TargetScript);
}
}
TargetScripts = ProjectTargetScripts;
}
if (!CommandUtils.IsNullOrEmpty(TargetScripts))
{
CommandUtils.LogVerbose("Found {0} target rule files:", TargetScripts.Count);
foreach (var Filename in TargetScripts)
{
CommandUtils.LogVerbose(" {0}", Filename);
}
// Check if the scripts require compilation
bool DoNotCompile = false;
if (!CommandUtils.IsBuildMachine && !CheckIfScriptAssemblyIsOutOfDate(TargetsDllFilename, TargetScripts))
{
Log.TraceVerbose("Targets DLL {0} is up to date.", TargetsDllFilename);
DoNotCompile = true;
}
if (!DoNotCompile && CommandUtils.FileExists_NoExceptions(TargetsDllFilename.FullName))
{
if (!CommandUtils.DeleteFile_NoExceptions(TargetsDllFilename.FullName, true))
{
DoNotCompile = true;
CommandUtils.LogVerbose("Could not delete {0} assuming it is up to date and reusable for a recursive UAT call.", TargetsDllFilename);
}
}
CompileAndLoadTargetsAssembly(Properties, TargetsDllFilename, DoNotCompile, TargetScripts);
}
}
/// <summary>
/// Reads an exported XGE task list
/// </summary>
/// <param name="TaskListFile">File to read from</param>
/// <param name="FileToEnvironment">Mapping from source file to compile environment</param>
public static void ReadTaskList(FileReference TaskListFile, DirectoryReference BaseDir, out Dictionary <FileReference, CompileEnvironment> FileToEnvironment)
{
XmlDocument Document = new XmlDocument();
Document.Load(TaskListFile.FullName);
// Read the standard include paths from the INCLUDE environment variable in the script
List <DirectoryReference> StandardIncludePaths = new List <DirectoryReference>();
foreach (XmlNode Node in Document.SelectNodes("BuildSet/Environments/Environment/Variables/Variable"))
{
XmlAttribute NameAttr = Node.Attributes["Name"];
if (NameAttr != null && String.Compare(NameAttr.InnerText, "INCLUDE") == 0)
{
foreach (string IncludePath in Node.Attributes["Value"].InnerText.Split(';'))
{
StandardIncludePaths.Add(new DirectoryReference(IncludePath));
}
}
}
// Read all the individual compiles
Dictionary <FileReference, CompileEnvironment> NewFileToEnvironment = new Dictionary <FileReference, CompileEnvironment>();
foreach (XmlNode Node in Document.SelectNodes("BuildSet/Environments/Environment/Tools/Tool"))
{
XmlAttribute ToolPathAttr = Node.Attributes["Path"];
if (ToolPathAttr != null)
{
// Get the full path to the tool
FileReference ToolLocation = new FileReference(ToolPathAttr.InnerText);
// Get the compiler type
CompilerType CompilerType;
if (GetCompilerType(ToolLocation, out CompilerType))
{
string Name = Node.Attributes["Name"].InnerText;
string Params = Node.Attributes["Params"].InnerText;
// Construct the compile environment
CompileEnvironment Environment = new CompileEnvironment(ToolLocation, CompilerType);
// Parse a list of tokens
List <string> Tokens = new List <string>();
ParseArgumentTokens(CompilerType, Params, Tokens);
// Parse it into a list of options, removing any that don't apply
List <FileReference> SourceFiles = new List <FileReference>();
List <FileReference> OutputFiles = new List <FileReference>();
for (int Idx = 0; Idx < Tokens.Count; Idx++)
{
if (Tokens[Idx] == "/Fo" || Tokens[Idx] == "/Fp" || Tokens[Idx] == "-o")
{
OutputFiles.Add(new FileReference(Tokens[++Idx]));
}
else if (Tokens[Idx].StartsWith("/Fo") || Tokens[Idx].StartsWith("/Fp"))
{
OutputFiles.Add(new FileReference(Tokens[Idx].Substring(3)));
}
else if (Tokens[Idx] == "/D" || Tokens[Idx] == "-D")
{
Environment.Definitions.Add(Tokens[++Idx]);
}
else if (Tokens[Idx].StartsWith("/D"))
{
Environment.Definitions.Add(Tokens[Idx].Substring(2));
}
else if (Tokens[Idx] == "/I" || Tokens[Idx] == "-I")
{
Environment.IncludePaths.Add(new DirectoryReference(DirectoryReference.Combine(BaseDir, Tokens[++Idx].Replace("//", "/")).FullName.ToLowerInvariant()));
}
else if (Tokens[Idx].StartsWith("-I"))
{
Environment.IncludePaths.Add(new DirectoryReference(DirectoryReference.Combine(BaseDir, Tokens[Idx].Substring(2).Replace("//", "/")).FullName.ToLowerInvariant()));
}
else if (Tokens[Idx] == "-Xclang" || Tokens[Idx] == "-target" || Tokens[Idx] == "--target" || Tokens[Idx] == "-x" || Tokens[Idx] == "-o")
{
Environment.Options.Add(new CompileOption(Tokens[Idx], Tokens[++Idx]));
}
else if (Tokens[Idx].StartsWith("/") || Tokens[Idx].StartsWith("-"))
{
Environment.Options.Add(new CompileOption(Tokens[Idx], null));
}
else
{
SourceFiles.Add(FileReference.Combine(BaseDir, Tokens[Idx]));
}
}
// Make sure we're not compiling a precompiled header
if (!OutputFiles.Any(x => x.HasExtension(".gch")) && !Environment.Options.Any(x => x.Name.StartsWith("/Yc")))
{
// Add all the standard include paths
Environment.IncludePaths.AddRange(StandardIncludePaths);
// Add to the output map if there are any source files. Use the extension to distinguish between a source file and linker invocation on Clang.
foreach (FileReference SourceFile in SourceFiles)
{
if (!SourceFile.HasExtension(".a"))
{
NewFileToEnvironment.Add(SourceFile, Environment);
}
}
}
}
}
}
FileToEnvironment = NewFileToEnvironment;
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
/// <returns>True if the task succeeded</returns>
public override bool Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Get the pattern to match against. If it's a simple pattern (eg. *.cpp, Engine/Build/...), automatically infer the source wildcard
string FromPattern = Parameters.From;
if (FromPattern == null)
{
List <string> Patterns = SplitDelimitedList(Parameters.Files);
if (Patterns.Count != 1 || Patterns[0].StartsWith("#"))
{
CommandUtils.LogError("Missing 'From' attribute specifying pattern to match source files against");
return(false);
}
FromPattern = Patterns[0];
int SlashIdx = FromPattern.LastIndexOfAny(new char[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar });
if (SlashIdx != -1)
{
FromPattern = FromPattern.Substring(SlashIdx + 1);
}
if (FromPattern.StartsWith("..."))
{
FromPattern = "*" + FromPattern.Substring(3);
}
}
// Convert the source pattern into a regex
string EscapedFromPattern = "^" + Regex.Escape(FromPattern) + "$";
EscapedFromPattern = EscapedFromPattern.Replace("\\*", "(.*)");
EscapedFromPattern = EscapedFromPattern.Replace("\\?", "(.)");
Regex FromRegex = new Regex(EscapedFromPattern, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
// Split the output pattern into fragments that we can insert captures between
string[] FromFragments = FromPattern.Split('*', '?');
string[] ToFragments = Parameters.To.Split('*', '?');
if (FromFragments.Length < ToFragments.Length)
{
CommandUtils.LogError("Too few capture groups in source pattern '{0}' to rename to '{1}'", FromPattern, Parameters.To);
return(false);
}
// Find the input files
HashSet <FileReference> InputFiles = ResolveFilespec(CommandUtils.RootDirectory, Parameters.Files, TagNameToFileSet);
// Find all the corresponding output files
Dictionary <FileReference, FileReference> RenameFiles = new Dictionary <FileReference, FileReference>();
foreach (FileReference InputFile in InputFiles)
{
Match Match = FromRegex.Match(InputFile.GetFileName());
if (Match.Success)
{
StringBuilder OutputName = new StringBuilder(ToFragments[0]);
for (int Idx = 1; Idx < ToFragments.Length; Idx++)
{
OutputName.Append(Match.Groups[Idx].Value);
OutputName.Append(ToFragments[Idx]);
}
RenameFiles[InputFile] = FileReference.Combine(InputFile.Directory, OutputName.ToString());
}
}
// Print out everything we're going to do
foreach (KeyValuePair <FileReference, FileReference> Pair in RenameFiles)
{
CommandUtils.RenameFile(Pair.Key.FullName, Pair.Value.FullName, true);
}
// Add the build product
BuildProducts.UnionWith(RenameFiles.Values);
// Apply the optional output tag to them
foreach (string TagName in FindTagNamesFromList(Parameters.Tag))
{
FindOrAddTagSet(TagNameToFileSet, TagName).UnionWith(RenameFiles.Values);
}
return(true);
}
private static void GenerateTempTarget(FileReference RawProjectPath)
{
DirectoryReference TempDir = DirectoryReference.Combine(RawProjectPath.Directory, "Intermediate", "Source");
DirectoryReference.CreateDirectory(TempDir);
// Get the project name for use in temporary files
string ProjectName = RawProjectPath.GetFileNameWithoutExtension();
// Create a target.cs file
MemoryStream TargetStream = new MemoryStream();
using (StreamWriter Writer = new StreamWriter(TargetStream))
{
Writer.WriteLine("using UnrealBuildTool;");
Writer.WriteLine();
Writer.WriteLine("public class {0}Target : TargetRules", ProjectName);
Writer.WriteLine("{");
Writer.WriteLine("\tpublic {0}Target(TargetInfo Target) : base(Target)", ProjectName);
Writer.WriteLine("\t{");
Writer.WriteLine("\t\tType = TargetType.Game;");
Writer.WriteLine("\t\tExtraModuleNames.Add(\"{0}\");", ProjectName);
Writer.WriteLine("\t}");
Writer.WriteLine("}");
}
FileReference TargetLocation = FileReference.Combine(TempDir, ProjectName + ".Target.cs");
FileReference.WriteAllBytesIfDifferent(TargetLocation, TargetStream.ToArray());
// Create a build.cs file
MemoryStream ModuleStream = new MemoryStream();
using (StreamWriter Writer = new StreamWriter(ModuleStream))
{
Writer.WriteLine("using UnrealBuildTool;");
Writer.WriteLine();
Writer.WriteLine("public class {0} : ModuleRules", ProjectName);
Writer.WriteLine("{");
Writer.WriteLine("\tpublic {0}(ReadOnlyTargetRules Target) : base(Target)", ProjectName);
Writer.WriteLine("\t{");
Writer.WriteLine("\t\tPCHUsage = PCHUsageMode.UseExplicitOrSharedPCHs;");
Writer.WriteLine();
Writer.WriteLine("\t\tPrivateDependencyModuleNames.Add(\"Core\");");
Writer.WriteLine("\t\tPrivateDependencyModuleNames.Add(\"Core\");");
Writer.WriteLine("\t}");
Writer.WriteLine("}");
}
FileReference ModuleLocation = FileReference.Combine(TempDir, ProjectName + ".Build.cs");
FileReference.WriteAllBytesIfDifferent(ModuleLocation, ModuleStream.ToArray());
// Create a main module cpp file
MemoryStream SourceFileStream = new MemoryStream();
using (StreamWriter Writer = new StreamWriter(SourceFileStream))
{
Writer.WriteLine("#include \"CoreTypes.h\"");
Writer.WriteLine("#include \"Modules/ModuleManager.h\"");
Writer.WriteLine();
Writer.WriteLine("IMPLEMENT_PRIMARY_GAME_MODULE(FDefaultModuleImpl, {0}, \"{0}\");", ProjectName);
}
FileReference SourceFileLocation = FileReference.Combine(TempDir, ProjectName + ".cpp");
FileReference.WriteAllBytesIfDifferent(SourceFileLocation, SourceFileStream.ToArray());
}
public override void GetFilesToDeployOrStage(ProjectParams Params, DeploymentContext SC)
{
// Stage all the build products
foreach (StageTarget Target in SC.StageTargets)
{
SC.StageBuildProductsFromReceipt(Target.Receipt, Target.RequireFilesExist, Params.bTreatNonShippingBinariesAsDebugFiles);
}
if (SC.bStageCrashReporter)
{
StagedDirectoryReference CrashReportClientPath = StagedDirectoryReference.Combine("Engine/Binaries", SC.PlatformDir, "CrashReportClient.app");
StageAppBundle(SC, DirectoryReference.Combine(SC.LocalRoot, "Engine/Binaries", SC.PlatformDir, "CrashReportClient.app"), CrashReportClientPath);
}
// Find the app bundle path
List <FileReference> Exes = GetExecutableNames(SC);
foreach (var Exe in Exes)
{
StagedDirectoryReference AppBundlePath = null;
if (Exe.IsUnderDirectory(DirectoryReference.Combine(SC.RuntimeProjectRootDir, "Binaries", SC.PlatformDir)))
{
AppBundlePath = StagedDirectoryReference.Combine(SC.ShortProjectName, "Binaries", SC.PlatformDir, Path.GetFileNameWithoutExtension(Exe.FullName) + ".app");
}
else if (Exe.IsUnderDirectory(DirectoryReference.Combine(SC.RuntimeRootDir, "Engine/Binaries", SC.PlatformDir)))
{
AppBundlePath = StagedDirectoryReference.Combine("Engine/Binaries", SC.PlatformDir, Path.GetFileNameWithoutExtension(Exe.FullName) + ".app");
}
// Copy the custom icon and Steam dylib, if needed
if (AppBundlePath != null)
{
FileReference AppIconsFile = FileReference.Combine(SC.ProjectRoot, "Build", "Mac", "Application.icns");
if (FileReference.Exists(AppIconsFile))
{
SC.StageFile(StagedFileType.NonUFS, AppIconsFile, StagedFileReference.Combine(AppBundlePath, "Contents", "Resources", "Application.icns"));
}
}
}
// Copy the splash screen, Mac specific
FileReference SplashImage = FileReference.Combine(SC.ProjectRoot, "Content", "Splash", "Splash.bmp");
if (FileReference.Exists(SplashImage))
{
SC.StageFile(StagedFileType.NonUFS, SplashImage);
}
// Stage the bootstrap executable
if (!Params.NoBootstrapExe)
{
foreach (StageTarget Target in SC.StageTargets)
{
BuildProduct Executable = Target.Receipt.BuildProducts.FirstOrDefault(x => x.Type == BuildProductType.Executable);
if (Executable != null)
{
// only create bootstraps for executables
List <StagedFileReference> StagedFiles = SC.FilesToStage.NonUFSFiles.Where(x => x.Value == Executable.Path).Select(x => x.Key).ToList();
if (StagedFiles.Count > 0 && Executable.Path.FullName.Replace("\\", "/").Contains("/" + TargetPlatformType.ToString() + "/"))
{
string BootstrapArguments = "";
if (!ShouldStageCommandLine(Params, SC))
{
if (!SC.IsCodeBasedProject)
{
BootstrapArguments = String.Format("../../../{0}/{0}.uproject", SC.ShortProjectName);
}
else
{
BootstrapArguments = SC.ShortProjectName;
}
}
string BootstrapExeName;
if (SC.StageTargetConfigurations.Count > 1)
{
BootstrapExeName = Path.GetFileName(Executable.Path.FullName) + ".app";
}
else if (Params.IsCodeBasedProject)
{
BootstrapExeName = Target.Receipt.TargetName + ".app";
}
else
{
BootstrapExeName = SC.ShortProjectName + ".app";
}
string AppPath = Executable.Path.FullName.Substring(0, Executable.Path.FullName.LastIndexOf(".app/") + 4);
foreach (var DestPath in StagedFiles)
{
string AppRelativePath = DestPath.Name.Substring(0, DestPath.Name.LastIndexOf(".app/") + 4);
StageBootstrapExecutable(SC, BootstrapExeName, AppPath, AppRelativePath, BootstrapArguments);
}
}
}
}
}
// Copy the ShaderCache files, if they exist
FileReference DrawCacheFile = FileReference.Combine(SC.ProjectRoot, "Content", "DrawCache.ushadercache");
if (FileReference.Exists(DrawCacheFile))
{
SC.StageFile(StagedFileType.UFS, DrawCacheFile);
}
FileReference ByteCodeCacheFile = FileReference.Combine(SC.ProjectRoot, "Content", "ByteCodeCache.ushadercode");
if (FileReference.Exists(ByteCodeCacheFile))
{
SC.StageFile(StagedFileType.UFS, ByteCodeCacheFile);
}
{
// Stage any *.metallib files as NonUFS.
// Get the final output directory for cooked data
DirectoryReference CookOutputDir;
if (!String.IsNullOrEmpty(Params.CookOutputDir))
{
CookOutputDir = DirectoryReference.Combine(new DirectoryReference(Params.CookOutputDir), SC.CookPlatform);
}
else if (Params.CookInEditor)
{
CookOutputDir = DirectoryReference.Combine(SC.ProjectRoot, "Saved", "EditorCooked", SC.CookPlatform);
}
else
{
CookOutputDir = DirectoryReference.Combine(SC.ProjectRoot, "Saved", "Cooked", SC.CookPlatform);
}
if (DirectoryReference.Exists(CookOutputDir))
{
List <FileReference> CookedFiles = DirectoryReference.EnumerateFiles(CookOutputDir, "*.metallib", SearchOption.AllDirectories).ToList();
foreach (FileReference CookedFile in CookedFiles)
{
SC.StageFile(StagedFileType.NonUFS, CookedFile, new StagedFileReference(CookedFile.MakeRelativeTo(CookOutputDir)));
}
}
}
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="Type">The type of probe to create</param>
/// <param name="Node">The node to optimize</param>
/// <param name="IntermediateDir">Directory for intermediate files</param>
/// <param name="UniqueName">Unique name prefix for all temporary files</param>
public SequenceProbe(SequenceProbeType Type, SourceFragment[] Fragments, Tuple <int, SourceFile>[] IncludeHistory, CompileEnvironment CompileEnvironment, DirectoryReference IntermediateDir, IEnumerable <DirectoryReference> ExtraSystemIncludePaths, string UniqueName)
{
this.Type = Type;
this.IntermediateDir = IntermediateDir;
this.Fragments = Fragments;
this.LastFragment = Fragments[Fragments.Length - 1];
this.UniqueName = UniqueName;
this.TaskStateFile = FileReference.Combine(IntermediateDir, UniqueName + ((Type == SequenceProbeType.Verify)? ".verify.state" : ".state"));
this.SummaryLogFile = FileReference.Combine(IntermediateDir, UniqueName + ".summary.txt");
this.CompileLogFile = FileReference.Combine(IntermediateDir, UniqueName + ".compile.txt");
// Get the file to use for trying to compile different permutations
FileReference PermutationFile = FileReference.Combine(IntermediateDir, UniqueName + ".permutation");
// Create the response file
FileReference ResponseFile = FileReference.Combine(IntermediateDir, UniqueName + ".response");
CompileEnvironment WorkerCompileEnvironment = new CompileEnvironment(CompileEnvironment);
if (WorkerCompileEnvironment.CompilerType == CompilerType.Clang)
{
WorkerCompileEnvironment.Options.Add(new CompileOption("-o", FileReference.Combine(IntermediateDir, UniqueName + ".o").FullName.Replace('\\', '/')));
}
else
{
WorkerCompileEnvironment.Options.RemoveAll(x => x.Name == "/Z7" || x.Name == "/Zi" || x.Name == "/ZI");
WorkerCompileEnvironment.Options.Add(new CompileOption("/Fo", FileReference.Combine(IntermediateDir, UniqueName + ".obj").FullName));
WorkerCompileEnvironment.Options.Add(new CompileOption("/WX", null));
}
WorkerCompileEnvironment.WriteResponseFile(ResponseFile, PermutationFile);
string ResponseFileDigest = Utility.ComputeDigest(ResponseFile);
// Keep track of the include stack, so we can format the flat fragment list with context
int IncludeHistoryIdx = 0;
List <SourceFile> IncludeStack = new List <SourceFile>();
// Create the script for the probe
List <Tuple <int, string> > Lines = new List <Tuple <int, string> >();
for (int Idx = 0; Idx < Fragments.Length; Idx++)
{
SourceFragment Fragment = Fragments[Idx];
// Figure out which tag it's bound to
int Tag = (Fragment.File.Counterpart == null)? Idx : -1;
// Update the stack for new includes
while (IncludeHistoryIdx < IncludeHistory.Length && IncludeHistory[IncludeHistoryIdx].Item1 == Idx)
{
if (IncludeHistory[IncludeHistoryIdx].Item2 == null)
{
SourceFile IncludeFile = IncludeStack[IncludeStack.Count - 1];
IncludeStack.RemoveAt(IncludeStack.Count - 1);
Lines.Add(new Tuple <int, string>(Tag, String.Format("{0}// END INCLUDE {1}", new string(' ', IncludeStack.Count * 4), IncludeFile.Location.FullName)));
IncludeHistoryIdx++;
}
else if (IncludeHistoryIdx + 1 < IncludeHistory.Length && IncludeHistory[IncludeHistoryIdx + 1].Item2 == null)
{
IncludeHistoryIdx += 2;
}
else
{
SourceFile IncludeFile = IncludeHistory[IncludeHistoryIdx].Item2;
Lines.Add(new Tuple <int, string>(Tag, String.Format("{0}// INCLUDE {1}", new string(' ', IncludeStack.Count * 4), IncludeFile.Location.FullName)));
IncludeStack.Add(IncludeFile);
IncludeHistoryIdx++;
}
}
// Get the indent at this point
string Indent = new string(' ', (IncludeStack.Count - 0) * 4);
// Write out the forward declarations for every symbol referenced in this fragment. We don't want false dependencies caused by forward declarations in other fragments
// if the heuristic for detecting when to use them doesn't work.
if ((Fragment.File.Flags & SourceFileFlags.TranslationUnit) == 0)
{
foreach (KeyValuePair <Symbol, SymbolReferenceType> ReferencedSymbol in Fragment.ReferencedSymbols)
{
if (!String.IsNullOrEmpty(ReferencedSymbol.Key.ForwardDeclaration))
{
Lines.Add(Tuple.Create(Tag, Indent + ReferencedSymbol.Key.ForwardDeclaration));
}
}
}
// Some Clang/GCC system header wrappers require including as system includes in order to make the #include_next directive work
DirectoryReference BaseSystemIncludePath = ExtraSystemIncludePaths.FirstOrDefault(x => Fragment.Location.IsUnderDirectory(x));
if (BaseSystemIncludePath != null)
{
Lines.Add(Tuple.Create(Tag, String.Format("{0}#include <{1}>", Indent, Fragment.Location.MakeRelativeTo(BaseSystemIncludePath))));
}
else
{
Lines.Add(Tuple.Create(Tag, String.Format("{0}#include \"{1}\"", Indent, Fragment.Location.FullName)));
}
}
// Create the new task
string[] FragmentFileNames = Fragments.Select(Fragment => Fragment.Location.FullName).ToArray();
string[] FragmentDigests = Fragments.Select(Fragment => Fragment.Digest ?? "").ToArray();
Worker = new SequenceWorker(PermutationFile.FullName, ResponseFile.FullName, ResponseFileDigest, FragmentFileNames, FragmentDigests, Lines.ToArray(), CompileEnvironment.Compiler.FullName);
AddDependency(Worker, Fragments, Fragments.Length - 1);
// Log the referenced symbols
if (LastFragment.ReferencedSymbols.Count > 0)
{
List <string> ReferenceLog = new List <string>();
ReferenceLog.Add(String.Format("Referenced symbols for {0}:", LastFragment.Location));
foreach (KeyValuePair <Symbol, SymbolReferenceType> Pair in LastFragment.ReferencedSymbols.OrderBy(x => x.Key.Type).ThenBy(x => x.Key.Name))
{
Symbol ReferencedSymbol = Pair.Key;
ReferenceLog.Add(String.Format(" {0}: {1} ({2}; {3})", ReferencedSymbol.Type.ToString(), ReferencedSymbol.Name, Pair.Value.ToString(), ReferencedSymbol.Fragment.Location));
}
ReferenceLog.Add("");
Worker.SummaryLog.InsertRange(0, ReferenceLog);
}
// Check to see if an existing version of the task exists which we can continue
if (Type == SequenceProbeType.Optimize && TaskStateFile.Exists())
{
// Try to read the old task
SequenceWorker OldWorker;
try
{
OldWorker = SequenceWorker.Deserialize(TaskStateFile.FullName);
}
catch (Exception)
{
OldWorker = null;
}
// If it succeeded, compare it to the new task
if (OldWorker != null)
{
SequenceWorker NewWorker = Worker;
// Create a list of fragments which can be ignored, because they're already determined to be not part of the result for the old task
HashSet <string> IgnoreFragments = new HashSet <string>();
for (int Idx = 0; Idx < OldWorker.FragmentCount; Idx++)
{
if (!OldWorker.KnownDependencies.Contains(Idx) && Idx >= OldWorker.RemainingFragmentCount)
{
IgnoreFragments.Add(OldWorker.FragmentFileNames[Idx]);
}
}
IgnoreFragments.ExceptWith(NewWorker.KnownDependencies.Select(x => NewWorker.FragmentFileNames[x]));
// Compute digests for the old and new tasks
string OldDigest = CreateDigest(OldWorker, IgnoreFragments);
string NewDigest = CreateDigest(NewWorker, IgnoreFragments);
if (OldDigest == NewDigest)
{
// Build a map of fragment file names to their new index
Dictionary <string, int> FragmentFileNameToNewIndex = new Dictionary <string, int>();
for (int Idx = 0; Idx < FragmentFileNames.Length; Idx++)
{
string FragmentFileName = FragmentFileNames[Idx];
FragmentFileNameToNewIndex[FragmentFileName] = Idx;
}
// Add known dependencies to the new worker
foreach (int OldKnownDependency in OldWorker.KnownDependencies)
{
string OldFragmentFileName = OldWorker.FragmentFileNames[OldKnownDependency];
int NewKnownDependency = FragmentFileNameToNewIndex[OldFragmentFileName];
NewWorker.KnownDependencies.Add(NewKnownDependency);
}
// Update the remaining count. All these fragments must match, because they're not part of the ignore list.
NewWorker.RemainingFragmentCount = OldWorker.RemainingFragmentCount;
}
}
}
// If this is a cpp file, make sure we have a dependency on the header file with the same name. It may specify linkage for the functions we declare.
FileReference MainFileLocation = Fragments[Fragments.Length - 1].File.Location;
if (MainFileLocation.HasExtension(".cpp"))
{
string HeaderFileName = Path.ChangeExtension(MainFileLocation.GetFileName(), ".h");
for (int FragmentIdx = 0; FragmentIdx < Fragments.Length; FragmentIdx++)
{
if (String.Compare(Fragments[FragmentIdx].File.Location.GetFileName(), HeaderFileName, true) == 0)
{
AddDependency(Worker, Fragments, FragmentIdx);
}
}
}
// Update the finished fragment if we're done, otherwise clear out all the intermediate files
if (Worker.RemainingFragmentCount == 0)
{
SetCompletedDependencies();
}
else
{
Worker.Serialize(TaskStateFile.FullName);
PermutationFile.Delete();
SummaryLogFile.Delete();
CompileLogFile.Delete();
}
}
public override void ExecuteBuild()
{
// just calling this DesiredTargetVersion because TargetVersion and TargetedVersion get super confusing.
string DesiredTargetVersion = this.ParseParamValue("TargetVersion", null);
if (string.IsNullOrEmpty(DesiredTargetVersion))
{
throw new AutomationException("-TargetVersion was not specified.");
}
CommandUtils.Log("Scanning for all csproj's...");
// Check for all csproj's in the engine dir
DirectoryReference EngineDir = CommandUtils.EngineDirectory;
// grab the targeted version.,
Regex FrameworkRegex = new Regex("<TargetFrameworkVersion>v(\\d\\.\\d\\.?\\d?)<\\/TargetFrameworkVersion>");
Regex PossibleAppConfigRegex = new Regex("<TargetFrameworkProfile>(.+)<\\/TargetFrameworkProfile>");
Regex AppConfigRegex = new Regex("<supportedRuntime version=\"v(\\d\\.\\d\\.?\\d?)\" sku=\"\\.NETFramework,Version=v(\\d\\.\\d\\.?\\d?),Profile=(.+)\"\\/>");
Regex DotNetCoreRegex = new Regex("<TargetFramework>(netcoreapp2.0|netstandard2.0)<\\/TargetFramework>");
foreach (FileReference CsProj in DirectoryReference.EnumerateFiles(EngineDir, "*.csproj", SearchOption.AllDirectories))
{
if (CsProj.ContainsName(new FileSystemName("ThirdParty"), EngineDir) ||
(CsProj.ContainsName(new FileSystemName("UE4TemplateProject"), EngineDir) && CsProj.GetFileName().Equals("ProjectTemplate.csproj")) ||
CsProj.GetFileNameWithoutExtension().ToLower().Contains("_mono") ||
CsProj.GetFileNameWithoutExtension().ToLower().Contains("unrealvs"))
{
continue;
}
// read in the file
string Contents = File.ReadAllText(CsProj.FullName);
Match Match = DotNetCoreRegex.Match(Contents);
// Check if we're a _NETCore app, ignore these.
if (Match.Success)
{
continue;
}
Match = FrameworkRegex.Match(Contents);
if (Match.Success)
{
string TargetedVersion = Match.Groups[1].Value;
// make sure we match, throw warning otherwise
if (!DesiredTargetVersion.Equals(TargetedVersion, StringComparison.InvariantCultureIgnoreCase))
{
CommandUtils.LogWarning("Targeted Framework version for project: {0} was not {1}! Targeted Version: {2}", CsProj, DesiredTargetVersion, TargetedVersion);
}
}
// if we don't have a TargetFrameworkVersion, check for the existence of TargetFrameworkProfile.
else
{
Match = PossibleAppConfigRegex.Match(Contents);
if (!Match.Success)
{
CommandUtils.Log("No TargetFrameworkVersion or TargetFrameworkProfile found for project {0}, is it a mono project? If not, does it compile properly?", CsProj);
continue;
}
// look for the app config
FileReference AppConfigFile = FileReference.Combine(CsProj.Directory, "app.config");
string Profile = Match.Groups[1].Value;
if (!FileReference.Exists(AppConfigFile))
{
CommandUtils.Log("Found TargetFrameworkProfile but no associated app.config containing the version for project {0}.", CsProj);
continue;
}
// read in the app config
Contents = File.ReadAllText(AppConfigFile.FullName);
Match = AppConfigRegex.Match(Contents);
if (!Match.Success)
{
CommandUtils.Log("Couldn't find a supportedRuntime match for the version in the app.config for project {0}.", CsProj);
continue;
}
// Version1 is the one that appears right after supportedRuntime
// Version2 is the one in the sku
// ProfileString should match the TargetFrameworkProfile from the csproj
string Version1String = Match.Groups[1].Value;
string Version2String = Match.Groups[2].Value;
string ProfileString = Match.Groups[3].Value;
// not sure how this is possible, but check for it anyway
if (!ProfileString.Equals(Profile, StringComparison.InvariantCultureIgnoreCase))
{
CommandUtils.LogWarning("The TargetFrameworkProfile in csproj {0} ({1}) doesn't match the sku in it's app.config ({2}).", CsProj, Profile, ProfileString);
continue;
}
// if the version numbers don't match the app.config is probably corrupt.
if (!Version1String.Equals(Version2String, StringComparison.InvariantCultureIgnoreCase))
{
CommandUtils.LogWarning("The supportedRunTimeVersion ({0}) and the sku version ({1}) in the app.config for project {2} don't match.", Version1String, Version2String, CsProj);
continue;
}
// make sure the versions match
if (!(DesiredTargetVersion.Equals(Version1String, StringComparison.InvariantCultureIgnoreCase)))
{
CommandUtils.LogWarning("Targeted Framework version for project: {0} was not {1}! Targeted Version: {2}", CsProj, DesiredTargetVersion, Version1String);
}
}
}
}
void StageBootstrapExecutable(DeploymentContext SC, string ExeName, string TargetFile, string StagedRelativeTargetPath, string StagedArguments)
{
DirectoryReference InputApp = DirectoryReference.Combine(SC.LocalRoot, "Engine", "Binaries", SC.PlatformDir, "BootstrapPackagedGame.app");
if (InternalUtils.SafeDirectoryExists(InputApp.FullName))
{
// Create the new bootstrap program
DirectoryReference IntermediateDir = DirectoryReference.Combine(SC.ProjectRoot, "Intermediate", "Staging");
InternalUtils.SafeCreateDirectory(IntermediateDir.FullName);
DirectoryReference IntermediateApp = DirectoryReference.Combine(IntermediateDir, ExeName);
if (DirectoryReference.Exists(IntermediateApp))
{
DirectoryReference.Delete(IntermediateApp, true);
}
CloneDirectory(InputApp.FullName, IntermediateApp.FullName);
// Rename the executable
string GameName = Path.GetFileNameWithoutExtension(ExeName);
FileReference.Move(FileReference.Combine(IntermediateApp, "Contents", "MacOS", "BootstrapPackagedGame"), FileReference.Combine(IntermediateApp, "Contents", "MacOS", GameName));
// Copy the icon
string SrcInfoPlistPath = CombinePaths(TargetFile, "Contents", "Info.plist");
string SrcInfoPlist = File.ReadAllText(SrcInfoPlistPath);
string IconName = GetValueFromInfoPlist(SrcInfoPlist, "CFBundleIconFile");
if (!string.IsNullOrEmpty(IconName))
{
string IconPath = CombinePaths(TargetFile, "Contents", "Resources", IconName + ".icns");
InternalUtils.SafeCreateDirectory(CombinePaths(IntermediateApp.FullName, "Contents", "Resources"));
File.Copy(IconPath, CombinePaths(IntermediateApp.FullName, "Contents", "Resources", IconName + ".icns"));
}
// Update Info.plist contents
string DestInfoPlistPath = CombinePaths(IntermediateApp.FullName, "Contents", "Info.plist");
string DestInfoPlist = File.ReadAllText(DestInfoPlistPath);
string AppIdentifier = GetValueFromInfoPlist(SrcInfoPlist, "CFBundleIdentifier");
if (AppIdentifier == "com.epicgames.UE4Game")
{
AppIdentifier = "";
}
string Copyright = GetValueFromInfoPlist(SrcInfoPlist, "NSHumanReadableCopyright");
string BundleVersion = GetValueFromInfoPlist(SrcInfoPlist, "CFBundleVersion", "1");
string ShortVersion = GetValueFromInfoPlist(SrcInfoPlist, "CFBundleShortVersionString", "1.0");
DestInfoPlist = DestInfoPlist.Replace("com.epicgames.BootstrapPackagedGame", string.IsNullOrEmpty(AppIdentifier) ? "com.epicgames." + GameName + "_bootstrap" : AppIdentifier + "_bootstrap");
DestInfoPlist = DestInfoPlist.Replace("BootstrapPackagedGame", GameName);
DestInfoPlist = DestInfoPlist.Replace("__UE4_ICON_FILE__", IconName);
DestInfoPlist = DestInfoPlist.Replace("__UE4_APP_TO_LAUNCH__", StagedRelativeTargetPath);
DestInfoPlist = DestInfoPlist.Replace("__UE4_COMMANDLINE__", StagedArguments);
DestInfoPlist = DestInfoPlist.Replace("__UE4_COPYRIGHT__", Copyright);
DestInfoPlist = DestInfoPlist.Replace("__UE4_BUNDLE_VERSION__", BundleVersion);
DestInfoPlist = DestInfoPlist.Replace("__UE4_SHORT_VERSION__", ShortVersion);
File.WriteAllText(DestInfoPlistPath, DestInfoPlist);
StageAppBundle(SC, IntermediateApp, new StagedDirectoryReference(ExeName));
}
}
public override void ExecuteBuild()
{
string BucketName = ParseRequiredStringParam("Bucket");
FileReference CredentialsFile = ParseRequiredFileReferenceParam("CredentialsFile");
string CredentialsKey = ParseRequiredStringParam("CredentialsKey");
DirectoryReference CacheDir = ParseRequiredDirectoryReferenceParam("CacheDir");
DirectoryReference FilterDir = ParseRequiredDirectoryReferenceParam("FilterDir");
int Days = ParseParamInt("Days", 7);
int MaxFileSize = ParseParamInt("MaxFileSize", 0);
string RootManifestPath = ParseRequiredStringParam("Manifest");
string KeyPrefix = ParseParamValue("KeyPrefix", "");
bool bReset = ParseParam("Reset");
// The credentials to upload with
AWSCredentials Credentials;
// Try to get the credentials by the key passed in from the script
CredentialProfileStoreChain CredentialsChain = new CredentialProfileStoreChain(CredentialsFile.FullName);
if (!CredentialsChain.TryGetAWSCredentials(CredentialsKey, out Credentials))
{
throw new AutomationException("Unknown credentials key: {0}", CredentialsKey);
}
// Create the new client
using (AmazonS3Client Client = new AmazonS3Client(Credentials, Region))
{
using (SemaphoreSlim RequestSemaphore = new SemaphoreSlim(4))
{
// Read the filters
HashSet <string> Paths = new HashSet <string>();
foreach (FileInfo FilterFile in FilterDir.ToDirectoryInfo().EnumerateFiles("*.txt"))
{
TimeSpan Age = DateTime.UtcNow - FilterFile.LastWriteTimeUtc;
if (Age < TimeSpan.FromDays(3))
{
Log.TraceInformation("Reading {0}", FilterFile.FullName);
string[] Lines = File.ReadAllLines(FilterFile.FullName);
foreach (string Line in Lines)
{
string TrimLine = Line.Trim().Replace('\\', '/');
if (TrimLine.Length > 0)
{
Paths.Add(TrimLine);
}
}
}
else if (Age > TimeSpan.FromDays(5))
{
try
{
Log.TraceInformation("Deleting {0}", FilterFile.FullName);
FilterFile.Delete();
}
catch (Exception Ex)
{
Log.TraceWarning("Unable to delete: {0}", Ex.Message);
Log.TraceLog(ExceptionUtils.FormatExceptionDetails(Ex));
}
}
}
Log.TraceInformation("Found {0:n0} files", Paths.Count);
// Enumerate all the files that are in the network DDC
Log.TraceInformation("");
Log.TraceInformation("Filtering files in {0}...", CacheDir);
List <DerivedDataFile> Files = ParallelExecute <string, DerivedDataFile>(Paths, (Path, FilesBag) => ReadFileInfo(CacheDir, Path, FilesBag));
// Filter to the maximum size
if (MaxFileSize != 0)
{
int NumRemovedMaxSize = Files.RemoveAll(x => x.Info.Length > MaxFileSize);
Log.TraceInformation("");
Log.TraceInformation("Removed {0} files above size limit ({1:n0} bytes)", NumRemovedMaxSize, MaxFileSize);
}
// Create the working directory
DirectoryReference WorkingDir = DirectoryReference.Combine(EngineDirectory, "Saved", "UploadDDC");
DirectoryReference.CreateDirectory(WorkingDir);
// Get the path to the manifest
FileReference RootManifestFile = FileReference.Combine(CommandUtils.RootDirectory, RootManifestPath);
// Read the old root manifest
RootManifest OldRootManifest = new RootManifest();
if (FileReference.Exists(RootManifestFile))
{
OldRootManifest.Read(JsonObject.Read(RootManifestFile));
}
// Read the old bundle manifest
BundleManifest OldBundleManifest = new BundleManifest();
if (OldRootManifest.Entries.Count > 0)
{
FileReference LocalBundleManifest = FileReference.Combine(WorkingDir, "OldBundleManifest.json");
if (TryDownloadFile(Client, BucketName, OldRootManifest.Entries.Last().Key, LocalBundleManifest))
{
OldBundleManifest.Read(JsonObject.Read(LocalBundleManifest));
}
}
// Create the new manifest
BundleManifest NewBundleManifest = new BundleManifest();
// Try to download the old manifest, and add all the bundles we want to keep to the new manifest
if (!bReset)
{
foreach (BundleManifest.Entry Bundle in OldBundleManifest.Entries)
{
FileReference BundleFile = FileReference.Combine(WorkingDir, Bundle.Name);
if (!FileReference.Exists(BundleFile))
{
Log.TraceInformation("Downloading {0}", BundleFile);
FileReference TempCompressedFile = new FileReference(BundleFile.FullName + ".incoming.gz");
if (!TryDownloadFile(Client, BucketName, Bundle.ObjectKey, TempCompressedFile))
{
Log.TraceWarning("Unable to download {0}", Bundle.ObjectKey);
continue;
}
FileReference TempUncompressedFile = new FileReference(BundleFile.FullName + ".incoming");
try
{
DecompressFile(TempCompressedFile, TempUncompressedFile);
}
catch (Exception Ex)
{
Log.TraceWarning("Unable to uncompress {0}: {1}", Bundle.ObjectKey, Ex.ToString());
continue;
}
FileReference.Move(TempUncompressedFile, BundleFile);
}
NewBundleManifest.Entries.Add(Bundle);
}
}
// Figure out all the item digests that we already have
Dictionary <BundleManifest.Entry, HashSet <ContentHash> > BundleToKeyHashes = new Dictionary <BundleManifest.Entry, HashSet <ContentHash> >();
foreach (BundleManifest.Entry Bundle in NewBundleManifest.Entries)
{
HashSet <ContentHash> KeyHashes = new HashSet <ContentHash>();
FileReference BundleFile = FileReference.Combine(WorkingDir, Bundle.Name);
using (FileStream Stream = FileReference.Open(BundleFile, FileMode.Open, FileAccess.Read, FileShare.Read))
{
BinaryReader Reader = new BinaryReader(Stream);
uint Signature = Reader.ReadUInt32();
if (Signature != BundleSignatureV1)
{
throw new Exception(String.Format("Invalid signature for {0}", BundleFile));
}
int NumEntries = Reader.ReadInt32();
for (int EntryIdx = 0; EntryIdx < NumEntries; EntryIdx++)
{
byte[] Digest = new byte[ContentHash.LengthSHA1];
if (Reader.Read(Digest, 0, ContentHash.LengthSHA1) != ContentHash.LengthSHA1)
{
throw new Exception("Unexpected EOF");
}
KeyHashes.Add(new ContentHash(Digest));
Stream.Seek(4, SeekOrigin.Current);
}
}
BundleToKeyHashes[Bundle] = KeyHashes;
}
// Calculate the download size of the manifest
long DownloadSize = NewBundleManifest.Entries.Sum(x => (long)x.CompressedLength);
// Remove any bundles which have less than the minimum required size in valid data. We don't mark the manifest as dirty yet; these
// files will only be rewritten if new content is added, to prevent the last bundle being rewritten multiple times.
foreach (KeyValuePair <BundleManifest.Entry, HashSet <ContentHash> > Pair in BundleToKeyHashes)
{
long ValidBundleSize = Files.Where(x => Pair.Value.Contains(x.KeyHash)).Sum(x => (long)x.Info.Length);
if (ValidBundleSize < MinBundleSize)
{
NewBundleManifest.Entries.Remove(Pair.Key);
}
}
// Find all the valid digests
HashSet <ContentHash> ReusedKeyHashes = new HashSet <ContentHash>();
foreach (BundleManifest.Entry Bundle in NewBundleManifest.Entries)
{
ReusedKeyHashes.UnionWith(BundleToKeyHashes[Bundle]);
}
// Remove all the files which already exist
int NumRemovedExist = Files.RemoveAll(x => ReusedKeyHashes.Contains(x.KeyHash));
if (NumRemovedExist > 0)
{
Log.TraceInformation("");
Log.TraceInformation("Removed {0:n0} files which already exist", NumRemovedExist);
}
// Read all the files we want to include
List <Tuple <DerivedDataFile, byte[]> > FilesToInclude = new List <Tuple <DerivedDataFile, byte[]> >();
if (Files.Count > 0)
{
Log.TraceInformation("");
Log.TraceInformation("Reading remaining {0:n0} files into memory ({1:n1}mb)...", Files.Count, (float)Files.Sum(x => (long)x.Info.Length) / (1024 * 1024));
FilesToInclude.AddRange(ParallelExecute <DerivedDataFile, Tuple <DerivedDataFile, byte[]> >(Files, (x, y) => ReadFileData(x, y)));
}
// Generate new data
using (RNGCryptoServiceProvider Crypto = new RNGCryptoServiceProvider())
{
// Flag for whether to update the manifest
bool bUpdateManifest = false;
// Upload the new bundle
Log.TraceInformation("");
if (FilesToInclude.Count == 0)
{
Log.TraceInformation("No new files to add.");
}
else
{
// Sort the files to include by creation time. This will bias towards grouping older, more "permanent", items together.
Log.TraceInformation("Sorting input files");
List <Tuple <DerivedDataFile, byte[]> > SortedFilesToInclude = FilesToInclude.OrderBy(x => x.Item1.Info.CreationTimeUtc).ToList();
// Get the target bundle size
long TotalSize = SortedFilesToInclude.Sum(x => (long)x.Item2.Length);
int NumBundles = (int)((TotalSize + (MaxBundleSize - 1)) / MaxBundleSize);
long TargetBundleSize = TotalSize / NumBundles;
// Split the input data into bundles
List <List <Tuple <DerivedDataFile, byte[]> > > BundleFilesToIncludeList = new List <List <Tuple <DerivedDataFile, byte[]> > >();
long BundleSize = 0;
for (int FileIdx = 0; FileIdx < SortedFilesToInclude.Count; BundleSize = BundleSize % TargetBundleSize)
{
List <Tuple <DerivedDataFile, byte[]> > BundleFilesToInclude = new List <Tuple <DerivedDataFile, byte[]> >();
for (; BundleSize < TargetBundleSize && FileIdx < SortedFilesToInclude.Count; FileIdx++)
{
BundleFilesToInclude.Add(SortedFilesToInclude[FileIdx]);
BundleSize += SortedFilesToInclude[FileIdx].Item2.Length;
}
BundleFilesToIncludeList.Add(BundleFilesToInclude);
}
// Upload each bundle
DateTime NewBundleTime = DateTime.UtcNow;
for (int BundleIdx = 0; BundleIdx < BundleFilesToIncludeList.Count; BundleIdx++)
{
List <Tuple <DerivedDataFile, byte[]> > BundleFilesToInclude = BundleFilesToIncludeList[BundleIdx];
// Get the new bundle info
string NewBundleSuffix = (BundleFilesToIncludeList.Count > 1) ? String.Format("-{0}_of_{1}", BundleIdx + 1, BundleFilesToIncludeList.Count) : "";
string NewBundleName = String.Format("Bundle-{0:yyyy.MM.dd-HH.mm}{1}.ddb", NewBundleTime.ToLocalTime(), NewBundleSuffix);
// Create a random number for the object key
string NewBundleObjectKey = KeyPrefix + "bulk/" + CreateObjectName(Crypto);
// Create the bundle header
byte[] Header;
using (MemoryStream HeaderStream = new MemoryStream())
{
BinaryWriter Writer = new BinaryWriter(HeaderStream);
Writer.Write(BundleSignatureV1);
Writer.Write(BundleFilesToInclude.Count);
foreach (Tuple <DerivedDataFile, byte[]> FileToInclude in BundleFilesToInclude)
{
Writer.Write(FileToInclude.Item1.KeyHash.Bytes, 0, ContentHash.LengthSHA1);
Writer.Write((int)FileToInclude.Item2.Length);
}
Header = HeaderStream.ToArray();
}
// Create the output file
FileReference NewBundleFile = FileReference.Combine(WorkingDir, NewBundleName + ".gz");
Log.TraceInformation("Writing {0}", NewBundleFile);
using (FileStream BundleStream = FileReference.Open(NewBundleFile, FileMode.Create, FileAccess.Write, FileShare.Read))
{
using (GZipStream ZipStream = new GZipStream(BundleStream, CompressionLevel.Optimal, true))
{
ZipStream.Write(Header, 0, Header.Length);
foreach (Tuple <DerivedDataFile, byte[]> FileToInclude in BundleFilesToInclude)
{
ZipStream.Write(FileToInclude.Item2, 0, FileToInclude.Item2.Length);
}
}
}
// Upload the file
long NewBundleCompressedLength = NewBundleFile.ToFileInfo().Length;
long NewBundleUncompressedLength = Header.Length + BundleFilesToInclude.Sum(x => (long)x.Item2.Length);
Log.TraceInformation("Uploading bundle to {0} ({1:n1}mb)", NewBundleObjectKey, NewBundleCompressedLength / (1024.0f * 1024.0f));
UploadFile(Client, BucketName, NewBundleFile, 0, NewBundleObjectKey, RequestSemaphore, null);
// Add the bundle to the new manifest
BundleManifest.Entry Bundle = new BundleManifest.Entry();
Bundle.Name = NewBundleName;
Bundle.ObjectKey = NewBundleObjectKey;
Bundle.Time = NewBundleTime;
Bundle.CompressedLength = (int)NewBundleCompressedLength;
Bundle.UncompressedLength = (int)NewBundleUncompressedLength;
NewBundleManifest.Entries.Add(Bundle);
// Mark the manifest as requiring an update
bUpdateManifest = true;
}
}
// Update the manifest
if (bUpdateManifest)
{
DateTime UtcNow = DateTime.UtcNow;
DateTime RemoveBundleManifestsBefore = UtcNow - TimeSpan.FromDays(3.0);
// Update the root manifest
RootManifest NewRootManifest = new RootManifest();
NewRootManifest.AccessKey = OldRootManifest.AccessKey;
NewRootManifest.SecretKey = OldRootManifest.SecretKey;
foreach (RootManifest.Entry Entry in OldRootManifest.Entries)
{
if (Entry.CreateTime >= RemoveBundleManifestsBefore)
{
NewRootManifest.Entries.Add(Entry);
}
}
// Make sure there's an entry for the last 24h
DateTime RequireBundleManifestAfter = UtcNow - TimeSpan.FromDays(1.0);
if (!NewRootManifest.Entries.Any(x => x.CreateTime > RequireBundleManifestAfter))
{
RootManifest.Entry NewEntry = new RootManifest.Entry();
NewEntry.CreateTime = UtcNow;
NewEntry.Key = KeyPrefix + CreateObjectName(Crypto);
NewRootManifest.Entries.Add(NewEntry);
}
// Save out the new bundle manifest
FileReference NewBundleManifestFile = FileReference.Combine(WorkingDir, "NewBundleManifest.json");
NewBundleManifest.Save(NewBundleManifestFile);
// Update all the bundle manifests still valid
foreach (RootManifest.Entry Entry in NewRootManifest.Entries)
{
Log.TraceInformation("Uploading bundle manifest to {0}", Entry.Key);
UploadFile(Client, BucketName, NewBundleManifestFile, 0, Entry.Key, RequestSemaphore, null);
}
// Overwrite all the existing manifests
if (AllowSubmit)
{
List <string> ExistingFiles = P4.Files(CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName));
// Create a changelist containing the new manifest
int ChangeNumber = P4.CreateChange(Description: "Updating DDC bundle manifest");
if (ExistingFiles.Count > 0)
{
P4.Edit(ChangeNumber, CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName));
NewRootManifest.Save(RootManifestFile);
}
else
{
NewRootManifest.Save(RootManifestFile);
P4.Add(ChangeNumber, CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName));
}
// Submit it
int SubmittedChangeNumber;
P4.Submit(ChangeNumber, out SubmittedChangeNumber, true);
if (SubmittedChangeNumber <= 0)
{
throw new AutomationException("Failed to submit change");
}
// Delete any bundles that are no longer referenced
HashSet <string> KeepObjectKeys = new HashSet <string>(NewBundleManifest.Entries.Select(x => x.ObjectKey));
foreach (BundleManifest.Entry OldEntry in OldBundleManifest.Entries)
{
if (!KeepObjectKeys.Contains(OldEntry.ObjectKey))
{
Log.TraceInformation("Deleting unreferenced bundle {0}", OldEntry.ObjectKey);
DeleteFile(Client, BucketName, OldEntry.ObjectKey, RequestSemaphore, null);
}
}
// Delete any bundle manifests which are no longer referenced
HashSet <string> KeepManifestKeys = new HashSet <string>(NewRootManifest.Entries.Select(x => x.Key));
foreach (RootManifest.Entry OldEntry in OldRootManifest.Entries)
{
if (!KeepManifestKeys.Contains(OldEntry.Key))
{
Log.TraceInformation("Deleting unreferenced manifest {0}", OldEntry.Key);
DeleteFile(Client, BucketName, OldEntry.Key, RequestSemaphore, null);
}
}
}
else
{
// Skip submitting
Log.TraceWarning("Skipping manifest submit due to missing -Submit argument.");
}
// Update the new download size
DownloadSize = NewBundleManifest.Entries.Sum(x => (long)x.CompressedLength);
}
}
// Print some stats about the final manifest
Log.TraceInformation("");
Log.TraceInformation("Total download size {0:n1}mb", DownloadSize / (1024.0 * 1024.0));
Log.TraceInformation("");
}
}
}
public override void ExecuteBuild()
{
var Params = new ProjectParams
(
Command: this,
// Shared
RawProjectPath: ProjectPath
);
LogInformation("********** CRYPTOKEYS COMMAND STARTED **********");
string UE4EditorExe = HostPlatform.Current.GetUE4ExePath(Params.UE4Exe);
if (!FileExists(UE4EditorExe))
{
throw new AutomationException("Missing " + UE4EditorExe + " executable. Needs to be built first.");
}
bool bCycleAllKeys = ParseParam("updateallkeys");
bool bCycleEncryptionKey = bCycleAllKeys || ParseParam("updateencryptionkey");
bool bCycleSigningKey = bCycleAllKeys || ParseParam("updatesigningkey");
if (!bCycleAllKeys && !bCycleEncryptionKey && !bCycleSigningKey)
{
throw new Exception("A target for key cycling must be specified when using the cryptokeys automation script\n\t-updateallkeys: Update all keys\n\t-updateencryptionkey: Update encryption key\n\t-updatesigningkey: Update signing key");
}
FileReference OutputFile = FileReference.Combine(ProjectPath.Directory, "Config", "DefaultCrypto.ini");
FileReference NoRedistOutputFile = FileReference.Combine(ProjectPath.Directory, "Config", "NoRedist", "DefaultCrypto.ini");
FileReference DestinationFile = OutputFile;
// If the project has a DefaultCrypto.ini in a NoRedist folder, we want to copy the newly generated file into that location
if (FileReference.Exists(NoRedistOutputFile))
{
DestinationFile = NoRedistOutputFile;
}
string ChangeDescription = "Automated update of ";
if (bCycleEncryptionKey)
{
ChangeDescription += "encryption";
}
if (bCycleSigningKey)
{
if (bCycleEncryptionKey)
{
ChangeDescription += " and ";
}
ChangeDescription += "signing";
}
ChangeDescription += " key";
if (bCycleEncryptionKey && bCycleSigningKey)
{
ChangeDescription += "s";
}
ChangeDescription += " for project " + Params.ShortProjectName;
P4Connection SubmitP4 = null;
int NewCL = 0;
if (CommandUtils.P4Enabled)
{
SubmitP4 = CommandUtils.P4;
NewCL = SubmitP4.CreateChange(Description: ChangeDescription);
SubmitP4.Revert(String.Format("-k \"{0}\"", DestinationFile.FullName));
SubmitP4.Sync(String.Format("-k \"{0}\"", DestinationFile.FullName), AllowSpew: false);
SubmitP4.Add(NewCL, String.Format("\"{0}\"", DestinationFile.FullName));
SubmitP4.Edit(NewCL, String.Format("\"{0}\"", DestinationFile.FullName));
}
else
{
LogInformation(ChangeDescription);
FileReference.MakeWriteable(OutputFile);
}
string CommandletParams = "";
if (bCycleAllKeys)
{
CommandletParams = "-updateallkeys";
}
else if (bCycleEncryptionKey)
{
CommandletParams = "-updateencryptionkey";
}
else if (bCycleSigningKey)
{
CommandletParams = "-updatesigningkey";
}
RunCommandlet(ProjectPath, UE4EditorExe, "CryptoKeys", CommandletParams);
if (DestinationFile != OutputFile)
{
File.Delete(DestinationFile.FullName);
FileReference.Move(OutputFile, DestinationFile);
}
if (SubmitP4 != null)
{
int ActualCL;
SubmitP4.Submit(NewCL, out ActualCL);
}
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
/// <returns>True if the task succeeded</returns>
public override bool Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Get the project path, and check it exists
FileReference ProjectFile = null;
if (Parameters.Project != null)
{
ProjectFile = ResolveFile(Parameters.Project);
if (!ProjectFile.Exists())
{
CommandUtils.LogError("Couldn't find project '{0}'", ProjectFile.FullName);
return(false);
}
}
// Get the directories used for staging this project
DirectoryReference SourceEngineDir = UnrealBuildTool.UnrealBuildTool.EngineDirectory;
DirectoryReference SourceProjectDir = (ProjectFile == null)? SourceEngineDir : ProjectFile.Directory;
// Get the output directories. We flatten the directory structure on output.
DirectoryReference TargetDir = ResolveDirectory(Parameters.ToDir);
DirectoryReference TargetEngineDir = DirectoryReference.Combine(TargetDir, "Engine");
DirectoryReference TargetProjectDir = DirectoryReference.Combine(TargetDir, ProjectFile.GetFileNameWithoutExtension());
// Get the path to the receipt
string ReceiptFileName = TargetReceipt.GetDefaultPath(SourceProjectDir.FullName, Parameters.Target, Parameters.Platform, Parameters.Configuration, Parameters.Architecture);
// Try to load it
TargetReceipt Receipt;
if (!TargetReceipt.TryRead(ReceiptFileName, out Receipt))
{
CommandUtils.LogError("Couldn't read receipt '{0}'", ReceiptFileName);
return(false);
}
// Expand all the paths from the receipt
Receipt.ExpandPathVariables(SourceEngineDir, SourceProjectDir);
// Stage all the build products needed at runtime
HashSet <FileReference> SourceFiles = new HashSet <FileReference>();
foreach (BuildProduct BuildProduct in Receipt.BuildProducts.Where(x => x.Type != BuildProductType.StaticLibrary && x.Type != BuildProductType.ImportLibrary))
{
SourceFiles.Add(new FileReference(BuildProduct.Path));
}
foreach (RuntimeDependency RuntimeDependency in Receipt.RuntimeDependencies.Where(x => x.Type != StagedFileType.UFS))
{
SourceFiles.UnionWith(CommandUtils.ResolveFilespec(CommandUtils.RootDirectory, RuntimeDependency.Path, new string[] { ".../*.umap", ".../*.uasset" }));
}
// Get all the target files
List <FileReference> TargetFiles = new List <FileReference>();
foreach (FileReference SourceFile in SourceFiles)
{
// Get the destination file to copy to, mapping to the new engine and project directories as appropriate
FileReference TargetFile;
if (SourceFile.IsUnderDirectory(SourceEngineDir))
{
TargetFile = FileReference.Combine(TargetEngineDir, SourceFile.MakeRelativeTo(SourceEngineDir));
}
else
{
TargetFile = FileReference.Combine(TargetProjectDir, SourceFile.MakeRelativeTo(SourceProjectDir));
}
// Fixup the case of the output file. Would expect Platform.DeployLowerCaseFilenames() to return true here, but seems not to be the case.
if (Parameters.Platform == UnrealTargetPlatform.PS4)
{
TargetFile = FileReference.Combine(TargetDir, TargetFile.MakeRelativeTo(TargetDir).ToLowerInvariant());
}
// Only copy the output file if it doesn't already exist. We can stage multiple targets to the same output directory.
if (Parameters.Overwrite || !TargetFile.Exists())
{
TargetFile.Directory.CreateDirectory();
CommandUtils.CopyFile(SourceFile.FullName, TargetFile.FullName);
}
// Add it to the list of target files
TargetFiles.Add(TargetFile);
}
// Apply the optional tag to the build products
foreach (string TagName in FindTagNamesFromList(Parameters.Tag))
{
FindOrAddTagSet(TagNameToFileSet, TagName).UnionWith(TargetFiles);
}
// Add the target file to the list of build products
BuildProducts.UnionWith(TargetFiles);
return(true);
}
/// <summary>
/// Find all the build products created by compiling the given project file
/// </summary>
/// <param name="ProjectFiles">Initial project file to read. All referenced projects will also be read.</param>
/// <param name="InitialProperties">Mapping of property name to value</param>
/// <param name="OutBuildProducts">Receives a set of build products on success</param>
/// <param name="OutReferences">Receives a set of non-private references on success</param>
/// <returns>True if the build products were found, false otherwise.</returns>
static void FindBuildProducts(HashSet <FileReference> ProjectFiles, Dictionary <string, string> InitialProperties, out HashSet <FileReference> OutBuildProducts, out HashSet <FileReference> OutReferences)
{
// Read all the project information into a dictionary
Dictionary <FileReference, CsProjectInfo> FileToProjectInfo = new Dictionary <FileReference, CsProjectInfo>();
foreach (FileReference ProjectFile in ProjectFiles)
{
ReadProjectsRecursively(ProjectFile, InitialProperties, FileToProjectInfo);
}
// Find all the build products and references
HashSet <FileReference> BuildProducts = new HashSet <FileReference>();
HashSet <FileReference> References = new HashSet <FileReference>();
foreach (KeyValuePair <FileReference, CsProjectInfo> Pair in FileToProjectInfo)
{
CsProjectInfo ProjectInfo = Pair.Value;
// Add the standard build products
DirectoryReference OutputDir = ProjectInfo.GetOutputDir(Pair.Key.Directory);
ProjectInfo.AddBuildProducts(OutputDir, BuildProducts);
// Add the referenced assemblies
foreach (KeyValuePair <FileReference, bool> Reference in ProjectInfo.References)
{
FileReference OtherAssembly = Reference.Key;
if (Reference.Value)
{
// Add reference from the output dir
FileReference OutputFile = FileReference.Combine(OutputDir, OtherAssembly.GetFileName());
BuildProducts.Add(OutputFile);
FileReference OutputSymbolFile = OutputFile.ChangeExtension(".pdb");
if (FileReference.Exists(OutputSymbolFile))
{
BuildProducts.Add(OutputSymbolFile);
}
}
else
{
// Add reference directly
References.Add(OtherAssembly);
FileReference SymbolFile = OtherAssembly.ChangeExtension(".pdb");
if (FileReference.Exists(SymbolFile))
{
References.Add(SymbolFile);
}
}
}
// Add build products from all the referenced projects. MSBuild only copy the directly referenced build products, not recursive references or other assemblies.
foreach (CsProjectInfo OtherProjectInfo in ProjectInfo.ProjectReferences.Where(x => x.Value).Select(x => FileToProjectInfo[x.Key]))
{
OtherProjectInfo.AddBuildProducts(OutputDir, BuildProducts);
}
}
// Update the output set
OutBuildProducts = BuildProducts;
OutReferences = References;
}
/// <summary>
/// Main entry point for the BuildGraph command
/// </summary>
public override ExitCode Execute()
{
// Parse the command line parameters
string ScriptFileName = ParseParamValue("Script", null);
string TargetNames = ParseParamValue("Target", null);
string DocumentationFileName = ParseParamValue("Documentation", null);
string SchemaFileName = ParseParamValue("Schema", null);
string ExportFileName = ParseParamValue("Export", null);
string PreprocessedFileName = ParseParamValue("Preprocess", null);
string SharedStorageDir = ParseParamValue("SharedStorageDir", null);
string SingleNodeName = ParseParamValue("SingleNode", null);
string TriggerName = ParseParamValue("Trigger", null);
string[] SkipTriggerNames = ParseParamValue("SkipTrigger", "").Split(new char[] { '+', ';' }, StringSplitOptions.RemoveEmptyEntries).ToArray();
bool bSkipTriggers = ParseParam("SkipTriggers");
string TokenSignature = ParseParamValue("TokenSignature", null);
bool bSkipTargetsWithoutTokens = ParseParam("SkipTargetsWithoutTokens");
bool bResume = SingleNodeName != null || ParseParam("Resume");
bool bListOnly = ParseParam("ListOnly");
bool bWriteToSharedStorage = ParseParam("WriteToSharedStorage") || CommandUtils.IsBuildMachine;
bool bPublicTasksOnly = ParseParam("PublicTasksOnly");
string ReportName = ParseParamValue("ReportName", null);
GraphPrintOptions PrintOptions = GraphPrintOptions.ShowCommandLineOptions;
if (ParseParam("ShowDeps"))
{
PrintOptions |= GraphPrintOptions.ShowDependencies;
}
if (ParseParam("ShowNotifications"))
{
PrintOptions |= GraphPrintOptions.ShowNotifications;
}
// Parse any specific nodes to clean
List <string> CleanNodes = new List <string>();
foreach (string NodeList in ParseParamValues("CleanNode"))
{
foreach (string NodeName in NodeList.Split('+', ';'))
{
CleanNodes.Add(NodeName);
}
}
// Set up the standard properties which build scripts might need
Dictionary <string, string> DefaultProperties = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase);
DefaultProperties["Branch"] = P4Enabled ? P4Env.BuildRootP4 : "Unknown";
DefaultProperties["EscapedBranch"] = P4Enabled ? P4Env.BuildRootEscaped : "Unknown";
DefaultProperties["Change"] = P4Enabled ? P4Env.Changelist.ToString() : "0";
DefaultProperties["CodeChange"] = P4Enabled ? P4Env.CodeChangelist.ToString() : "0";
DefaultProperties["RootDir"] = CommandUtils.RootDirectory.FullName;
DefaultProperties["IsBuildMachine"] = IsBuildMachine ? "true" : "false";
DefaultProperties["HostPlatform"] = HostPlatform.Current.HostEditorPlatform.ToString();
DefaultProperties["RestrictedFolderNames"] = String.Join(";", PlatformExports.RestrictedFolderNames);
DefaultProperties["RestrictedFolderFilter"] = String.Join(";", PlatformExports.RestrictedFolderNames.Select(x => String.Format(".../{0}/...", x)));
// Attempt to read existing Build Version information
BuildVersion Version;
if (BuildVersion.TryRead(FileReference.Combine(CommandUtils.RootDirectory, "Engine", "Build", "Build.version").FullName, out Version))
{
DefaultProperties["EngineMajorVersion"] = Version.MajorVersion.ToString();
DefaultProperties["EngineMinorVersion"] = Version.MinorVersion.ToString();
DefaultProperties["EnginePatchVersion"] = Version.PatchVersion.ToString();
}
// Add any additional custom arguments from the command line (of the form -Set:X=Y)
Dictionary <string, string> Arguments = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase);
foreach (string Param in Params)
{
const string Prefix = "set:";
if (Param.StartsWith(Prefix, StringComparison.InvariantCultureIgnoreCase))
{
int EqualsIdx = Param.IndexOf('=');
if (EqualsIdx >= 0)
{
Arguments[Param.Substring(Prefix.Length, EqualsIdx - Prefix.Length)] = Param.Substring(EqualsIdx + 1);
}
else
{
LogWarning("Missing value for '{0}'", Param.Substring(Prefix.Length));
}
}
}
// Find all the tasks from the loaded assemblies
Dictionary <string, ScriptTask> NameToTask = new Dictionary <string, ScriptTask>();
if (!FindAvailableTasks(NameToTask, bPublicTasksOnly))
{
return(ExitCode.Error_Unknown);
}
// Generate documentation
if (DocumentationFileName != null)
{
GenerateDocumentation(NameToTask, new FileReference(DocumentationFileName));
return(ExitCode.Success);
}
// Create a schema for the given tasks
ScriptSchema Schema = new ScriptSchema(NameToTask);
if (SchemaFileName != null)
{
FileReference FullSchemaFileName = new FileReference(SchemaFileName);
Log("Writing schema to {0}...", FullSchemaFileName.FullName);
Schema.Export(FullSchemaFileName);
if (ScriptFileName == null)
{
return(ExitCode.Success);
}
}
// Check there was a script specified
if (ScriptFileName == null)
{
LogError("Missing -Script= parameter for BuildGraph");
return(ExitCode.Error_Unknown);
}
// Read the script from disk
Graph Graph;
if (!ScriptReader.TryRead(new FileReference(ScriptFileName), Arguments, DefaultProperties, Schema, out Graph))
{
return(ExitCode.Error_Unknown);
}
// Create the temp storage handler
DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);
TempStorage Storage = new TempStorage(RootDir, DirectoryReference.Combine(RootDir, "Engine", "Saved", "BuildGraph"), (SharedStorageDir == null)? null : new DirectoryReference(SharedStorageDir), bWriteToSharedStorage);
if (!bResume)
{
Storage.CleanLocal();
}
foreach (string CleanNode in CleanNodes)
{
Storage.CleanLocalNode(CleanNode);
}
// Convert the supplied target references into nodes
HashSet <Node> TargetNodes = new HashSet <Node>();
if (TargetNames == null)
{
if (!bListOnly)
{
LogError("Missing -Target= parameter for BuildGraph");
return(ExitCode.Error_Unknown);
}
TargetNodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes));
}
else
{
foreach (string TargetName in TargetNames.Split(new char[] { '+', ';' }, StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim()))
{
Node[] Nodes;
if (!Graph.TryResolveReference(TargetName, out Nodes))
{
LogError("Target '{0}' is not in graph", TargetName);
return(ExitCode.Error_Unknown);
}
TargetNodes.UnionWith(Nodes);
}
}
// Try to acquire tokens for all the target nodes we want to build
if (TokenSignature != null)
{
// Find all the lock files
HashSet <FileReference> RequiredTokens = new HashSet <FileReference>(TargetNodes.SelectMany(x => x.RequiredTokens));
// List out all the required tokens
if (SingleNodeName == null)
{
CommandUtils.Log("Required tokens:");
foreach (Node Node in TargetNodes)
{
foreach (FileReference RequiredToken in Node.RequiredTokens)
{
CommandUtils.Log(" '{0}' requires {1}", Node, RequiredToken);
}
}
}
// Try to create all the lock files
List <FileReference> CreatedTokens = new List <FileReference>();
if (!bListOnly)
{
CreatedTokens.AddRange(RequiredTokens.Where(x => WriteTokenFile(x, TokenSignature)));
}
// Find all the tokens that we don't have
Dictionary <FileReference, string> MissingTokens = new Dictionary <FileReference, string>();
foreach (FileReference RequiredToken in RequiredTokens)
{
string CurrentOwner = ReadTokenFile(RequiredToken);
if (CurrentOwner != null && CurrentOwner != TokenSignature)
{
MissingTokens.Add(RequiredToken, CurrentOwner);
}
}
// If we want to skip all the nodes with missing locks, adjust the target nodes to account for it
if (MissingTokens.Count > 0)
{
if (bSkipTargetsWithoutTokens)
{
// Find all the nodes we're going to skip
HashSet <Node> SkipNodes = new HashSet <Node>();
foreach (IGrouping <string, FileReference> MissingTokensForBuild in MissingTokens.GroupBy(x => x.Value, x => x.Key))
{
Log("Skipping the following nodes due to {0}:", MissingTokensForBuild.Key);
foreach (FileReference MissingToken in MissingTokensForBuild)
{
foreach (Node SkipNode in TargetNodes.Where(x => x.RequiredTokens.Contains(MissingToken) && SkipNodes.Add(x)))
{
Log(" {0}", SkipNode);
}
}
}
// Write a list of everything left over
if (SkipNodes.Count > 0)
{
TargetNodes.ExceptWith(SkipNodes);
Log("Remaining target nodes:");
foreach (Node TargetNode in TargetNodes)
{
Log(" {0}", TargetNode);
}
if (TargetNodes.Count == 0)
{
Log(" None.");
}
}
}
else
{
foreach (KeyValuePair <FileReference, string> Pair in MissingTokens)
{
List <Node> SkipNodes = TargetNodes.Where(x => x.RequiredTokens.Contains(Pair.Key)).ToList();
LogError("Cannot run {0} due to previous build: {1}", String.Join(", ", SkipNodes), Pair.Value);
}
foreach (FileReference CreatedToken in CreatedTokens)
{
FileReference.Delete(CreatedToken);
}
return(ExitCode.Error_Unknown);
}
}
}
// Cull the graph to include only those nodes
Graph.Select(TargetNodes);
// Collapse any triggers in the graph which are marked to be skipped
HashSet <ManualTrigger> SkipTriggers = new HashSet <ManualTrigger>();
if (bSkipTriggers)
{
SkipTriggers.UnionWith(Graph.NameToTrigger.Values);
}
else
{
foreach (string SkipTriggerName in SkipTriggerNames)
{
ManualTrigger SkipTrigger;
if (!Graph.NameToTrigger.TryGetValue(TriggerName, out SkipTrigger))
{
LogError("Couldn't find trigger '{0}'", TriggerName);
return(ExitCode.Error_Unknown);
}
SkipTriggers.Add(SkipTrigger);
}
}
Graph.SkipTriggers(SkipTriggers);
// If a report for the whole build was requested, insert it into the graph
if (ReportName != null)
{
Report NewReport = new Report(ReportName);
NewReport.Nodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes));
Graph.NameToReport.Add(ReportName, NewReport);
}
// Write out the preprocessed script
if (PreprocessedFileName != null)
{
Graph.Write(new FileReference(PreprocessedFileName), (SchemaFileName != null)? new FileReference(SchemaFileName) : null);
}
// Find the triggers which we are explicitly running.
ManualTrigger Trigger = null;
if (TriggerName != null && !Graph.NameToTrigger.TryGetValue(TriggerName, out Trigger))
{
LogError("Couldn't find trigger '{0}'", TriggerName);
return(ExitCode.Error_Unknown);
}
// If we're just building a single node, find it
Node SingleNode = null;
if (SingleNodeName != null && !Graph.NameToNode.TryGetValue(SingleNodeName, out SingleNode))
{
LogError("Node '{0}' is not in the trimmed graph", SingleNodeName);
return(ExitCode.Error_Unknown);
}
// If we just want to show the contents of the graph, do so and exit.
if (bListOnly)
{
HashSet <Node> CompletedNodes = FindCompletedNodes(Graph, Storage);
Graph.Print(CompletedNodes, PrintOptions);
return(ExitCode.Success);
}
// Print out all the diagnostic messages which still apply, unless we're running a step as part of a build system or just listing the contents of the file.
if (SingleNode == null)
{
IEnumerable <GraphDiagnostic> Diagnostics = Graph.Diagnostics.Where(x => x.EnclosingTrigger == Trigger);
foreach (GraphDiagnostic Diagnostic in Diagnostics)
{
if (Diagnostic.EventType == LogEventType.Warning)
{
CommandUtils.LogWarning(Diagnostic.Message);
}
else
{
CommandUtils.LogError(Diagnostic.Message);
}
}
if (Diagnostics.Any(x => x.EventType == LogEventType.Error))
{
return(ExitCode.Error_Unknown);
}
}
// Execute the command
if (ExportFileName != null)
{
HashSet <Node> CompletedNodes = FindCompletedNodes(Graph, Storage);
Graph.Print(CompletedNodes, PrintOptions);
Graph.Export(new FileReference(ExportFileName), Trigger, CompletedNodes);
}
else if (SingleNode != null)
{
if (!BuildNode(new JobContext(this), Graph, SingleNode, Storage, bWithBanner: true))
{
return(ExitCode.Error_Unknown);
}
}
else
{
if (!BuildAllNodes(new JobContext(this), Graph, Storage))
{
return(ExitCode.Error_Unknown);
}
}
return(ExitCode.Success);
}
/// <summary>
/// Tries to resolve an include path from a file, given a set of candidates
/// </summary>
/// <param name="FromFile">The file containing the #include directive</param>
/// <param name="IncludePath">The path being included</param>
/// <param name="InitialCandidateFiles">Set of candidate files to pick from</param>
/// <param name="bIsSystemInclude">Whether the include path is for a system header. If these aren't found, we don't fail.</param>
/// <param name="IncludedFile">On success, the matching file</param>
/// <param name="Log">Log instance for output messages</param>
/// <returns>True if the include path was resolved, false otherwise</returns>
public static bool ResolveInclude(WorkspaceFile FromFile, string IncludePath, IEnumerable <WorkspaceFile> InitialCandidateFiles, bool bIsSystemInclude, out WorkspaceFile IncludedFile, TextWriter Log)
{
List <WorkspaceFile> CandidateFiles = new List <WorkspaceFile>(InitialCandidateFiles);
// Check if it's a direct file reference
WorkspaceFile DirectlyReferencedFile = CandidateFiles.FirstOrDefault(x => x.Location == FileReference.Combine(FromFile.Location.Directory, IncludePath));
if (DirectlyReferencedFile != null)
{
IncludedFile = DirectlyReferencedFile;
return(true);
}
// Apply any of the ignore patterns
for (int Idx = 0; Idx < IgnoreIncludePatterns.GetLength(0); Idx++)
{
if (FromFile.NormalizedPathFromBranchRoot.StartsWith(IgnoreIncludePatterns[Idx, 0]))
{
CandidateFiles.RemoveAll(x => x.NormalizedPathFromBranchRoot.Contains(IgnoreIncludePatterns[Idx, 1]));
}
}
// Remove any candidate files that don't have the same suffix as the include path
string IncludeSuffix = IncludePath.ToLowerInvariant().Replace('\\', '/');
while (IncludeSuffix.StartsWith("../"))
{
IncludeSuffix = IncludeSuffix.Substring(3);
}
CandidateFiles.RemoveAll(x => !x.NormalizedPathFromBranchRoot.EndsWith(IncludeSuffix));
// Remove any files from a different project. We do occasionally cross this boundary via explicitly referenced paths, so skip if there's already only one match.
if (CandidateFiles.Count > 1)
{
if (FromFile.ProjectDirectory == null)
{
CandidateFiles.RemoveAll(x => x.ProjectDirectory != null);
}
else
{
CandidateFiles.RemoveAll(x => x.ProjectDirectory != FromFile.ProjectDirectory && x.ProjectDirectory != null);
}
}
// If we're including Engine.h and haven't explicitly referenced the UEngine header, assume it's referencing the header for the engine module
if (CandidateFiles.Count > 2 && IncludeSuffix == "engine.h")
{
if (CandidateFiles.Any(x => x.NormalizedPathFromBranchRoot == "/engine/source/runtime/engine/public/engine.h"))
{
CandidateFiles.RemoveAll(x => x.NormalizedPathFromBranchRoot == "/engine/source/runtime/engine/classes/engine/engine.h");
}
}
// Use case to disambiguate if we still have mismatches. This actually weeds out quite a lot of false positives, expecially with TPS
// (the typical pattern is that Epic code uses TitleCase but OSS uses all lowercase).
if (CandidateFiles.Count > 1)
{
List <WorkspaceFile> ExactCaseMatches = CandidateFiles.Where(x => x.Location.FullName.EndsWith(IncludePath.Replace('/', Path.DirectorySeparatorChar))).ToList();
if (ExactCaseMatches.Count == 1)
{
CandidateFiles.Clear();
CandidateFiles.AddRange(ExactCaseMatches);
}
}
// Use case to disambiguate if we still have mismatches. This actually weeds out quite a lot of false positives, expecially with TPS
// (the typical pattern is that Epic code uses TitleCase but OSS uses all lowercase).
if (CandidateFiles.Count > 1)
{
List <WorkspaceFile> NonThirdPartyCandidates = CandidateFiles.Where(x => !x.NormalizedPathFromBranchRoot.Contains("/thirdparty/")).ToList();
if (NonThirdPartyCandidates.Count == 1)
{
CandidateFiles.Clear();
CandidateFiles.AddRange(NonThirdPartyCandidates);
}
}
// Check if we've got an exact match
if (CandidateFiles.Count == 1)
{
IncludedFile = CandidateFiles[0];
return(true);
}
// If it's a system header, it's ok if we didn't resolve it
if (CandidateFiles.Count == 0 && bIsSystemInclude)
{
IncludedFile = null;
return(true);
}
// If they're all third party files, just treat it as an external include
if (CandidateFiles.Count > 1 && CandidateFiles.All(x => x.NormalizedPathFromBranchRoot.Contains("/thirdparty/")))
{
IncludedFile = null;
return(true);
}
// Allow generated headers to be excluded for now
if (IncludePath.EndsWith("Classes.h") || IncludePath.EndsWith(".generated.h"))
{
IncludedFile = null;
return(true);
}
// Otherwise print the remaining candidates
Log.WriteLine("Failed to resolve include of \"{0}\" from \"{1}\". {2} candidate files.", IncludePath, FromFile.Location, CandidateFiles.Count);
foreach (WorkspaceFile FinalCandidateFile in CandidateFiles)
{
Log.WriteLine(" Could be: {0}", FinalCandidateFile.Location);
}
Log.WriteLine();
// ...and fail
IncludedFile = null;
return(false);
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
public override void Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Find the directories we're going to rebase relative to
HashSet <DirectoryReference> RebaseDirs = new HashSet <DirectoryReference> {
CommandUtils.RootDirectory
};
if (Parameters.RebaseDir != null)
{
RebaseDirs.UnionWith(Parameters.RebaseDir);
}
// Get the output parameter
FileReference OutputFile = Parameters.Output;
// Check for a ResponseFile parameter
FileReference ResponseFile = Parameters.ResponseFile;
if (ResponseFile == null)
{
// Get a unique filename for the response file
ResponseFile = FileReference.Combine(new DirectoryReference(CommandUtils.CmdEnv.LogFolder), String.Format("PakList_{0}.txt", OutputFile.GetFileNameWithoutExtension()));
for (int Idx = 2; FileReference.Exists(ResponseFile); Idx++)
{
ResponseFile = FileReference.Combine(ResponseFile.Directory, String.Format("PakList_{0}_{1}.txt", OutputFile.GetFileNameWithoutExtension(), Idx));
}
// Write out the response file
HashSet <FileReference> Files = ResolveFilespec(CommandUtils.RootDirectory, Parameters.Files, TagNameToFileSet);
using (StreamWriter Writer = new StreamWriter(ResponseFile.FullName, false, new System.Text.UTF8Encoding(true)))
{
foreach (FileReference File in Files)
{
string RelativePath = FindShortestRelativePath(File, RebaseDirs);
if (RelativePath == null)
{
throw new AutomationException("Couldn't find relative path for '{0}' - not under any rebase directories", File.FullName);
}
Writer.WriteLine("\"{0}\" \"{1}\"{2}", File.FullName, RelativePath, Parameters.Compress ? " -compress" : "");
}
}
}
// Format the command line
StringBuilder CommandLine = new StringBuilder();
CommandLine.AppendFormat("{0} -create={1}", CommandUtils.MakePathSafeToUseWithCommandLine(OutputFile.FullName), CommandUtils.MakePathSafeToUseWithCommandLine(ResponseFile.FullName));
if (Parameters.Sign != null)
{
CommandLine.AppendFormat(" -sign={0}", CommandUtils.MakePathSafeToUseWithCommandLine(Parameters.Sign.FullName));
}
if (Parameters.Order != null)
{
CommandLine.AppendFormat(" -order={0}", CommandUtils.MakePathSafeToUseWithCommandLine(Parameters.Order.FullName));
}
if (CommandUtils.IsEngineInstalled())
{
CommandLine.Append(" -installed");
}
if (GlobalCommandLine.UTF8Output)
{
CommandLine.AppendFormat(" -UTF8Output");
}
// Get the executable path
FileReference UnrealPakExe;
if (HostPlatform.Current.HostEditorPlatform == UnrealTargetPlatform.Win64)
{
UnrealPakExe = ResolveFile("Engine/Binaries/Win64/UnrealPak.exe");
}
else
{
UnrealPakExe = ResolveFile(String.Format("Engine/Binaries/{0}/UnrealPak", HostPlatform.Current.HostEditorPlatform.ToString()));
}
// Run it
CommandUtils.LogInformation("Running '{0} {1}'", CommandUtils.MakePathSafeToUseWithCommandLine(UnrealPakExe.FullName), CommandLine.ToString());
CommandUtils.RunAndLog(CommandUtils.CmdEnv, UnrealPakExe.FullName, CommandLine.ToString(), Options: CommandUtils.ERunOptions.Default | CommandUtils.ERunOptions.UTF8Output);
BuildProducts.Add(OutputFile);
// Apply the optional tag to the output file
foreach (string TagName in FindTagNamesFromList(Parameters.Tag))
{
FindOrAddTagSet(TagNameToFileSet, TagName).Add(OutputFile);
}
}
public override void ExecuteBuild()
{
// Get the plugin filename
string PluginParam = ParseParamValue("Plugin");
if (PluginParam == null)
{
throw new AutomationException("Missing -Plugin=... argument");
}
// Check it exists
FileReference PluginFile = new FileReference(PluginParam);
if (!FileReference.Exists(PluginFile))
{
throw new AutomationException("Plugin '{0}' not found", PluginFile.FullName);
}
// Get the output directory
string PackageParam = ParseParamValue("Package");
if (PackageParam == null)
{
throw new AutomationException("Missing -Package=... argument");
}
// Make sure the packaging directory is valid
DirectoryReference PackageDir = new DirectoryReference(PackageParam);
if (PluginFile.IsUnderDirectory(PackageDir))
{
throw new AutomationException("Packaged plugin output directory must be different to source");
}
if (PackageDir.IsUnderDirectory(DirectoryReference.Combine(CommandUtils.RootDirectory, "Engine")))
{
throw new AutomationException("Output directory for packaged plugin must be outside engine directory");
}
// Clear the output directory of existing stuff
if (DirectoryReference.Exists(PackageDir))
{
CommandUtils.DeleteDirectoryContents(PackageDir.FullName);
}
else
{
DirectoryReference.CreateDirectory(PackageDir);
}
// Create a placeholder FilterPlugin.ini with instructions on how to use it
FileReference SourceFilterFile = FileReference.Combine(PluginFile.Directory, "Config", "FilterPlugin.ini");
if (!FileReference.Exists(SourceFilterFile))
{
List <string> Lines = new List <string>();
Lines.Add("[FilterPlugin]");
Lines.Add("; This section lists additional files which will be packaged along with your plugin. Paths should be listed relative to the root plugin directory, and");
Lines.Add("; may include \"...\", \"*\", and \"?\" wildcards to match directories, files, and individual characters respectively.");
Lines.Add(";");
Lines.Add("; Examples:");
Lines.Add("; /README.txt");
Lines.Add("; /Extras/...");
Lines.Add("; /Binaries/ThirdParty/*.dll");
DirectoryReference.CreateDirectory(SourceFilterFile.Directory);
CommandUtils.WriteAllLines_NoExceptions(SourceFilterFile.FullName, Lines.ToArray());
}
// Create a host project for the plugin. For script generator plugins, we need to have UHT be able to load it, which can only happen if it's enabled in a project.
FileReference HostProjectFile = FileReference.Combine(PackageDir, "HostProject", "HostProject.uproject");
FileReference HostProjectPluginFile = CreateHostProject(HostProjectFile, PluginFile);
// Read the plugin
CommandUtils.Log("Reading plugin from {0}...", HostProjectPluginFile);
PluginDescriptor Plugin = PluginDescriptor.FromFile(HostProjectPluginFile, false);
// Compile the plugin for all the target platforms
List <UnrealTargetPlatform> HostPlatforms = ParseParam("NoHostPlatform")? new List <UnrealTargetPlatform>() : new List <UnrealTargetPlatform> {
BuildHostPlatform.Current.Platform
};
List <UnrealTargetPlatform> TargetPlatforms = GetTargetPlatforms(this, BuildHostPlatform.Current.Platform);
FileReference[] BuildProducts = CompilePlugin(HostProjectFile, HostProjectPluginFile, Plugin, HostPlatforms, TargetPlatforms, "");
// Package up the final plugin data
PackagePlugin(HostProjectPluginFile, BuildProducts, PackageDir);
// Remove the host project
if (!ParseParam("NoDeleteHostProject"))
{
CommandUtils.DeleteDirectory(HostProjectFile.Directory.FullName);
}
}
public override void GetFilesToDeployOrStage(ProjectParams Params, DeploymentContext SC)
{
// Engine non-ufs (binaries)
if (SC.bStageCrashReporter)
{
FileReference ReceiptFileName = TargetReceipt.GetDefaultPath(CommandUtils.EngineDirectory, "CrashReportClient", SC.StageTargetPlatform.PlatformType, UnrealTargetConfiguration.Shipping, null);
if (FileReference.Exists(ReceiptFileName))
{
TargetReceipt Receipt = TargetReceipt.Read(ReceiptFileName);
SC.StageBuildProductsFromReceipt(Receipt, true, false);
}
}
// Stage all the build products
foreach (StageTarget Target in SC.StageTargets)
{
SC.StageBuildProductsFromReceipt(Target.Receipt, Target.RequireFilesExist, Params.bTreatNonShippingBinariesAsDebugFiles);
}
// Copy the splash screen, windows specific
FileReference SplashImage = FileReference.Combine(SC.ProjectRoot, "Content", "Splash", "Splash.bmp");
if (FileReference.Exists(SplashImage))
{
SC.StageFile(StagedFileType.NonUFS, SplashImage);
}
// Stage the bootstrap executable
if (!Params.NoBootstrapExe)
{
foreach (StageTarget Target in SC.StageTargets)
{
BuildProduct Executable = Target.Receipt.BuildProducts.FirstOrDefault(x => x.Type == BuildProductType.Executable);
if (Executable != null)
{
// only create bootstraps for executables
List <StagedFileReference> StagedFiles = SC.FilesToStage.NonUFSFiles.Where(x => x.Value == Executable.Path).Select(x => x.Key).ToList();
if (StagedFiles.Count > 0 && Executable.Path.HasExtension(".exe"))
{
string BootstrapArguments = "";
if (!ShouldStageCommandLine(Params, SC))
{
if (!SC.IsCodeBasedProject)
{
BootstrapArguments = String.Format("..\\..\\..\\{0}\\{0}.uproject", SC.ShortProjectName);
}
else
{
BootstrapArguments = SC.ShortProjectName;
}
}
string BootstrapExeName;
if (SC.StageTargetConfigurations.Count > 1)
{
BootstrapExeName = Executable.Path.GetFileName();
}
else if (Params.IsCodeBasedProject)
{
BootstrapExeName = Target.Receipt.TargetName + ".exe";
}
else
{
BootstrapExeName = SC.ShortProjectName + ".exe";
}
foreach (StagedFileReference StagePath in StagedFiles)
{
StageBootstrapExecutable(SC, BootstrapExeName, Executable.Path, StagePath, BootstrapArguments);
}
}
}
}
}
}
public override void ExecuteBuild()
{
CommandUtils.LogInformation("************************* List Third Party Software");
string ProjectPath = ParseParamValue("Project", String.Empty);
//Add quotes to avoid issues with spaces in project path
if (ProjectPath != String.Empty)
{
ProjectPath = "\"" + ProjectPath + "\"";
}
// Parse the list of targets to list TPS for. Each target is specified by -Target="Name|Configuration|Platform" on the command line.
HashSet <FileReference> TpsFiles = new HashSet <FileReference>();
foreach (string Target in ParseParamValues(Params, "Target"))
{
// Get the path to store the exported JSON target data
FileReference OutputFile = FileReference.Combine(CommandUtils.EngineDirectory, "Intermediate", "Build", "ThirdParty.json");
IProcessResult Result;
Result = Run(UE4Build.GetUBTExecutable(), String.Format("{0} {1} -jsonexport=\"{2}\" -skipbuild", Target.Replace('|', ' '), ProjectPath, OutputFile.FullName), Options: ERunOptions.Default);
if (Result.ExitCode != 0)
{
throw new AutomationException("Failed to run UBT");
}
// Read the exported target info back in
JsonObject Object = JsonObject.Read(OutputFile);
// Get the project file if there is one
FileReference ProjectFile = null;
string ProjectFileName;
if (Object.TryGetStringField("ProjectFile", out ProjectFileName))
{
ProjectFile = new FileReference(ProjectFileName);
}
// Get the default paths to search
HashSet <DirectoryReference> DirectoriesToScan = new HashSet <DirectoryReference>();
DirectoriesToScan.Add(DirectoryReference.Combine(CommandUtils.EngineDirectory, "Shaders"));
DirectoriesToScan.Add(DirectoryReference.Combine(CommandUtils.EngineDirectory, "Content"));
if (ProjectFile != null)
{
DirectoriesToScan.Add(DirectoryReference.Combine(ProjectFile.Directory, "Content"));
}
// Get the variables to be expanded in any runtime dependencies variables
Dictionary <string, string> Variables = new Dictionary <string, string>();
Variables.Add("EngineDir", CommandUtils.EngineDirectory.FullName);
if (ProjectFile != null)
{
Variables.Add("ProjectDir", ProjectFile.Directory.FullName);
}
// Add all the paths for each module, and its runtime dependencies
JsonObject Modules = Object.GetObjectField("Modules");
foreach (string ModuleName in Modules.KeyNames)
{
JsonObject Module = Modules.GetObjectField(ModuleName);
DirectoriesToScan.Add(new DirectoryReference(Module.GetStringField("Directory")));
foreach (JsonObject RuntimeDependency in Module.GetObjectArrayField("RuntimeDependencies"))
{
string RuntimeDependencyPath = RuntimeDependency.GetStringField("Path");
RuntimeDependencyPath = Utils.ExpandVariables(RuntimeDependencyPath, Variables);
DirectoriesToScan.Add(new FileReference(RuntimeDependencyPath).Directory);
}
}
// Remove any directories that are under other directories, and sort the output list
List <DirectoryReference> SortedDirectoriesToScan = new List <DirectoryReference>();
foreach (DirectoryReference DirectoryToScan in DirectoriesToScan.OrderBy(x => x.FullName))
{
if (SortedDirectoriesToScan.Count == 0 || !DirectoryToScan.IsUnderDirectory(SortedDirectoriesToScan[SortedDirectoriesToScan.Count - 1]))
{
SortedDirectoriesToScan.Add(DirectoryToScan);
}
}
// Get the platforms to exclude
List <UnrealTargetPlatform> SupportedPlatforms = new List <UnrealTargetPlatform> {
(UnrealTargetPlatform)Enum.Parse(typeof(UnrealTargetPlatform), Object.GetStringField("Platform"))
};
string[] ExcludePlatformNames = Utils.MakeListOfUnsupportedPlatforms(SupportedPlatforms).ToArray();
// Find all the TPS files under the engine directory which match
foreach (DirectoryReference DirectoryToScan in SortedDirectoriesToScan)
{
foreach (FileReference TpsFile in DirectoryReference.EnumerateFiles(DirectoryToScan, "*.tps", SearchOption.AllDirectories))
{
if (!TpsFile.ContainsAnyNames(ExcludePlatformNames, DirectoryToScan))
{
TpsFiles.Add(TpsFile);
}
}
}
}
// Also add any redirects
List <string> OutputMessages = new List <string>();
foreach (FileReference TpsFile in TpsFiles)
{
string Message = TpsFile.FullName;
string[] Lines = FileReference.ReadAllLines(TpsFile);
foreach (string Line in Lines)
{
const string RedirectPrefix = "Redirect:";
int Idx = Line.IndexOf(RedirectPrefix, StringComparison.InvariantCultureIgnoreCase);
if (Idx >= 0)
{
FileReference RedirectTpsFile = FileReference.Combine(TpsFile.Directory, Line.Substring(Idx + RedirectPrefix.Length).Trim());
Message = String.Format("{0} (redirect from {1})", RedirectTpsFile.FullName, TpsFile.FullName);
break;
}
}
OutputMessages.Add(Message);
}
OutputMessages.Sort();
// Print them all out
foreach (string OutputMessage in OutputMessages)
{
CommandUtils.LogInformation(OutputMessage);
}
}
public override bool PublishSymbols(DirectoryReference SymbolStoreDirectory, List <FileReference> Files, string Product, string BuildVersion = null)
{
// Get the SYMSTORE.EXE path, using the latest SDK version we can find.
FileReference SymStoreExe = GetSymStoreExe();
List <FileReference> FilesToAdd = Files.Where(x => x.HasExtension(".pdb") || x.HasExtension(".exe") || x.HasExtension(".dll")).ToList();
if (FilesToAdd.Count > 0)
{
DateTime Start = DateTime.Now;
DirectoryReference TempSymStoreDir = DirectoryReference.Combine(RootDirectory, "Saved", "SymStore");
if (DirectoryReference.Exists(TempSymStoreDir))
{
CommandUtils.DeleteDirectory(TempSymStoreDir);
DirectoryReference.CreateDirectory(TempSymStoreDir);
}
string TempFileName = Path.GetTempFileName();
try
{
File.WriteAllLines(TempFileName, FilesToAdd.Select(x => x.FullName), Encoding.ASCII);
// Copy everything to the temp symstore
ProcessStartInfo StartInfo = new ProcessStartInfo();
StartInfo.FileName = SymStoreExe.FullName;
StartInfo.Arguments = string.Format("add /f \"@{0}\" /s \"{1}\" /t \"{2}\"", TempFileName, TempSymStoreDir, Product);
StartInfo.UseShellExecute = false;
StartInfo.CreateNoWindow = true;
if (Utils.RunLocalProcessAndLogOutput(StartInfo) != 0)
{
return(false);
}
}
finally
{
File.Delete(TempFileName);
}
DateTime CompressDone = DateTime.Now;
LogInformation("Took {0}s to compress the symbol files to temp path {1}", (CompressDone - Start).TotalSeconds, TempSymStoreDir);
int CopiedCount = 0;
// Take each new compressed file made and try and copy it to the real symstore. Exclude any symstore admin files
foreach (FileReference File in DirectoryReference.EnumerateFiles(TempSymStoreDir, "*.*", SearchOption.AllDirectories).Where(File => IsSymbolFile(File)))
{
string RelativePath = File.MakeRelativeTo(DirectoryReference.Combine(TempSymStoreDir));
FileReference ActualDestinationFile = FileReference.Combine(SymbolStoreDirectory, RelativePath);
// Try and add a version file. Do this before checking to see if the symbol is there already in the case of exact matches (multiple builds could use the same pdb, for example)
if (!string.IsNullOrWhiteSpace(BuildVersion))
{
FileReference BuildVersionFile = FileReference.Combine(ActualDestinationFile.Directory, string.Format("{0}.version", BuildVersion));
// Attempt to create the file. Just continue if it fails.
try
{
DirectoryReference.CreateDirectory(BuildVersionFile.Directory);
FileReference.WriteAllText(BuildVersionFile, string.Empty);
}
catch (Exception Ex)
{
LogWarning("Failed to write the version file, reason {0}", Ex.ToString());
}
}
// Don't bother copying the temp file if the destination file is there already.
if (FileReference.Exists(ActualDestinationFile))
{
LogInformation("Destination file {0} already exists, skipping", ActualDestinationFile.FullName);
continue;
}
FileReference TempDestinationFile = new FileReference(ActualDestinationFile.FullName + Guid.NewGuid().ToString());
try
{
CommandUtils.CopyFile(File.FullName, TempDestinationFile.FullName);
}
catch (Exception Ex)
{
throw new AutomationException("Couldn't copy the symbol file to the temp store! Reason: {0}", Ex.ToString());
}
// Move the file in the temp store over.
try
{
FileReference.Move(TempDestinationFile, ActualDestinationFile);
//LogVerbose("Moved {0} to {1}", TempDestinationFile, ActualDestinationFile);
CopiedCount++;
}
catch (Exception Ex)
{
// If the file is there already, it was likely either copied elsewhere (and this is an ioexception) or it had a file handle open already.
// Either way, it's fine to just continue on.
if (FileReference.Exists(ActualDestinationFile))
{
LogInformation("Destination file {0} already exists or was in use, skipping.", ActualDestinationFile.FullName);
continue;
}
// If it doesn't exist, we actually failed to copy it entirely.
else
{
LogWarning("Couldn't move temp file {0} to the symbol store at location {1}! Reason: {2}", TempDestinationFile.FullName, ActualDestinationFile.FullName, Ex.ToString());
}
}
// Delete the temp one no matter what, don't want them hanging around in the symstore
finally
{
FileReference.Delete(TempDestinationFile);
}
}
LogInformation("Took {0}s to copy {1} symbol files to the store at {2}", (DateTime.Now - CompressDone).TotalSeconds, CopiedCount, SymbolStoreDirectory);
FileReference PingmeFile = FileReference.Combine(SymbolStoreDirectory, "pingme.txt");
if (!FileReference.Exists(PingmeFile))
{
LogInformation("Creating {0} to mark path as three-tiered symbol location", PingmeFile);
File.WriteAllText(PingmeFile.FullName, "Exists to mark this as a three-tiered symbol location");
}
}
return(true);
}
private void RunCsvTool(string ToolName, string Arguments)
{
var ToolPath = FileReference.Combine(DirectoryReference.Combine(RootDir, "Engine", "Programs", "NotForLicensees", "CSVTools", "Binaries"), ToolName + ".exe");
Run(ToolPath.FullName, Arguments);
}
/// <summary>
/// Creates a file mapping between a set of source patterns and a target pattern. All patterns should have a matching order and number of wildcards.
/// </summary>
/// <param name="Files">Files to use for the mapping</param>
/// <param name="SourcePatterns">List of source patterns</param>
/// <param name="TargetPattern">Matching output pattern</param>
/// <param name="Filter">Filter to apply to source files</param>
/// <param name="TargetFileToSourceFile">Dictionary to receive a mapping from target file to source file. An exception is thrown if multiple source files map to one target file, or a source file is also used as a target file.</param>
public static Dictionary <FileReference, FileReference> CreateMapping(HashSet <FileReference> Files, FilePattern SourcePattern, FilePattern TargetPattern)
{
// If the source pattern ends in a directory separator, or a set of input files are specified and it doesn't contain wildcards, treat it as a full directory match
if (SourcePattern.EndsWithDirectorySeparator())
{
SourcePattern = new FilePattern(SourcePattern.BaseDirectory, String.Join("", SourcePattern.Tokens) + "...");
}
else if (Files != null)
{
SourcePattern = SourcePattern.AsDirectoryPattern();
}
// If we have multiple potential source files, but no wildcards in the output pattern, assume it's a directory and append the pattern from the source.
if (SourcePattern.ContainsWildcards() && !TargetPattern.ContainsWildcards())
{
StringBuilder NewPattern = new StringBuilder();
foreach (string Token in TargetPattern.Tokens)
{
NewPattern.Append(Token);
}
if (NewPattern.Length > 0 && NewPattern[NewPattern.Length - 1] != Path.DirectorySeparatorChar)
{
NewPattern.Append(Path.DirectorySeparatorChar);
}
foreach (string Token in SourcePattern.Tokens)
{
NewPattern.Append(Token);
}
TargetPattern = new FilePattern(TargetPattern.BaseDirectory, NewPattern.ToString());
}
// If the target pattern ends with a directory separator, treat it as a full directory match if it has wildcards, or a copy of the source pattern if not
if (TargetPattern.EndsWithDirectorySeparator())
{
TargetPattern = new FilePattern(TargetPattern.BaseDirectory, String.Join("", TargetPattern.Tokens) + "...");
}
// Handle the case where source and target pattern are both individual files
Dictionary <FileReference, FileReference> TargetFileToSourceFile = new Dictionary <FileReference, FileReference>();
if (SourcePattern.ContainsWildcards() || TargetPattern.ContainsWildcards())
{
// Check the two patterns are compatible
if (!SourcePattern.IsCompatibleWith(TargetPattern))
{
throw new AutomationException("File patterns '{0}' and '{1}' do not have matching wildcards", SourcePattern, TargetPattern);
}
// Create a filter to match the source files
FileFilter Filter = new FileFilter(FileFilterType.Exclude);
Filter.Include(String.Join("", SourcePattern.Tokens));
// Apply it to the source directory
List <FileReference> SourceFiles;
if (Files == null)
{
SourceFiles = Filter.ApplyToDirectory(SourcePattern.BaseDirectory, true);
}
else
{
SourceFiles = CheckInputFiles(Files, SourcePattern.BaseDirectory);
}
// Map them onto output files
FileReference[] TargetFiles = new FileReference[SourceFiles.Count];
// Get the source and target regexes
string SourceRegex = SourcePattern.GetRegexPattern();
string TargetRegex = TargetPattern.GetRegexReplacementPattern();
for (int Idx = 0; Idx < SourceFiles.Count; Idx++)
{
string SourceRelativePath = SourceFiles[Idx].MakeRelativeTo(SourcePattern.BaseDirectory);
string TargetRelativePath = Regex.Replace(SourceRelativePath, SourceRegex, TargetRegex);
TargetFiles[Idx] = FileReference.Combine(TargetPattern.BaseDirectory, TargetRelativePath);
}
// Add them to the output map
for (int Idx = 0; Idx < TargetFiles.Length; Idx++)
{
FileReference ExistingSourceFile;
if (TargetFileToSourceFile.TryGetValue(TargetFiles[Idx], out ExistingSourceFile) && ExistingSourceFile != SourceFiles[Idx])
{
throw new AutomationException("Output file '{0}' is mapped from '{1}' and '{2}'", TargetFiles[Idx], ExistingSourceFile, SourceFiles[Idx]);
}
TargetFileToSourceFile[TargetFiles[Idx]] = SourceFiles[Idx];
}
}
else
{
// Just copy a single file
FileReference SourceFile = SourcePattern.GetSingleFile();
if (FileReference.Exists(SourceFile))
{
FileReference TargetFile = TargetPattern.GetSingleFile();
TargetFileToSourceFile[TargetFile] = SourceFile;
}
else
{
throw new AutomationException("Source file '{0}' does not exist", SourceFile);
}
}
// Check that no source file is also destination file
foreach (FileReference SourceFile in TargetFileToSourceFile.Values)
{
if (TargetFileToSourceFile.ContainsKey(SourceFile))
{
throw new AutomationException("'{0}' is listed as a source and target file", SourceFile);
}
}
// Return the map
return(TargetFileToSourceFile);
}
public override void ExecuteBuild()
{
// Command parameters - not all required if using existing manifest
string ProjectPath = ParseParamValue("ProjectPath");
string ManifestFile = ParseParamValue("ManifestFile");
string UE4Exe = ParseParamValue("UE4Exe", "UE4Editor-Cmd.exe");
string ReplacedPaths = ParseParamValue("ReplacedPaths", "");
string ReplacedClasses = ParseParamValue("ReplacedClasses", "");
string ExcludedPaths = ParseParamValue("ExcludedPaths", "");
string ExcludedClasses = ParseParamValue("ExcludedClasses", "");
DirectoryReference BaseDir = new DirectoryReference(ParseParamValue("BaseDir"));
DirectoryReference AssetSourcePath = new DirectoryReference(ParseParamValue("AssetSourcePath"));
bool UseExistingManifest = ParseParam("UseExistingManifest");
if (!UseExistingManifest || !FileExists_NoExceptions(ManifestFile))
{
// Run commandlet to generate list of assets to replace
FileReference Project = new FileReference(ProjectPath);
var Dir = Path.GetDirectoryName(ManifestFile);
var Filename = Path.GetFileName(ManifestFile);
if (String.IsNullOrEmpty(Dir) || String.IsNullOrEmpty(Filename))
{
throw new AutomationException("GenerateDistillFileSets should have a full path and file for {0}.", ManifestFile);
}
CreateDirectory(Dir);
if (FileExists_NoExceptions(ManifestFile))
{
DeleteFile(ManifestFile);
}
RunCommandlet(Project, UE4Exe, "GenerateAssetManifest", String.Format("-ManifestFile={0} -IncludedPaths={1} -IncludedClasses={2} -ExcludedPaths={3} -ExcludedClasses={4}", CommandUtils.MakePathSafeToUseWithCommandLine(ManifestFile), ReplacedPaths, ReplacedClasses, ExcludedPaths, ExcludedClasses));
if (!FileExists_NoExceptions(ManifestFile))
{
throw new AutomationException("GenerateAssetManifest did not produce a manifest for {0}.", Project);
}
}
// Read Source Files from Manifest
List <string> Lines = new List <string>(ReadAllLines(ManifestFile));
if (Lines.Count < 1)
{
throw new AutomationException("Manifest file {0} does not list any files.", ManifestFile);
}
List <string> SourceFiles = new List <string>();
foreach (var ThisFile in Lines)
{
var TestFile = CombinePaths(ThisFile);
if (!FileExists_NoExceptions(TestFile))
{
throw new AutomationException("GenerateAssetManifest produced {0}, but {1} doesn't exist.", ThisFile, TestFile);
}
// we correct the case here
var TestFileInfo = new FileInfo(TestFile);
var FinalFile = CombinePaths(TestFileInfo.FullName);
if (!FileExists_NoExceptions(FinalFile))
{
throw new AutomationException("GenerateDistillFileSets produced {0}, but {1} doesn't exist.", ThisFile, FinalFile);
}
SourceFiles.Add(FinalFile);
}
// Delete all original files - might not always be an asset to replace them with
foreach (string SourceFile in SourceFiles)
{
CommandUtils.DeleteFile(SourceFile);
}
// Convert Source file paths to Asset Source Paths
foreach (string ReplacedFile in SourceFiles)
{
FileReference ReplacementFile = FileReference.Combine(AssetSourcePath, new FileReference(ReplacedFile).MakeRelativeTo(BaseDir));
if (FileReference.Exists(ReplacementFile))
{
CommandUtils.CopyFile(ReplacementFile.FullName, ReplacedFile);
}
}
}
public override void ExecuteBuild()
{
// Get the plugin filename
string PluginParam = ParseParamValue("Plugin");
if (PluginParam == null)
{
throw new AutomationException("Plugin file name was not specified via the -plugin argument");
}
// Read the plugin
FileReference PluginFile = new FileReference(PluginParam);
DirectoryReference PluginDirectory = PluginFile.Directory;
PluginDescriptor Plugin = PluginDescriptor.FromFile(PluginFile);
// Clean the intermediate build directory
DirectoryReference IntermediateBuildDirectory = DirectoryReference.Combine(PluginDirectory, "Intermediate", "Build");
if (CommandUtils.DirectoryExists(IntermediateBuildDirectory.FullName))
{
CommandUtils.DeleteDirectory(IntermediateBuildDirectory.FullName);
}
// Create a host project for the plugin. For script generator plugins, we need to have UHT be able to load it - and that can only happen if it's enabled in a project.
DirectoryReference HostProjectDirectory = DirectoryReference.Combine(new DirectoryReference(CommandUtils.CmdEnv.LocalRoot), "HostProject");
if (CommandUtils.DirectoryExists(HostProjectDirectory.FullName))
{
CommandUtils.DeleteDirectory(HostProjectDirectory.FullName);
}
DirectoryReference HostProjectPluginDirectory = DirectoryReference.Combine(HostProjectDirectory, "Plugins", PluginFile.GetFileNameWithoutExtension());
string[] CopyPluginFiles = Directory.EnumerateFiles(PluginDirectory.FullName, "*", SearchOption.AllDirectories).ToArray();
foreach (string CopyPluginFile in CopyPluginFiles)
{
CommandUtils.CopyFile(CopyPluginFile, CommandUtils.MakeRerootedFilePath(CopyPluginFile, PluginDirectory.FullName, HostProjectPluginDirectory.FullName));
}
FileReference HostProjectPluginFile = FileReference.Combine(HostProjectPluginDirectory, PluginFile.GetFileName());
FileReference HostProjectFile = FileReference.Combine(HostProjectDirectory, "HostProject.uproject");
File.WriteAllText(HostProjectFile.FullName, "{ \"FileVersion\": 3, \"Plugins\": [ { \"Name\": \"" + PluginFile.GetFileNameWithoutExtension() + "\", \"Enabled\": true } ] }");
// Get any additional arguments from the commandline
string AdditionalArgs = "";
// Build the host platforms
List <string> ReceiptFileNames = new List <string>();
UnrealTargetPlatform HostPlatform = BuildHostPlatform.Current.Platform;
if (!ParseParam("NoHostPlatform"))
{
if (Plugin.bCanBeUsedWithUnrealHeaderTool)
{
BuildPluginWithUBT(PluginFile, Plugin, null, "UnrealHeaderTool", TargetRules.TargetType.Program, HostPlatform, UnrealTargetConfiguration.Development, ReceiptFileNames, String.Format("{0} -plugin {1}", AdditionalArgs, CommandUtils.MakePathSafeToUseWithCommandLine(HostProjectPluginFile.FullName)));
}
BuildPluginWithUBT(PluginFile, Plugin, HostProjectFile, "UE4Editor", TargetRules.TargetType.Editor, HostPlatform, UnrealTargetConfiguration.Development, ReceiptFileNames, AdditionalArgs);
}
// Add the game targets
List <UnrealTargetPlatform> TargetPlatforms = Rocket.RocketBuild.GetTargetPlatforms(this, HostPlatform);
foreach (UnrealTargetPlatform TargetPlatform in TargetPlatforms)
{
if (Rocket.RocketBuild.IsCodeTargetPlatform(HostPlatform, TargetPlatform))
{
BuildPluginWithUBT(PluginFile, Plugin, HostProjectFile, "UE4Game", TargetRules.TargetType.Game, TargetPlatform, UnrealTargetConfiguration.Development, ReceiptFileNames, AdditionalArgs);
BuildPluginWithUBT(PluginFile, Plugin, HostProjectFile, "UE4Game", TargetRules.TargetType.Game, TargetPlatform, UnrealTargetConfiguration.Shipping, ReceiptFileNames, AdditionalArgs);
}
}
// Package the plugin to the output folder
string PackageDirectory = ParseParamValue("Package");
if (PackageDirectory != null)
{
List <BuildProduct> BuildProducts = GetBuildProductsFromReceipts(UnrealBuildTool.UnrealBuildTool.EngineDirectory, HostProjectDirectory, ReceiptFileNames);
PackagePlugin(HostProjectPluginFile, BuildProducts, PackageDirectory);
}
}
public override void GetFilesToDeployOrStage(ProjectParams Params, DeploymentContext SC)
{
if (SC.bStageCrashReporter)
{
FileReference ReceiptFileName = TargetReceipt.GetDefaultPath(CommandUtils.EngineDirectory, "CrashReportClient", SC.StageTargetPlatform.PlatformType, UnrealTargetConfiguration.Shipping, null);
if (FileReference.Exists(ReceiptFileName))
{
DirectoryReference EngineDir = CommandUtils.EngineDirectory;
DirectoryReference ProjectDir = DirectoryReference.FromFile(Params.RawProjectPath);
TargetReceipt Receipt = TargetReceipt.Read(ReceiptFileName, EngineDir, ProjectDir);
SC.StageBuildProductsFromReceipt(Receipt, true, false);
}
}
// Stage all the build products
Console.WriteLine("Staging all {0} build products", SC.StageTargets.Count);
int BuildProductIdx = 0;
foreach (StageTarget Target in SC.StageTargets)
{
Console.WriteLine(" Product {0}: {1}", BuildProductIdx, Target.Receipt.TargetName);
SC.StageBuildProductsFromReceipt(Target.Receipt, Target.RequireFilesExist, Params.bTreatNonShippingBinariesAsDebugFiles);
++BuildProductIdx;
}
FileReference SplashImage = FileReference.Combine(SC.ProjectRoot, "Content", "Splash", "Splash.bmp");
if (FileReference.Exists(SplashImage))
{
SC.StageFile(StagedFileType.NonUFS, SplashImage);
}
// Stage the bootstrap executable
if (!Params.NoBootstrapExe)
{
foreach (StageTarget Target in SC.StageTargets)
{
BuildProduct Executable = Target.Receipt.BuildProducts.FirstOrDefault(x => x.Type == BuildProductType.Executable);
if (Executable != null)
{
// only create bootstraps for executables
string FullExecutablePath = Path.GetFullPath(Executable.Path.FullName);
if (Executable.Path.FullName.Replace("\\", "/").Contains("/" + TargetPlatformType.ToString() + "/"))
{
string BootstrapArguments = "";
if (!ShouldStageCommandLine(Params, SC))
{
if (!SC.IsCodeBasedProject)
{
BootstrapArguments = String.Format("\\\"../../../{0}/{0}.uproject\\\"", SC.ShortProjectName);
}
else
{
BootstrapArguments = SC.ShortProjectName;
}
}
string BootstrapExeName;
if (SC.StageTargetConfigurations.Count > 1)
{
BootstrapExeName = Path.GetFileName(Executable.Path.FullName);
}
else if (Params.IsCodeBasedProject)
{
BootstrapExeName = Target.Receipt.TargetName;
}
else
{
BootstrapExeName = SC.ShortProjectName;
}
List <StagedFileReference> StagePaths = SC.FilesToStage.NonUFSFiles.Where(x => x.Value == Executable.Path).Select(x => x.Key).ToList();
foreach (StagedFileReference StagePath in StagePaths)
{
//#nv begin #Blast Linux build
StageBootstrapExecutable(SC, Target, BootstrapExeName + ".sh", FullExecutablePath, StagePath.Name, BootstrapArguments); //@third party code - NVSTUDIOS Set LD_LIBRARY_PATH
//nv end
}
}
}
}
}
}
public override void GetFilesToDeployOrStage(ProjectParams Params, DeploymentContext SC)
{
// if (UnrealBuildTool.BuildHostPlatform.Current.Platform != UnrealTargetPlatform.Mac)
{
// copy the icons/launch screens from the engine
{
DirectoryReference SourcePath = DirectoryReference.Combine(SC.LocalRoot, "Engine", "Binaries", "TVOS", "AssetCatalog");
SC.StageFiles(StagedFileType.NonUFS, SourcePath, "Assets.car", false, null, StagedDirectoryReference.Root, true, false);
}
// copy any additional framework assets that will be needed at runtime
{
DirectoryReference SourcePath = DirectoryReference.Combine((SC.IsCodeBasedProject ? SC.ProjectRoot : SC.EngineRoot), "Intermediate", "TVOS", "FrameworkAssets");
if (DirectoryReference.Exists(SourcePath))
{
SC.StageFiles(StagedFileType.NonUFS, SourcePath, "*.*", true, null, StagedDirectoryReference.Root, true, false);
}
}
// copy the icons/launch screens from the game (may stomp the engine copies)
{
DirectoryReference SourcePath = DirectoryReference.Combine(SC.ProjectRoot, "Binaries", "TVOS", "AssetCatalog");
SC.StageFiles(StagedFileType.NonUFS, SourcePath, "Assets.car", false, null, StagedDirectoryReference.Root, true, false);
}
// copy the plist (only if code signing, as it's protected by the code sign blob in the executable and can't be modified independently)
if (GetCodeSignDesirability(Params))
{
DirectoryReference SourcePath = DirectoryReference.Combine((SC.IsCodeBasedProject ? SC.ProjectRoot : SC.EngineRoot), "Intermediate", "TVOS");
FileReference TargetPListFile = FileReference.Combine(SourcePath, (SC.IsCodeBasedProject ? SC.ShortProjectName : "UE4Game") + "-Info.plist");
// if (!File.Exists(TargetPListFile))
{
// ensure the plist, entitlements, and provision files are properly copied
Console.WriteLine("CookPlat {0}, this {1}", GetCookPlatform(false, false), ToString());
if (!SC.IsCodeBasedProject)
{
UnrealBuildTool.PlatformExports.SetRemoteIniPath(SC.ProjectRoot.FullName);
}
if (SC.StageTargetConfigurations.Count != 1)
{
throw new AutomationException("iOS is currently only able to package one target configuration at a time, but StageTargetConfigurations contained {0} configurations", SC.StageTargetConfigurations.Count);
}
var TargetConfiguration = SC.StageTargetConfigurations[0];
bool bSupportsPortrait = false;
bool bSupportsLandscape = false;
bool bSkipIcons = false;
DeployGeneratePList(SC.RawProjectPath, TargetConfiguration, (SC.IsCodeBasedProject ? SC.ProjectRoot : SC.EngineRoot), !SC.IsCodeBasedProject, (SC.IsCodeBasedProject ? SC.ShortProjectName : "UE4Game"), SC.ShortProjectName, SC.EngineRoot, DirectoryReference.Combine((SC.IsCodeBasedProject ? SC.ProjectRoot : SC.EngineRoot), "Binaries", "TVOS", "Payload", (SC.IsCodeBasedProject ? SC.ShortProjectName : "UE4Game") + ".app"), out bSupportsPortrait, out bSupportsLandscape, out bSkipIcons);
}
SC.StageFiles(StagedFileType.NonUFS, SourcePath, TargetPListFile.GetFileName(), false, null, StagedDirectoryReference.Root, false, false, "Info.plist");
}
}
// copy the movies from the project
{
SC.StageFiles(StagedFileType.NonUFS, DirectoryReference.Combine(SC.ProjectRoot, "Build/TVOS/Resources/Movies"), "*", false, null, StagedDirectoryReference.Root, true, false);
SC.StageFiles(StagedFileType.NonUFS, DirectoryReference.Combine(SC.ProjectRoot, "Content/Movies"), "*", true, null, StagedDirectoryReference.Root, true, false);
}
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
public override void Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Get the project path, and check it exists
FileReference ProjectFile = Parameters.Project;
if (Parameters.Project != null && !FileReference.Exists(ProjectFile))
{
throw new AutomationException("Couldn't find project '{0}'", ProjectFile.FullName);
}
// Get the directories used for staging this project
DirectoryReference SourceEngineDir = CommandUtils.EngineDirectory;
DirectoryReference SourceProjectDir = (ProjectFile == null)? SourceEngineDir : ProjectFile.Directory;
// Get the output directories. We flatten the directory structure on output.
DirectoryReference TargetDir = Parameters.ToDir;
DirectoryReference TargetEngineDir = DirectoryReference.Combine(TargetDir, "Engine");
DirectoryReference TargetProjectDir = DirectoryReference.Combine(TargetDir, ProjectFile.GetFileNameWithoutExtension());
// Get the path to the receipt
FileReference ReceiptFileName = TargetReceipt.GetDefaultPath(SourceProjectDir, Parameters.Target, Parameters.Platform, Parameters.Configuration, Parameters.Architecture);
// Try to load it
TargetReceipt Receipt;
if (!TargetReceipt.TryRead(ReceiptFileName, out Receipt))
{
throw new AutomationException("Couldn't read receipt '{0}'", ReceiptFileName);
}
// Stage all the build products needed at runtime
HashSet <FileReference> SourceFiles = new HashSet <FileReference>();
foreach (BuildProduct BuildProduct in Receipt.BuildProducts)
{
SourceFiles.Add(BuildProduct.Path);
}
foreach (RuntimeDependency RuntimeDependency in Receipt.RuntimeDependencies.Where(x => x.Type != StagedFileType.UFS))
{
SourceFiles.Add(RuntimeDependency.Path);
}
// Get all the target files
List <FileReference> TargetFiles = new List <FileReference>();
foreach (FileReference SourceFile in SourceFiles)
{
// Get the destination file to copy to, mapping to the new engine and project directories as appropriate
FileReference TargetFile;
if (SourceFile.IsUnderDirectory(SourceEngineDir))
{
TargetFile = FileReference.Combine(TargetEngineDir, SourceFile.MakeRelativeTo(SourceEngineDir));
}
else
{
TargetFile = FileReference.Combine(TargetProjectDir, SourceFile.MakeRelativeTo(SourceProjectDir));
}
// Fixup the case of the output file. Would expect Platform.DeployLowerCaseFilenames() to return true here, but seems not to be the case.
if (Parameters.Platform == UnrealTargetPlatform.PS4)
{
TargetFile = FileReference.Combine(TargetDir, TargetFile.MakeRelativeTo(TargetDir).ToLowerInvariant());
}
// Only copy the output file if it doesn't already exist. We can stage multiple targets to the same output directory.
if (Parameters.Overwrite || !FileReference.Exists(TargetFile))
{
DirectoryReference.CreateDirectory(TargetFile.Directory);
CommandUtils.CopyFile(SourceFile.FullName, TargetFile.FullName);
// Force all destination files to not readonly.
CommandUtils.SetFileAttributes(TargetFile.FullName, ReadOnly: false);
}
// Add it to the list of target files
TargetFiles.Add(TargetFile);
}
// Apply the optional tag to the build products
foreach (string TagName in FindTagNamesFromList(Parameters.Tag))
{
FindOrAddTagSet(TagNameToFileSet, TagName).UnionWith(TargetFiles);
}
// Add the target file to the list of build products
BuildProducts.UnionWith(TargetFiles);
}
/// <summary>
/// Gets the path to the manifest created for a node's output.
/// </summary>
/// <param name="BaseDir">A local or shared temp storage root directory.</param>
/// <param name="NodeName">Name of the node to get the file for</param>
/// <param name="OutputName">Name of the output to get the manifest for</param>
static FileReference GetManifestFile(DirectoryReference BaseDir, string NodeName, string OutputName)
{
return(FileReference.Combine(BaseDir, NodeName, String.Format("Output-{0}.xml", OutputName)));
}