public void StageAppLocalDependencies(ProjectParams Params, DeploymentContext SC, string PlatformDir)
{
Dictionary <string, string> PathVariables = new Dictionary <string, string>();
PathVariables["EngineDir"] = SC.EngineRoot.FullName;
PathVariables["ProjectDir"] = SC.ProjectRoot.FullName;
// support multiple comma-separated paths
string[] AppLocalDirectories = Params.AppLocalDirectory.Split(';');
foreach (string AppLocalDirectory in AppLocalDirectories)
{
string ExpandedAppLocalDir = Utils.ExpandVariables(AppLocalDirectory, PathVariables);
DirectoryReference BaseAppLocalDependenciesPath = Path.IsPathRooted(ExpandedAppLocalDir) ? new DirectoryReference(CombinePaths(ExpandedAppLocalDir, PlatformDir)) : DirectoryReference.Combine(SC.ProjectRoot, ExpandedAppLocalDir, PlatformDir);
if (DirectoryReference.Exists(BaseAppLocalDependenciesPath))
{
StagedDirectoryReference ProjectBinaryPath = new StagedDirectoryReference(SC.ProjectBinariesFolder.MakeRelativeTo(SC.ProjectRoot.ParentDirectory));
StagedDirectoryReference EngineBinaryPath = new StagedDirectoryReference(CombinePaths("Engine", "Binaries", PlatformDir));
Log("Copying AppLocal dependencies from {0} to {1} and {2}", BaseAppLocalDependenciesPath, ProjectBinaryPath, EngineBinaryPath);
// Stage files in subdirs
foreach (DirectoryReference DependencyDirectory in DirectoryReference.EnumerateDirectories(BaseAppLocalDependenciesPath))
{
SC.StageFiles(StagedFileType.NonUFS, DependencyDirectory, StageFilesSearch.TopDirectoryOnly, ProjectBinaryPath);
SC.StageFiles(StagedFileType.NonUFS, DependencyDirectory, StageFilesSearch.TopDirectoryOnly, EngineBinaryPath);
}
// stage loose files here
SC.StageFiles(StagedFileType.NonUFS, BaseAppLocalDependenciesPath, StageFilesSearch.AllDirectories, ProjectBinaryPath);
SC.StageFiles(StagedFileType.NonUFS, BaseAppLocalDependenciesPath, StageFilesSearch.AllDirectories, EngineBinaryPath);
}
else
{
LogWarning("Unable to deploy AppLocalDirectory dependencies. No such path: {0}", BaseAppLocalDependenciesPath);
}
}
}
private void RunCsvTool(string ToolName, string Arguments)
{
var ToolPath = FileReference.Combine(DirectoryReference.Combine(RootDir, "Engine", "Source", "Programs", "CSVTools", "Binaries"), ToolName + ".exe");
Run(ToolPath.FullName, Arguments);
}
public override void ExecuteBuild()
{
string[] ProjectParams = ParseParamValues("Project");
string UpdateDirParam = ParseParamValue("UpdateDir", null);
if (UpdateDirParam == null)
{
throw new AutomationException("Missing -UpdateDir=... parameter");
}
DirectoryReference UpdateDir = new DirectoryReference(UpdateDirParam);
bool bWrite = ParseParam("Write");
// Get all the root dirs
HashSet <DirectoryReference> RootDirs = new HashSet <DirectoryReference>();
RootDirs.Add(EngineDirectory);
// Add the enterprise edirectory
DirectoryReference EnterpriseDirectory = DirectoryReference.Combine(RootDirectory, "Enterprise");
if (DirectoryReference.Exists(EnterpriseDirectory))
{
RootDirs.Add(EnterpriseDirectory);
}
// Add the project directories
foreach (string ProjectParam in ProjectParams)
{
FileReference ProjectLocation = new FileReference(ProjectParam);
if (!FileReference.Exists(ProjectLocation))
{
throw new AutomationException("Unable to find project '{0}'", ProjectLocation);
}
RootDirs.Add(ProjectLocation.Directory);
}
// Find all the modules
HashSet <DirectoryReference> ModuleDirs = new HashSet <DirectoryReference>();
foreach (DirectoryReference RootDir in RootDirs)
{
// Find all the modules from the source folder
DirectoryReference SourceDir = DirectoryReference.Combine(RootDir, "Source");
if (DirectoryReference.Exists(SourceDir))
{
foreach (FileReference ModuleFile in DirectoryReference.EnumerateFiles(SourceDir, "*.Build.cs", SearchOption.AllDirectories))
{
ModuleDirs.Add(ModuleFile.Directory);
}
}
// Find all the modules under the plugins folder
DirectoryReference PluginsDir = DirectoryReference.Combine(RootDir, "Plugins");
foreach (FileReference PluginFile in DirectoryReference.EnumerateFiles(PluginsDir, "*.uplugin", SearchOption.AllDirectories))
{
DirectoryReference PluginSourceDir = DirectoryReference.Combine(PluginFile.Directory, "Source");
if (DirectoryReference.Exists(PluginSourceDir))
{
foreach (FileReference PluginModuleFile in DirectoryReference.EnumerateFiles(PluginSourceDir, "*.Build.cs", SearchOption.AllDirectories))
{
ModuleDirs.Add(PluginModuleFile.Directory);
}
}
}
}
// Find a mapping from old to new include paths
Dictionary <string, Tuple <string, FileReference> > RemapIncludePaths = new Dictionary <string, Tuple <string, FileReference> >(StringComparer.InvariantCultureIgnoreCase);
foreach (DirectoryReference ModuleDir in ModuleDirs)
{
DirectoryReference ModulePublicDir = DirectoryReference.Combine(ModuleDir, "Public");
if (DirectoryReference.Exists(ModulePublicDir))
{
foreach (FileReference HeaderFile in DirectoryReference.EnumerateFiles(ModulePublicDir, "*.h", SearchOption.AllDirectories))
{
string BaseIncludeFile = HeaderFile.GetFileName();
Tuple <string, FileReference> ExistingIncludeName;
if (RemapIncludePaths.TryGetValue(BaseIncludeFile, out ExistingIncludeName))
{
LogWarning("Multiple include paths for {0}: {1}, {2}", BaseIncludeFile, ExistingIncludeName.Item2, HeaderFile);
}
else
{
RemapIncludePaths.Add(BaseIncludeFile, Tuple.Create(HeaderFile.MakeRelativeTo(ModulePublicDir).Replace('\\', '/'), HeaderFile));
}
}
}
}
// List of folders to exclude from updates
string[] ExcludeFoldersFromUpdate =
{
"Intermediate",
"ThirdParty"
};
// Enumerate all the files to update
HashSet <FileReference> UpdateFiles = new HashSet <FileReference>();
foreach (FileReference UpdateFile in DirectoryReference.EnumerateFiles(UpdateDir, "*", SearchOption.AllDirectories))
{
if (!UpdateFile.ContainsAnyNames(ExcludeFoldersFromUpdate, UpdateDir))
{
if (UpdateFile.HasExtension(".cpp") | UpdateFile.HasExtension(".h") || UpdateFile.HasExtension(".inl"))
{
UpdateFiles.Add(UpdateFile);
}
}
}
// Process all the source files
Dictionary <FileReference, string[]> ModifiedFiles = new Dictionary <FileReference, string[]>();
foreach (FileReference UpdateFile in UpdateFiles)
{
bool bModifiedFile = false;
string[] Lines = FileReference.ReadAllLines(UpdateFile);
for (int Idx = 0; Idx < Lines.Length; Idx++)
{
Match Match = Regex.Match(Lines[Idx], "^(\\s*#\\s*include\\s+\\\")([^\"]+)(\\\".*)$");
if (Match.Success)
{
string IncludePath = Match.Groups[2].Value;
Tuple <string, FileReference> NewIncludePath;
if (RemapIncludePaths.TryGetValue(IncludePath, out NewIncludePath))
{
if (IncludePath != NewIncludePath.Item1)
{
// Log("{0}: Changing {1} -> {2}", UpdateFile, IncludePath, NewIncludePath.Item1);
Lines[Idx] = String.Format("{0}{1}{2}", Match.Groups[1].Value, NewIncludePath.Item1, Match.Groups[3].Value);
bModifiedFile = true;
}
}
}
}
if (bModifiedFile)
{
ModifiedFiles.Add(UpdateFile, Lines);
}
}
// Output them all to disk
if (bWrite && ModifiedFiles.Count > 0)
{
LogInformation("Updating {0} files...", ModifiedFiles.Count);
List <FileReference> FilesToCheckOut = new List <FileReference>();
foreach (FileReference ModifiedFile in ModifiedFiles.Keys)
{
if ((FileReference.GetAttributes(ModifiedFile) & FileAttributes.ReadOnly) != 0)
{
FilesToCheckOut.Add(ModifiedFile);
}
}
if (FilesToCheckOut.Count > 0)
{
if (!P4Enabled)
{
throw new AutomationException("{0} files have been modified, but are read only. Run with -P4 to enable Perforce checkout.\n{1}", FilesToCheckOut.Count, String.Join("\n", FilesToCheckOut.Select(x => " " + x)));
}
LogInformation("Checking out files from Perforce");
int ChangeNumber = P4.CreateChange(Description: "Updating source files");
P4.Edit(ChangeNumber, FilesToCheckOut.Select(x => x.FullName).ToList(), false);
}
foreach (KeyValuePair <FileReference, string[]> FileToWrite in ModifiedFiles)
{
LogInformation("Writing {0}", FileToWrite.Key);
FileReference.WriteAllLines(FileToWrite.Key, FileToWrite.Value);
}
}
}
public override void GetFilesToDeployOrStage(ProjectParams Params, DeploymentContext SC)
{
// Stage all the build products
foreach (StageTarget Target in SC.StageTargets)
{
SC.StageBuildProductsFromReceipt(Target.Receipt, Target.RequireFilesExist, Params.bTreatNonShippingBinariesAsDebugFiles);
}
if (SC.bStageCrashReporter)
{
StagedDirectoryReference CrashReportClientPath = StagedDirectoryReference.Combine("Engine/Binaries", SC.PlatformDir, "CrashReportClient.app");
StageAppBundle(SC, StagedFileType.NonUFS, DirectoryReference.Combine(SC.LocalRoot, "Engine/Binaries", SC.PlatformDir, "CrashReportClient.app"), CrashReportClientPath);
}
// Find the app bundle path
List <FileReference> Exes = GetExecutableNames(SC);
foreach (var Exe in Exes)
{
string AppBundlePath = "";
if (Exe.IsUnderDirectory(DirectoryReference.Combine(SC.RuntimeProjectRootDir, "Binaries", SC.PlatformDir)))
{
AppBundlePath = CombinePaths(SC.ShortProjectName, "Binaries", SC.PlatformDir, Path.GetFileNameWithoutExtension(Exe.FullName) + ".app");
}
else if (Exe.IsUnderDirectory(DirectoryReference.Combine(SC.RuntimeRootDir, "Engine/Binaries", SC.PlatformDir)))
{
AppBundlePath = CombinePaths("Engine/Binaries", SC.PlatformDir, Path.GetFileNameWithoutExtension(Exe.FullName) + ".app");
}
// Copy the custom icon and Steam dylib, if needed
if (!string.IsNullOrEmpty(AppBundlePath))
{
SC.StageFiles(StagedFileType.NonUFS, DirectoryReference.Combine(SC.ProjectRoot, "Build/Mac"), "Application.icns", false, null, new StagedDirectoryReference(CombinePaths(AppBundlePath, "Contents/Resources")), true);
}
}
// Copy the splash screen, Mac specific
SC.StageFiles(StagedFileType.NonUFS, DirectoryReference.Combine(SC.ProjectRoot, "Content/Splash"), "Splash.bmp", false, null, null, true);
// Stage the bootstrap executable
if (!Params.NoBootstrapExe)
{
foreach (StageTarget Target in SC.StageTargets)
{
BuildProduct Executable = Target.Receipt.BuildProducts.FirstOrDefault(x => x.Type == BuildProductType.Executable);
if (Executable != null)
{
// only create bootstraps for executables
List <StagedFileReference> StagedFiles = SC.FilesToStage.NonUFSStagingFiles.Where(x => x.Value == Executable.Path).Select(x => x.Key).ToList();
if (StagedFiles.Count > 0 && Executable.Path.FullName.Replace("\\", "/").Contains("/" + TargetPlatformType.ToString() + "/"))
{
string BootstrapArguments = "";
if (!ShouldStageCommandLine(Params, SC))
{
if (!SC.IsCodeBasedProject)
{
BootstrapArguments = String.Format("../../../{0}/{0}.uproject", SC.ShortProjectName);
}
else
{
BootstrapArguments = SC.ShortProjectName;
}
}
string BootstrapExeName;
if (SC.StageTargetConfigurations.Count > 1)
{
BootstrapExeName = Path.GetFileName(Executable.Path.FullName) + ".app";
}
else if (Params.IsCodeBasedProject)
{
BootstrapExeName = Target.Receipt.TargetName + ".app";
}
else
{
BootstrapExeName = SC.ShortProjectName + ".app";
}
string AppPath = Executable.Path.FullName.Substring(0, Executable.Path.FullName.LastIndexOf(".app/") + 4);
foreach (var DestPath in StagedFiles)
{
string AppRelativePath = DestPath.Name.Substring(0, DestPath.Name.LastIndexOf(".app/") + 4);
StageBootstrapExecutable(SC, BootstrapExeName, AppPath, AppRelativePath, BootstrapArguments);
}
}
}
}
}
// Copy the ShaderCache files, if they exist
SC.StageFiles(StagedFileType.UFS, DirectoryReference.Combine(SC.ProjectRoot, "Content"), "DrawCache.ushadercache", false, null, null, true);
SC.StageFiles(StagedFileType.UFS, DirectoryReference.Combine(SC.ProjectRoot, "Content"), "ByteCodeCache.ushadercode", false, null, null, true);
}
public DeploymentContext(
FileReference RawProjectPathOrName,
DirectoryReference InLocalRoot,
DirectoryReference BaseStageDirectory,
DirectoryReference BaseArchiveDirectory,
Platform InSourcePlatform,
Platform InTargetPlatform,
List <UnrealTargetConfiguration> InTargetConfigurations,
IEnumerable <StageTarget> InStageTargets,
List <String> InStageExecutables,
bool InServer,
bool InCooked,
bool InStageCrashReporter,
bool InStage,
bool InCookOnTheFly,
bool InArchive,
bool InProgram,
bool IsClientInsteadOfNoEditor,
bool InForceChunkManifests,
bool InSeparateDebugStageDirectory
)
{
bStageCrashReporter = InStageCrashReporter;
RawProjectPath = RawProjectPathOrName;
DedicatedServer = InServer;
LocalRoot = InLocalRoot;
CookSourcePlatform = InSourcePlatform;
StageTargetPlatform = InTargetPlatform;
StageTargetConfigurations = new List <UnrealTargetConfiguration>(InTargetConfigurations);
StageTargets = new List <StageTarget>(InStageTargets);
StageExecutables = InStageExecutables;
IsCodeBasedProject = ProjectUtils.IsCodeBasedUProjectFile(RawProjectPath, StageTargetConfigurations);
ShortProjectName = ProjectUtils.GetShortProjectName(RawProjectPath);
Stage = InStage;
Archive = InArchive;
if (CookSourcePlatform != null && InCooked)
{
CookPlatform = CookSourcePlatform.GetCookPlatform(DedicatedServer, IsClientInsteadOfNoEditor);
}
else if (CookSourcePlatform != null && InProgram)
{
CookPlatform = CookSourcePlatform.GetCookPlatform(false, false);
}
else
{
CookPlatform = "";
}
if (StageTargetPlatform != null && InCooked)
{
FinalCookPlatform = StageTargetPlatform.GetCookPlatform(DedicatedServer, IsClientInsteadOfNoEditor);
}
else if (StageTargetPlatform != null && InProgram)
{
FinalCookPlatform = StageTargetPlatform.GetCookPlatform(false, false);
}
else
{
FinalCookPlatform = "";
}
PlatformDir = StageTargetPlatform.PlatformType.ToString();
if (BaseStageDirectory != null)
{
StageDirectory = DirectoryReference.Combine(BaseStageDirectory, FinalCookPlatform);
DebugStageDirectory = InSeparateDebugStageDirectory? DirectoryReference.Combine(BaseStageDirectory, FinalCookPlatform + "Debug") : StageDirectory;
}
if (BaseArchiveDirectory != null)
{
ArchiveDirectory = DirectoryReference.Combine(BaseArchiveDirectory, FinalCookPlatform);
}
if (!FileReference.Exists(RawProjectPath))
{
throw new AutomationException("Can't find uproject file {0}.", RawProjectPathOrName);
}
EngineRoot = DirectoryReference.Combine(LocalRoot, "Engine");
ProjectRoot = RawProjectPath.Directory;
RelativeProjectRootForStage = new StagedDirectoryReference(ShortProjectName);
ProjectArgForCommandLines = CommandUtils.MakePathSafeToUseWithCommandLine(RawProjectPath.FullName);
CookSourceRuntimeRootDir = RuntimeRootDir = LocalRoot;
RuntimeProjectRootDir = ProjectRoot;
if (Stage)
{
CommandUtils.CreateDirectory(StageDirectory.FullName);
RuntimeRootDir = StageDirectory;
CookSourceRuntimeRootDir = DirectoryReference.Combine(BaseStageDirectory, CookPlatform);
RuntimeProjectRootDir = DirectoryReference.Combine(StageDirectory, RelativeProjectRootForStage.Name);
ProjectArgForCommandLines = CommandUtils.MakePathSafeToUseWithCommandLine(UProjectCommandLineArgInternalRoot + RelativeProjectRootForStage.Name + "/" + ShortProjectName + ".uproject");
}
if (Archive)
{
CommandUtils.CreateDirectory(ArchiveDirectory.FullName);
}
ProjectArgForCommandLines = ProjectArgForCommandLines.Replace("\\", "/");
ProjectBinariesFolder = DirectoryReference.Combine(ProjectUtils.GetClientProjectBinariesRootPath(RawProjectPath, TargetType.Game, IsCodeBasedProject), PlatformDir);
// Build a list of restricted folder names. This will comprise all other restricted platforms, plus standard restricted folder names such as NoRedist, NotForLicensees, etc...
RestrictedFolderNames.UnionWith(PlatformExports.GetPlatformFolderNames());
foreach (UnrealTargetPlatform StagePlatform in StageTargetPlatform.GetStagePlatforms())
{
RestrictedFolderNames.ExceptWith(PlatformExports.GetIncludedFolderNames(StagePlatform));
}
RestrictedFolderNames.UnionWith(RestrictedFolders.Names);
RestrictedFolderNames.Remove(StageTargetPlatform.IniPlatformType.ToString());
// Read the game config files
ConfigHierarchy GameConfig = ConfigCache.ReadHierarchy(ConfigHierarchyType.Game, ProjectRoot, InTargetPlatform.PlatformType);
// Read the list of directories to remap when staging
List <string> RemapDirectoriesList;
if (GameConfig.GetArray("Staging", "RemapDirectories", out RemapDirectoriesList))
{
foreach (string RemapDirectory in RemapDirectoriesList)
{
Dictionary <string, string> Properties;
if (!ConfigHierarchy.TryParse(RemapDirectory, out Properties))
{
throw new AutomationException("Unable to parse '{0}'", RemapDirectory);
}
string FromDir;
if (!Properties.TryGetValue("From", out FromDir))
{
throw new AutomationException("Missing 'From' property in '{0}'", RemapDirectory);
}
string ToDir;
if (!Properties.TryGetValue("To", out ToDir))
{
throw new AutomationException("Missing 'To' property in '{0}'", RemapDirectory);
}
RemapDirectories.Add(Tuple.Create(new StagedDirectoryReference(FromDir), new StagedDirectoryReference(ToDir)));
}
}
// Read the list of directories to whitelist from restricted folder warnings
List <string> WhitelistDirectoriesList;
if (GameConfig.GetArray("Staging", "WhitelistDirectories", out WhitelistDirectoriesList))
{
foreach (string WhitelistDirectory in WhitelistDirectoriesList)
{
WhitelistDirectories.Add(new StagedDirectoryReference(WhitelistDirectory));
}
}
List <string> BlacklistLocTargetsList;
if (GameConfig.GetArray("Staging", "BlacklistLocalizationTargets", out BlacklistLocTargetsList))
{
foreach (string BlacklistLocTarget in BlacklistLocTargetsList)
{
BlacklistLocalizationTargets.Add(BlacklistLocTarget);
}
}
// Read the list of files which are whitelisted to be staged
ReadConfigFileList(GameConfig, "Staging", "WhitelistConfigFiles", WhitelistConfigFiles);
ReadConfigFileList(GameConfig, "Staging", "BlacklistConfigFiles", BlacklistConfigFiles);
// If we were configured to use manifests across the whole project, then this platform should use manifests.
// Otherwise, read whether we are generating chunks from the ProjectPackagingSettings ini.
if (InForceChunkManifests)
{
PlatformUsesChunkManifests = true;
}
else
{
ConfigHierarchy GameIni = ConfigCache.ReadHierarchy(ConfigHierarchyType.Game, ProjectRoot, InTargetPlatform.PlatformType);
String IniPath = "/Script/UnrealEd.ProjectPackagingSettings";
bool bSetting = false;
if (GameIni.GetBool(IniPath, "bGenerateChunks", out bSetting))
{
PlatformUsesChunkManifests = bSetting;
}
}
}
public override void ExecuteBuild()
{
// Get the plugin filename
string PluginParam = ParseParamValue("Plugin");
if (PluginParam == null)
{
throw new AutomationException("Missing -Plugin=... argument");
}
// Check it exists
FileReference PluginFile = new FileReference(PluginParam);
if (!PluginFile.Exists())
{
throw new AutomationException("Plugin '{0}' not found", PluginFile.FullName);
}
// Get the output directory
string PackageParam = ParseParamValue("Package");
if (PackageParam == null)
{
throw new AutomationException("Missing -Package=... argument");
}
// Make sure the packaging directory is valid
DirectoryReference PackageDir = new DirectoryReference(PackageParam);
if (PluginFile.IsUnderDirectory(PackageDir))
{
throw new AutomationException("Packaged plugin output directory must be different to source");
}
if (PackageDir.IsUnderDirectory(DirectoryReference.Combine(CommandUtils.RootDirectory, "Engine")))
{
throw new AutomationException("Output directory for packaged plugin must be outside engine directory");
}
// Clear the output directory of existing stuff
if (PackageDir.Exists())
{
CommandUtils.DeleteDirectoryContents(PackageDir.FullName);
}
else
{
PackageDir.CreateDirectory();
}
// Create a placeholder FilterPlugin.ini with instructions on how to use it
FileReference SourceFilterFile = FileReference.Combine(PluginFile.Directory, "Config", "FilterPlugin.ini");
if (!SourceFilterFile.Exists())
{
List <string> Lines = new List <string>();
Lines.Add("[FilterPlugin]");
Lines.Add("; This section lists additional files which will be packaged along with your plugin. Paths should be listed relative to the root plugin directory, and");
Lines.Add("; may include \"...\", \"*\", and \"?\" wildcards to match directories, files, and individual characters respectively.");
Lines.Add(";");
Lines.Add("; Examples:");
Lines.Add("; /README.txt");
Lines.Add("; /Extras/...");
Lines.Add("; /Binaries/ThirdParty/*.dll");
SourceFilterFile.Directory.CreateDirectory();
CommandUtils.WriteAllLines_NoExceptions(SourceFilterFile.FullName, Lines.ToArray());
}
// Create a host project for the plugin. For script generator plugins, we need to have UHT be able to load it, which can only happen if it's enabled in a project.
FileReference HostProjectFile = FileReference.Combine(PackageDir, "HostProject", "HostProject.uproject");
FileReference HostProjectPluginFile = CreateHostProject(HostProjectFile, PluginFile);
// Read the plugin
CommandUtils.Log("Reading plugin from {0}...", HostProjectPluginFile);
PluginDescriptor Plugin = PluginDescriptor.FromFile(HostProjectPluginFile, false);
// Compile the plugin for all the target platforms
List <UnrealTargetPlatform> HostPlatforms = ParseParam("NoHostPlatform")? new List <UnrealTargetPlatform>() : new List <UnrealTargetPlatform> {
BuildHostPlatform.Current.Platform
};
List <UnrealTargetPlatform> TargetPlatforms = GetTargetPlatforms(this, BuildHostPlatform.Current.Platform);
FileReference[] BuildProducts = CompilePlugin(HostProjectFile, HostProjectPluginFile, Plugin, HostPlatforms, TargetPlatforms, "");
// Package up the final plugin data
PackagePlugin(HostProjectPluginFile, BuildProducts, PackageDir);
// Remove the host project
if (!ParseParam("NoDeleteHostProject"))
{
CommandUtils.DeleteDirectory(HostProjectFile.Directory.FullName);
}
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
/// <returns>True if the task succeeded</returns>
public override bool Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Get the project path, and check it exists
FileReference ProjectFile = null;
if (Parameters.Project != null)
{
ProjectFile = ResolveFile(Parameters.Project);
if (!ProjectFile.Exists())
{
CommandUtils.LogError("Couldn't find project '{0}'", ProjectFile.FullName);
return(false);
}
}
// Get the directories used for staging this project
DirectoryReference SourceEngineDir = UnrealBuildTool.UnrealBuildTool.EngineDirectory;
DirectoryReference SourceProjectDir = (ProjectFile == null)? SourceEngineDir : ProjectFile.Directory;
// Get the output directories. We flatten the directory structure on output.
DirectoryReference TargetDir = ResolveDirectory(Parameters.ToDir);
DirectoryReference TargetEngineDir = DirectoryReference.Combine(TargetDir, "Engine");
DirectoryReference TargetProjectDir = DirectoryReference.Combine(TargetDir, ProjectFile.GetFileNameWithoutExtension());
// Get the path to the receipt
string ReceiptFileName = TargetReceipt.GetDefaultPath(SourceProjectDir.FullName, Parameters.Target, Parameters.Platform, Parameters.Configuration, Parameters.Architecture);
// Try to load it
TargetReceipt Receipt;
if (!TargetReceipt.TryRead(ReceiptFileName, out Receipt))
{
CommandUtils.LogError("Couldn't read receipt '{0}'", ReceiptFileName);
return(false);
}
// Expand all the paths from the receipt
Receipt.ExpandPathVariables(SourceEngineDir, SourceProjectDir);
// Stage all the build products needed at runtime
HashSet <FileReference> SourceFiles = new HashSet <FileReference>();
foreach (BuildProduct BuildProduct in Receipt.BuildProducts.Where(x => x.Type != BuildProductType.StaticLibrary && x.Type != BuildProductType.ImportLibrary))
{
SourceFiles.Add(new FileReference(BuildProduct.Path));
}
foreach (RuntimeDependency RuntimeDependency in Receipt.RuntimeDependencies.Where(x => x.Type != StagedFileType.UFS))
{
SourceFiles.UnionWith(CommandUtils.ResolveFilespec(CommandUtils.RootDirectory, RuntimeDependency.Path, new string[] { ".../*.umap", ".../*.uasset" }));
}
// Get all the target files
List <FileReference> TargetFiles = new List <FileReference>();
foreach (FileReference SourceFile in SourceFiles)
{
// Get the destination file to copy to, mapping to the new engine and project directories as appropriate
FileReference TargetFile;
if (SourceFile.IsUnderDirectory(SourceEngineDir))
{
TargetFile = FileReference.Combine(TargetEngineDir, SourceFile.MakeRelativeTo(SourceEngineDir));
}
else
{
TargetFile = FileReference.Combine(TargetProjectDir, SourceFile.MakeRelativeTo(SourceProjectDir));
}
// Fixup the case of the output file. Would expect Platform.DeployLowerCaseFilenames() to return true here, but seems not to be the case.
if (Parameters.Platform == UnrealTargetPlatform.PS4)
{
TargetFile = FileReference.Combine(TargetDir, TargetFile.MakeRelativeTo(TargetDir).ToLowerInvariant());
}
// Only copy the output file if it doesn't already exist. We can stage multiple targets to the same output directory.
if (Parameters.Overwrite || !TargetFile.Exists())
{
TargetFile.Directory.CreateDirectory();
CommandUtils.CopyFile(SourceFile.FullName, TargetFile.FullName);
}
// Add it to the list of target files
TargetFiles.Add(TargetFile);
}
// Apply the optional tag to the build products
foreach (string TagName in FindTagNamesFromList(Parameters.Tag))
{
FindOrAddTagSet(TagNameToFileSet, TagName).UnionWith(TargetFiles);
}
// Add the target file to the list of build products
BuildProducts.UnionWith(TargetFiles);
return(true);
}
private static DirectoryReference GetProjectDirectory(TargetPlatformData TargetData, WindowsCompiler TargetWindowsCompiler = WindowsCompiler.VisualStudio2015_DEPRECATED)
{
DirectoryReference Directory = SourceRootDirectory;
return(DirectoryReference.Combine(Directory, "compiler", GetCMakeTargetDirectoryName(TargetData, TargetWindowsCompiler)));
}
private static void SetupBuildForTargetLibAndPlatform(TargetPlatformData TargetData, List <string> TargetConfigurations, List <WindowsCompiler> TargetWindowsCompilers, bool bCleanOnly)
{
string CMakeName = GetCMakeNameAndSetupEnv(TargetData);
if (TargetData.Platform == UnrealTargetPlatform.Win64)
{
// for windows platforms we support building against multiple compilers
foreach (WindowsCompiler TargetWindowsCompiler in TargetWindowsCompilers)
{
DirectoryReference CMakeTargetDirectory = GetProjectDirectory(TargetData, TargetWindowsCompiler);
MakeFreshDirectoryIfRequired(CMakeTargetDirectory);
if (!bCleanOnly)
{
LogInformation("Generating projects for lib " + TargetData.ToString());
ProcessStartInfo StartInfo = new ProcessStartInfo();
StartInfo.FileName = CMakeName;
StartInfo.WorkingDirectory = CMakeTargetDirectory.ToString();
StartInfo.Arguments = GetCMakeArguments(TargetData, "", TargetWindowsCompiler);
if (RunLocalProcessAndLogOutput(StartInfo) != 0)
{
throw new AutomationException(String.Format("Unable to generate projects for {0}.", TargetData.ToString()));
}
}
}
}
else if (TargetData.Platform == UnrealTargetPlatform.Linux)
{
foreach (string BuildConfig in TargetConfigurations)
{
DirectoryReference CMakeTargetDirectory = GetProjectDirectory(TargetData);
CMakeTargetDirectory = DirectoryReference.Combine(CMakeTargetDirectory, BuildConfig);
MakeFreshDirectoryIfRequired(CMakeTargetDirectory);
if (!bCleanOnly)
{
LogInformation("Generating projects for lib " + TargetData.ToString());
ProcessStartInfo StartInfo = new ProcessStartInfo();
StartInfo.FileName = CMakeName;
StartInfo.WorkingDirectory = CMakeTargetDirectory.ToString();
StartInfo.Arguments = GetCMakeArguments(TargetData, BuildConfig);
System.Console.WriteLine("Working in '{0}'", StartInfo.WorkingDirectory);
LogInformation("Working in '{0}'", StartInfo.WorkingDirectory);
System.Console.WriteLine("{0} {1}", StartInfo.FileName, StartInfo.Arguments);
LogInformation("{0} {1}", StartInfo.FileName, StartInfo.Arguments);
if (RunLocalProcessAndLogOutput(StartInfo) != 0)
{
throw new AutomationException(String.Format("Unable to generate projects for {0}.", TargetData.ToString()));
}
}
}
}
else if (TargetData.Platform == UnrealTargetPlatform.Mac)
{
DirectoryReference CMakeTargetDirectory = GetProjectDirectory(TargetData);
MakeFreshDirectoryIfRequired(CMakeTargetDirectory);
if (!bCleanOnly)
{
LogInformation("Generating projects for lib " + TargetData.ToString());
ProcessStartInfo StartInfo = new ProcessStartInfo();
StartInfo.FileName = CMakeName;
StartInfo.WorkingDirectory = CMakeTargetDirectory.ToString();
StartInfo.Arguments = GetCMakeArguments(TargetData);
if (RunLocalProcessAndLogOutput(StartInfo) != 0)
{
throw new AutomationException(String.Format("Unable to generate projects for {0}.", TargetData.ToString()));
}
}
}
else
{
throw new AutomationException(String.Format("Unable to generate projects for {0}.", TargetData.ToString()));
}
}
public void StageAppLocalDependencies(ProjectParams Params, DeploymentContext SC, string PlatformDir)
{
Dictionary <string, string> PathVariables = new Dictionary <string, string>();
PathVariables["EngineDir"] = SC.EngineRoot.FullName;
PathVariables["ProjectDir"] = SC.ProjectRoot.FullName;
// support multiple comma-separated paths
string[] AppLocalDirectories = Params.AppLocalDirectory.Split(';');
foreach (string AppLocalDirectory in AppLocalDirectories)
{
string ExpandedAppLocalDir = Utils.ExpandVariables(AppLocalDirectory, PathVariables);
DirectoryReference BaseAppLocalDependenciesPath = Path.IsPathRooted(ExpandedAppLocalDir) ? new DirectoryReference(CombinePaths(ExpandedAppLocalDir, PlatformDir)) : DirectoryReference.Combine(SC.ProjectRoot, ExpandedAppLocalDir, PlatformDir);
if (DirectoryReference.Exists(BaseAppLocalDependenciesPath))
{
StageAppLocalDependenciesToDir(SC, BaseAppLocalDependenciesPath, StagedDirectoryReference.Combine("Engine", "Binaries", PlatformDir));
StageAppLocalDependenciesToDir(SC, BaseAppLocalDependenciesPath, StagedDirectoryReference.Combine(SC.RelativeProjectRootForStage, "Binaries", PlatformDir));
}
else
{
LogWarning("Unable to deploy AppLocalDirectory dependencies. No such path: {0}", BaseAppLocalDependenciesPath);
}
}
}
public override bool PublishSymbols(DirectoryReference SymbolStoreDirectory, List <FileReference> Files, string Product, string BuildVersion = null)
{
// Get the SYMSTORE.EXE path, using the latest SDK version we can find.
FileReference SymStoreExe = GetSymStoreExe();
List <FileReference> FilesToAdd = Files.Where(x => x.HasExtension(".pdb") || x.HasExtension(".exe") || x.HasExtension(".dll")).ToList();
if (FilesToAdd.Count > 0)
{
DateTime Start = DateTime.Now;
DirectoryReference TempSymStoreDir = DirectoryReference.Combine(RootDirectory, "Saved", "SymStore");
string TempFileName = Path.GetTempFileName();
try
{
File.WriteAllLines(TempFileName, FilesToAdd.Select(x => x.FullName), Encoding.ASCII);
// Copy everything to the temp symstore
ProcessStartInfo StartInfo = new ProcessStartInfo();
StartInfo.FileName = SymStoreExe.FullName;
StartInfo.Arguments = string.Format("add /f \"@{0}\" /s \"{1}\" /t \"{2}\" /compress", TempFileName, TempSymStoreDir, Product);
StartInfo.UseShellExecute = false;
StartInfo.CreateNoWindow = true;
if (Utils.RunLocalProcessAndLogOutput(StartInfo) != 0)
{
return(false);
}
}
finally
{
File.Delete(TempFileName);
}
DateTime CompressDone = DateTime.Now;
LogInformation("Took {0}s to compress the symbol files", (CompressDone - Start).TotalSeconds);
// Take each new compressed file made and try and copy it to the real symstore. Exclude any symstore admin files
foreach (FileReference File in DirectoryReference.EnumerateFiles(TempSymStoreDir, "*.*", SearchOption.AllDirectories).Where(File => File.HasExtension(".dl_") || File.HasExtension(".ex_") || File.HasExtension(".pd_")))
{
string RelativePath = File.MakeRelativeTo(DirectoryReference.Combine(TempSymStoreDir));
FileReference ActualDestinationFile = FileReference.Combine(SymbolStoreDirectory, RelativePath);
// Try and add a version file. Do this before checking to see if the symbol is there already in the case of exact matches (multiple builds could use the same pdb, for example)
if (!string.IsNullOrWhiteSpace(BuildVersion))
{
FileReference BuildVersionFile = FileReference.Combine(ActualDestinationFile.Directory, string.Format("{0}.version", BuildVersion));
// Attempt to create the file. Just continue if it fails.
try
{
DirectoryReference.CreateDirectory(BuildVersionFile.Directory);
FileReference.WriteAllText(BuildVersionFile, string.Empty);
}
catch (Exception Ex)
{
LogWarning("Failed to write the version file, reason {0}", Ex.ToString());
}
}
// Don't bother copying the temp file if the destination file is there already.
if (FileReference.Exists(ActualDestinationFile))
{
LogInformation("Destination file {0} already exists, skipping", ActualDestinationFile.FullName);
continue;
}
FileReference TempDestinationFile = new FileReference(ActualDestinationFile.FullName + Guid.NewGuid().ToString());
try
{
CommandUtils.CopyFile(File.FullName, TempDestinationFile.FullName);
}
catch (Exception Ex)
{
throw new AutomationException("Couldn't copy the symbol file to the temp store! Reason: {0}", Ex.ToString());
}
// Move the file in the temp store over.
try
{
FileReference.Move(TempDestinationFile, ActualDestinationFile);
}
catch (Exception Ex)
{
// If the file is there already, it was likely either copied elsewhere (and this is an ioexception) or it had a file handle open already.
// Either way, it's fine to just continue on.
if (FileReference.Exists(ActualDestinationFile))
{
LogInformation("Destination file {0} already exists or was in use, skipping.", ActualDestinationFile.FullName);
continue;
}
// If it doesn't exist, we actually failed to copy it entirely.
else
{
LogWarning("Couldn't move temp file {0} to the symbol store at location {1}! Reason: {2}", TempDestinationFile.FullName, ActualDestinationFile.FullName, Ex.ToString());
}
}
// Delete the temp one no matter what, don't want them hanging around in the symstore
finally
{
FileReference.Delete(TempDestinationFile);
}
}
LogInformation("Took {0}s to copy the symbol files to the store", (DateTime.Now - CompressDone).TotalSeconds);
}
return(true);
}
/// <summary>
/// Reads an exported XGE task list
/// </summary>
/// <param name="TaskListFile">File to read from</param>
/// <param name="FileToEnvironment">Mapping from source file to compile environment</param>
public static void ReadTaskList(FileReference TaskListFile, DirectoryReference BaseDir, out Dictionary <FileReference, CompileEnvironment> FileToEnvironment)
{
XmlDocument Document = new XmlDocument();
Document.Load(TaskListFile.FullName);
// Read the standard include paths from the INCLUDE environment variable in the script
List <DirectoryReference> StandardIncludePaths = new List <DirectoryReference>();
foreach (XmlNode Node in Document.SelectNodes("BuildSet/Environments/Environment/Variables/Variable"))
{
XmlAttribute NameAttr = Node.Attributes["Name"];
if (NameAttr != null && String.Compare(NameAttr.InnerText, "INCLUDE") == 0)
{
foreach (string IncludePath in Node.Attributes["Value"].InnerText.Split(';'))
{
StandardIncludePaths.Add(new DirectoryReference(IncludePath));
}
}
}
// Read all the individual compiles
Dictionary <FileReference, CompileEnvironment> NewFileToEnvironment = new Dictionary <FileReference, CompileEnvironment>();
foreach (XmlNode Node in Document.SelectNodes("BuildSet/Environments/Environment/Tools/Tool"))
{
XmlAttribute ToolPathAttr = Node.Attributes["Path"];
if (ToolPathAttr != null)
{
// Get the full path to the tool
FileReference ToolLocation = new FileReference(ToolPathAttr.InnerText);
// Get the compiler type
CompilerType CompilerType;
if (GetCompilerType(ToolLocation, out CompilerType))
{
string Name = Node.Attributes["Name"].InnerText;
string Params = Node.Attributes["Params"].InnerText;
// Construct the compile environment
CompileEnvironment Environment = new CompileEnvironment(ToolLocation, CompilerType);
// Parse a list of tokens
List <string> Tokens = new List <string>();
ParseArgumentTokens(CompilerType, Params, Tokens);
// Parse it into a list of options, removing any that don't apply
List <FileReference> SourceFiles = new List <FileReference>();
List <FileReference> OutputFiles = new List <FileReference>();
for (int Idx = 0; Idx < Tokens.Count; Idx++)
{
if (Tokens[Idx] == "/Fo" || Tokens[Idx] == "/Fp" || Tokens[Idx] == "-o")
{
OutputFiles.Add(new FileReference(Tokens[++Idx]));
}
else if (Tokens[Idx].StartsWith("/Fo") || Tokens[Idx].StartsWith("/Fp"))
{
OutputFiles.Add(new FileReference(Tokens[Idx].Substring(3)));
}
else if (Tokens[Idx] == "/D" || Tokens[Idx] == "-D")
{
Environment.Definitions.Add(Tokens[++Idx]);
}
else if (Tokens[Idx].StartsWith("/D"))
{
Environment.Definitions.Add(Tokens[Idx].Substring(2));
}
else if (Tokens[Idx] == "/I" || Tokens[Idx] == "-I")
{
Environment.IncludePaths.Add(new DirectoryReference(DirectoryReference.Combine(BaseDir, Tokens[++Idx].Replace("//", "/")).FullName.ToLowerInvariant()));
}
else if (Tokens[Idx].StartsWith("-I"))
{
Environment.IncludePaths.Add(new DirectoryReference(DirectoryReference.Combine(BaseDir, Tokens[Idx].Substring(2).Replace("//", "/")).FullName.ToLowerInvariant()));
}
else if (Tokens[Idx].StartsWith("/FI"))
{
Environment.ForceIncludeFiles.Add(new FileReference(Tokens[Idx].Substring(2)));
}
else if (Tokens[Idx] == "-include")
{
Environment.ForceIncludeFiles.Add(new FileReference(Tokens[++Idx]));
}
else if (Tokens[Idx] == "-Xclang" || Tokens[Idx] == "-target" || Tokens[Idx] == "--target" || Tokens[Idx] == "-x" || Tokens[Idx] == "-o")
{
Environment.Options.Add(new CompileOption(Tokens[Idx], Tokens[++Idx]));
}
else if (Tokens[Idx].StartsWith("/") || Tokens[Idx].StartsWith("-"))
{
Environment.Options.Add(new CompileOption(Tokens[Idx], null));
}
else
{
SourceFiles.Add(FileReference.Combine(BaseDir, Tokens[Idx]));
}
}
// Make sure we're not compiling a precompiled header
if (!OutputFiles.Any(x => x.HasExtension(".gch")) && !Environment.Options.Any(x => x.Name.StartsWith("/Yc")))
{
// Add all the standard include paths
Environment.IncludePaths.AddRange(StandardIncludePaths);
// Add to the output map if there are any source files. Use the extension to distinguish between a source file and linker invocation on Clang.
foreach (FileReference SourceFile in SourceFiles)
{
if (!SourceFile.HasExtension(".a"))
{
NewFileToEnvironment.Add(SourceFile, Environment);
}
}
}
}
}
}
FileToEnvironment = NewFileToEnvironment;
}
/// <summary>
/// Gather compile time telemetry for the given files
/// </summary>
/// <param name="FileToCompileEnvironment">Mapping of source file to the environment used to compile it</param>
/// <param name="WorkingDir">The working directory for output files</param>
/// <param name="NumSamples">Number of samples to take</param>
/// <param name="MaxParallel">Maximum number of tasks to run in parallel.</param>
/// <param name="Log">Log writer</param>
public static void Generate(DirectoryReference InputDir, DirectoryReference WorkingDir, Dictionary <SourceFile, CompileEnvironment> FileToCompileEnvironment, int NumSamples, int Shard, int NumShards, int MaxParallel, TextWriter Log)
{
Stopwatch Timer = Stopwatch.StartNew();
// Create an intermediate directory
DirectoryReference IntermediateDir = DirectoryReference.Combine(WorkingDir, "Timing");
IntermediateDir.CreateDirectory();
// Map of unique fragment to timing data
Dictionary <SourceFragment, FragmentTimingData> FragmentToTimingData = new Dictionary <SourceFragment, FragmentTimingData>();
// Map of unique fragment key to timing data
Dictionary <string, FragmentTimingData> DigestToTimingData = new Dictionary <string, FragmentTimingData>();
// List of all the sequences to time
HashSet <string> UniqueNames = new HashSet <string>();
foreach (KeyValuePair <SourceFile, CompileEnvironment> Pair in FileToCompileEnvironment)
{
// Find all the fragments in this file
List <SourceFragment> Fragments = new List <SourceFragment>();
List <Tuple <int, SourceFile> > IncludeHistory = new List <Tuple <int, SourceFile> >();
Pair.Key.FindIncludedFragments(Fragments, IncludeHistory, new HashSet <SourceFile>());
// Create a sequence for each unique fragment
FragmentTimingData PrevTimingData = null;
for (int Idx = 0; Idx < Fragments.Count; Idx++)
{
FragmentTimingData TimingData = null;
if (!FragmentToTimingData.ContainsKey(Fragments[Idx]) || (Idx + 1 < Fragments.Count && !FragmentToTimingData.ContainsKey(Fragments[Idx + 1])))
{
// Create a sequence for this fragment
SourceFragment LastFragment = Fragments[Idx];
// Create a unique key for this sequence by concatenating all the fragment names
string Digest = Utility.ComputeDigest(String.Join("\n", Fragments.Take(Idx + 1).Select(x => x.Location.FullName)));
// Try to get an existing sequence for this key, otherwise create a new one;
if (!DigestToTimingData.TryGetValue(Digest, out TimingData))
{
// Find a unique name for this sequence
string UniqueName = LastFragment.Location.GetFileName();
for (int NameIdx = 2; !UniqueNames.Add(UniqueName); NameIdx++)
{
UniqueName = String.Format("{0}_{1}{2}", LastFragment.Location.GetFileNameWithoutExtension(), NameIdx, LastFragment.Location.GetExtension());
}
// Add the object for this sequence
FileReference IntermediateFile = FileReference.Combine(IntermediateDir, UniqueName);
TimingData = new FragmentTimingData(UniqueName, Digest, PrevTimingData, Fragments.Take(Idx + 1).ToArray(), IncludeHistory, IntermediateFile, Pair.Value);
DigestToTimingData.Add(Digest, TimingData);
}
// Add it to the unique mapping of fragments
if (!FragmentToTimingData.ContainsKey(LastFragment))
{
FragmentToTimingData[LastFragment] = TimingData;
}
}
PrevTimingData = TimingData;
}
}
// Read any existing shard timing data in the output folder
foreach (FileReference IntermediateFile in IntermediateDir.EnumerateFileReferences("*.csv"))
{
string[] Lines = File.ReadAllLines(IntermediateFile.FullName);
foreach (string Line in Lines.Skip(1))
{
string[] Tokens = Line.Split(',');
if (Tokens.Length == 5)
{
FragmentTimingData TimingData;
if (DigestToTimingData.TryGetValue(Tokens[1], out TimingData) && TimingData.Samples.Count < NumSamples)
{
FragmentTimingSample Sample = new FragmentTimingSample();
Sample.TotalTime = Double.Parse(Tokens[2]);
Sample.FrontendTime = Double.Parse(Tokens[3]);
Sample.BackendTime = Double.Parse(Tokens[4]);
TimingData.Samples.Add(Sample);
}
}
}
}
// Find all the remaining fragments, and repeat each one by the number of times it has to be executed
List <FragmentTimingData> FilteredFragments = DigestToTimingData.Values.ToList();
FilteredFragments.RemoveAll(x => (int)(Math.Abs((long)x.Digest.GetHashCode()) % NumShards) != (Shard - 1));
// Get the initial number of compile times for each fragment. We avoid saving before this number.
List <int> InitialCompileCount = FilteredFragments.Select(x => x.Samples.Count).ToList();
// Create all the actions to execute
List <Action> Actions = new List <Action>();
foreach (FragmentTimingData Fragment in FilteredFragments)
{
FragmentTimingData FragmentCopy = Fragment;
for (int SampleIdx = Fragment.Samples.Count; SampleIdx < NumSamples; SampleIdx++)
{
int SampleIdxCopy = SampleIdx;
Actions.Add(() => FragmentCopy.Compile(IntermediateDir, SampleIdxCopy));
}
}
// Randomize the order to ensure that compile times are not consistently affected by other files being compiled simultaneously.
Random Random = new Random();
Actions = Actions.OrderBy(x => Random.Next()).ToList();
// Compile them all
if (Actions.Count > 0)
{
Utility.ParallelForWithStatus("Compiling fragments...", 0, Actions.Count, new ParallelOptions {
MaxDegreeOfParallelism = MaxParallel
}, Idx => Actions[Idx](), Log);
}
// Write out the results
if (NumShards > 1)
{
// If we're running a sharded build, write out intermediate files containing the results
FileReference OutputFile = FileReference.Combine(IntermediateDir, String.Format("Shard{0}.csv", Shard));
using (StreamWriter Writer = new StreamWriter(OutputFile.FullName))
{
Writer.WriteLine("Name,Digest,TotalTime,FrontendTime,BackendTime");
for (int Idx = 0; Idx < FilteredFragments.Count; Idx++)
{
FragmentTimingData FilteredFragment = FilteredFragments[Idx];
for (int SampleIdx = InitialCompileCount[Idx]; SampleIdx < FilteredFragment.Samples.Count; SampleIdx++)
{
FragmentTimingSample Sample = FilteredFragment.Samples[SampleIdx];
Writer.WriteLine("{0},{1},{2},{3},{4}", FilteredFragment.UniqueName, FilteredFragment.Digest, Sample.TotalTime, Sample.FrontendTime, Sample.BackendTime);
}
}
}
}
else
{
// Write out the fragment report
FileReference FragmentReport = FileReference.Combine(WorkingDir, "Timing.csv");
Log.WriteLine("Writing {0}...", FragmentReport);
using (StreamWriter Writer = new StreamWriter(FragmentReport.FullName))
{
// Write the header
Writer.Write("Fragment,MinLine,MaxLine");
// Write the labels for each sample type
string[] Types = new string[] { "Total", "Frontend", "Backend" };
for (int Idx = 0; Idx < Types.Length; Idx++)
{
for (int SampleIdx = 0; SampleIdx < NumSamples; SampleIdx++)
{
Writer.Write(",{0}{1}", Types[Idx], SampleIdx + 1);
}
Writer.Write(",{0}Min,{0}Max,{0}Avg,{0}Exc", Types[Idx]);
}
Writer.WriteLine();
// Write all the results
Func <FragmentTimingSample, double>[] TimeFieldDelegates = new Func <FragmentTimingSample, double>[] { x => x.TotalTime, x => x.FrontendTime, x => x.BackendTime };
foreach (FragmentTimingData TimingData in FragmentToTimingData.Values)
{
Writer.Write("{0},{1},{2}", TimingData.Fragment.Location.GetFileName(), TimingData.Fragment.MarkupMin + 1, TimingData.Fragment.MarkupMax + 1);
foreach (Func <FragmentTimingSample, double> TimeFieldDelegate in TimeFieldDelegates)
{
foreach (FragmentTimingSample Sample in TimingData.Samples)
{
Writer.Write(",{0:0.000}", TimeFieldDelegate(Sample));
}
Writer.Write(",{0:0.000}", TimingData.Samples.Min(x => TimeFieldDelegate(x)));
Writer.Write(",{0:0.000}", TimingData.Samples.Max(x => TimeFieldDelegate(x)));
Writer.Write(",{0:0.000}", TimingData.Samples.Average(x => TimeFieldDelegate(x)));
if (TimingData.PrevFragmentData == null)
{
Writer.Write(",{0:0.000}", TimingData.Samples.Average(x => TimeFieldDelegate(x)));
}
else
{
Writer.Write(",{0:0.000}", TimingData.Samples.Average(x => TimeFieldDelegate(x)) - TimingData.PrevFragmentData.Samples.Average(x => TimeFieldDelegate(x)));
}
}
Writer.WriteLine();
}
}
}
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
public override void Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Get the project path, and check it exists
FileReference ProjectFile = Parameters.Project;
if (Parameters.Project != null && !FileReference.Exists(ProjectFile))
{
throw new AutomationException("Couldn't find project '{0}'", ProjectFile.FullName);
}
// Get the directories used for staging this project
DirectoryReference SourceEngineDir = CommandUtils.EngineDirectory;
DirectoryReference SourceProjectDir = (ProjectFile == null)? SourceEngineDir : ProjectFile.Directory;
// Get the output directories. We flatten the directory structure on output.
DirectoryReference TargetDir = Parameters.ToDir;
DirectoryReference TargetEngineDir = DirectoryReference.Combine(TargetDir, "Engine");
DirectoryReference TargetProjectDir = DirectoryReference.Combine(TargetDir, ProjectFile.GetFileNameWithoutExtension());
// Get the path to the receipt
FileReference ReceiptFileName = TargetReceipt.GetDefaultPath(SourceProjectDir, Parameters.Target, Parameters.Platform, Parameters.Configuration, Parameters.Architecture);
// Try to load it
TargetReceipt Receipt;
if (!TargetReceipt.TryRead(ReceiptFileName, SourceEngineDir, SourceProjectDir, out Receipt))
{
throw new AutomationException("Couldn't read receipt '{0}'", ReceiptFileName);
}
// Stage all the build products needed at runtime
HashSet <FileReference> SourceFiles = new HashSet <FileReference>();
foreach (BuildProduct BuildProduct in Receipt.BuildProducts)
{
SourceFiles.Add(BuildProduct.Path);
}
foreach (RuntimeDependency RuntimeDependency in Receipt.RuntimeDependencies.Where(x => x.Type != StagedFileType.UFS))
{
SourceFiles.Add(RuntimeDependency.Path);
}
// Get all the target files
List <FileReference> TargetFiles = new List <FileReference>();
foreach (FileReference SourceFile in SourceFiles)
{
// Get the destination file to copy to, mapping to the new engine and project directories as appropriate
FileReference TargetFile;
if (SourceFile.IsUnderDirectory(SourceEngineDir))
{
TargetFile = FileReference.Combine(TargetEngineDir, SourceFile.MakeRelativeTo(SourceEngineDir));
}
else
{
TargetFile = FileReference.Combine(TargetProjectDir, SourceFile.MakeRelativeTo(SourceProjectDir));
}
// Fixup the case of the output file. Would expect Platform.DeployLowerCaseFilenames() to return true here, but seems not to be the case.
if (Parameters.Platform == UnrealTargetPlatform.PS4)
{
TargetFile = FileReference.Combine(TargetDir, TargetFile.MakeRelativeTo(TargetDir).ToLowerInvariant());
}
// Only copy the output file if it doesn't already exist. We can stage multiple targets to the same output directory.
if (Parameters.Overwrite || !FileReference.Exists(TargetFile))
{
DirectoryReference.CreateDirectory(TargetFile.Directory);
CommandUtils.CopyFile(SourceFile.FullName, TargetFile.FullName);
// Force all destination files to not readonly.
CommandUtils.SetFileAttributes(TargetFile.FullName, ReadOnly: false);
}
// Add it to the list of target files
TargetFiles.Add(TargetFile);
}
// Apply the optional tag to the build products
foreach (string TagName in FindTagNamesFromList(Parameters.Tag))
{
FindOrAddTagSet(TagNameToFileSet, TagName).UnionWith(TargetFiles);
}
// Add the target file to the list of build products
BuildProducts.UnionWith(TargetFiles);
}
/// <summary>
/// Main entry point for the BuildGraph command
/// </summary>
public override ExitCode Execute()
{
// Parse the command line parameters
string ScriptFileName = ParseParamValue("Script", null);
string TargetNames = ParseParamValue("Target", null);
string DocumentationFileName = ParseParamValue("Documentation", null);
string SchemaFileName = ParseParamValue("Schema", null);
string ExportFileName = ParseParamValue("Export", null);
string PreprocessedFileName = ParseParamValue("Preprocess", null);
string SharedStorageDir = ParseParamValue("SharedStorageDir", null);
string SingleNodeName = ParseParamValue("SingleNode", null);
string TriggerName = ParseParamValue("Trigger", null);
string[] SkipTriggerNames = ParseParamValue("SkipTrigger", "").Split(new char[] { '+', ';' }, StringSplitOptions.RemoveEmptyEntries).ToArray();
bool bSkipTriggers = ParseParam("SkipTriggers");
string TokenSignature = ParseParamValue("TokenSignature", null);
bool bSkipTargetsWithoutTokens = ParseParam("SkipTargetsWithoutTokens");
bool bResume = SingleNodeName != null || ParseParam("Resume");
bool bListOnly = ParseParam("ListOnly");
bool bWriteToSharedStorage = ParseParam("WriteToSharedStorage") || CommandUtils.IsBuildMachine;
bool bPublicTasksOnly = ParseParam("PublicTasksOnly");
string ReportName = ParseParamValue("ReportName", null);
GraphPrintOptions PrintOptions = GraphPrintOptions.ShowCommandLineOptions;
if (ParseParam("ShowDeps"))
{
PrintOptions |= GraphPrintOptions.ShowDependencies;
}
if (ParseParam("ShowNotifications"))
{
PrintOptions |= GraphPrintOptions.ShowNotifications;
}
// Parse any specific nodes to clean
List <string> CleanNodes = new List <string>();
foreach (string NodeList in ParseParamValues("CleanNode"))
{
foreach (string NodeName in NodeList.Split('+', ';'))
{
CleanNodes.Add(NodeName);
}
}
// Set up the standard properties which build scripts might need
Dictionary <string, string> DefaultProperties = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase);
DefaultProperties["Branch"] = P4Enabled ? P4Env.Branch : "Unknown";
DefaultProperties["EscapedBranch"] = P4Enabled ? CommandUtils.EscapePath(P4Env.Branch) : "Unknown";
DefaultProperties["Change"] = P4Enabled ? P4Env.Changelist.ToString() : "0";
DefaultProperties["CodeChange"] = P4Enabled ? P4Env.CodeChangelist.ToString() : "0";
DefaultProperties["RootDir"] = CommandUtils.RootDirectory.FullName;
DefaultProperties["IsBuildMachine"] = IsBuildMachine ? "true" : "false";
DefaultProperties["HostPlatform"] = HostPlatform.Current.HostEditorPlatform.ToString();
DefaultProperties["RestrictedFolderNames"] = String.Join(";", PlatformExports.RestrictedFolderNames.Select(x => x.DisplayName));
DefaultProperties["RestrictedFolderFilter"] = String.Join(";", PlatformExports.RestrictedFolderNames.Select(x => String.Format(".../{0}/...", x.DisplayName)));
// Attempt to read existing Build Version information
BuildVersion Version;
if (BuildVersion.TryRead(BuildVersion.GetDefaultFileName(), out Version))
{
DefaultProperties["EngineMajorVersion"] = Version.MajorVersion.ToString();
DefaultProperties["EngineMinorVersion"] = Version.MinorVersion.ToString();
DefaultProperties["EnginePatchVersion"] = Version.PatchVersion.ToString();
}
// Add any additional custom arguments from the command line (of the form -Set:X=Y)
Dictionary <string, string> Arguments = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase);
foreach (string Param in Params)
{
const string Prefix = "set:";
if (Param.StartsWith(Prefix, StringComparison.InvariantCultureIgnoreCase))
{
int EqualsIdx = Param.IndexOf('=');
if (EqualsIdx >= 0)
{
Arguments[Param.Substring(Prefix.Length, EqualsIdx - Prefix.Length)] = Param.Substring(EqualsIdx + 1);
}
else
{
LogWarning("Missing value for '{0}'", Param.Substring(Prefix.Length));
}
}
}
// Find all the tasks from the loaded assemblies
Dictionary <string, ScriptTask> NameToTask = new Dictionary <string, ScriptTask>();
if (!FindAvailableTasks(NameToTask, bPublicTasksOnly))
{
return(ExitCode.Error_Unknown);
}
// Generate documentation
if (DocumentationFileName != null)
{
GenerateDocumentation(NameToTask, new FileReference(DocumentationFileName));
return(ExitCode.Success);
}
// Create a schema for the given tasks
ScriptSchema Schema = new ScriptSchema(NameToTask);
if (SchemaFileName != null)
{
FileReference FullSchemaFileName = new FileReference(SchemaFileName);
Log("Writing schema to {0}...", FullSchemaFileName.FullName);
Schema.Export(FullSchemaFileName);
if (ScriptFileName == null)
{
return(ExitCode.Success);
}
}
// Check there was a script specified
if (ScriptFileName == null)
{
LogError("Missing -Script= parameter for BuildGraph");
return(ExitCode.Error_Unknown);
}
// Read the script from disk
Graph Graph;
if (!ScriptReader.TryRead(new FileReference(ScriptFileName), Arguments, DefaultProperties, Schema, out Graph))
{
return(ExitCode.Error_Unknown);
}
// Create the temp storage handler
DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);
TempStorage Storage = new TempStorage(RootDir, DirectoryReference.Combine(RootDir, "Engine", "Saved", "BuildGraph"), (SharedStorageDir == null)? null : new DirectoryReference(SharedStorageDir), bWriteToSharedStorage);
if (!bResume)
{
Storage.CleanLocal();
}
foreach (string CleanNode in CleanNodes)
{
Storage.CleanLocalNode(CleanNode);
}
// Convert the supplied target references into nodes
HashSet <Node> TargetNodes = new HashSet <Node>();
if (TargetNames == null)
{
if (!bListOnly)
{
LogError("Missing -Target= parameter for BuildGraph");
return(ExitCode.Error_Unknown);
}
TargetNodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes));
}
else
{
foreach (string TargetName in TargetNames.Split(new char[] { '+', ';' }, StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim()))
{
Node[] Nodes;
if (!Graph.TryResolveReference(TargetName, out Nodes))
{
LogError("Target '{0}' is not in graph", TargetName);
return(ExitCode.Error_Unknown);
}
TargetNodes.UnionWith(Nodes);
}
}
// Try to acquire tokens for all the target nodes we want to build
if (TokenSignature != null)
{
// Find all the lock files
HashSet <FileReference> RequiredTokens = new HashSet <FileReference>(TargetNodes.SelectMany(x => x.RequiredTokens));
// List out all the required tokens
if (SingleNodeName == null)
{
CommandUtils.Log("Required tokens:");
foreach (Node Node in TargetNodes)
{
foreach (FileReference RequiredToken in Node.RequiredTokens)
{
CommandUtils.Log(" '{0}' requires {1}", Node, RequiredToken);
}
}
}
// Try to create all the lock files
List <FileReference> CreatedTokens = new List <FileReference>();
if (!bListOnly)
{
CreatedTokens.AddRange(RequiredTokens.Where(x => WriteTokenFile(x, TokenSignature)));
}
// Find all the tokens that we don't have
Dictionary <FileReference, string> MissingTokens = new Dictionary <FileReference, string>();
foreach (FileReference RequiredToken in RequiredTokens)
{
string CurrentOwner = ReadTokenFile(RequiredToken);
if (CurrentOwner != null && CurrentOwner != TokenSignature)
{
MissingTokens.Add(RequiredToken, CurrentOwner);
}
}
// If we want to skip all the nodes with missing locks, adjust the target nodes to account for it
if (MissingTokens.Count > 0)
{
if (bSkipTargetsWithoutTokens)
{
// Find all the nodes we're going to skip
HashSet <Node> SkipNodes = new HashSet <Node>();
foreach (IGrouping <string, FileReference> MissingTokensForBuild in MissingTokens.GroupBy(x => x.Value, x => x.Key))
{
Log("Skipping the following nodes due to {0}:", MissingTokensForBuild.Key);
foreach (FileReference MissingToken in MissingTokensForBuild)
{
foreach (Node SkipNode in TargetNodes.Where(x => x.RequiredTokens.Contains(MissingToken) && SkipNodes.Add(x)))
{
Log(" {0}", SkipNode);
}
}
}
// Write a list of everything left over
if (SkipNodes.Count > 0)
{
TargetNodes.ExceptWith(SkipNodes);
Log("Remaining target nodes:");
foreach (Node TargetNode in TargetNodes)
{
Log(" {0}", TargetNode);
}
if (TargetNodes.Count == 0)
{
Log(" None.");
}
}
}
else
{
foreach (KeyValuePair <FileReference, string> Pair in MissingTokens)
{
List <Node> SkipNodes = TargetNodes.Where(x => x.RequiredTokens.Contains(Pair.Key)).ToList();
LogError("Cannot run {0} due to previous build: {1}", String.Join(", ", SkipNodes), Pair.Value);
}
foreach (FileReference CreatedToken in CreatedTokens)
{
FileReference.Delete(CreatedToken);
}
return(ExitCode.Error_Unknown);
}
}
}
// Cull the graph to include only those nodes
Graph.Select(TargetNodes);
// Collapse any triggers in the graph which are marked to be skipped
HashSet <ManualTrigger> SkipTriggers = new HashSet <ManualTrigger>();
if (bSkipTriggers)
{
SkipTriggers.UnionWith(Graph.NameToTrigger.Values);
}
else
{
foreach (string SkipTriggerName in SkipTriggerNames)
{
ManualTrigger SkipTrigger;
if (!Graph.NameToTrigger.TryGetValue(TriggerName, out SkipTrigger))
{
LogError("Couldn't find trigger '{0}'", TriggerName);
return(ExitCode.Error_Unknown);
}
SkipTriggers.Add(SkipTrigger);
}
}
Graph.SkipTriggers(SkipTriggers);
// If a report for the whole build was requested, insert it into the graph
if (ReportName != null)
{
Report NewReport = new Report(ReportName);
NewReport.Nodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes));
Graph.NameToReport.Add(ReportName, NewReport);
}
// Write out the preprocessed script
if (PreprocessedFileName != null)
{
Graph.Write(new FileReference(PreprocessedFileName), (SchemaFileName != null)? new FileReference(SchemaFileName) : null);
}
// Find the triggers which we are explicitly running.
ManualTrigger Trigger = null;
if (TriggerName != null && !Graph.NameToTrigger.TryGetValue(TriggerName, out Trigger))
{
LogError("Couldn't find trigger '{0}'", TriggerName);
return(ExitCode.Error_Unknown);
}
// If we're just building a single node, find it
Node SingleNode = null;
if (SingleNodeName != null && !Graph.NameToNode.TryGetValue(SingleNodeName, out SingleNode))
{
LogError("Node '{0}' is not in the trimmed graph", SingleNodeName);
return(ExitCode.Error_Unknown);
}
// If we just want to show the contents of the graph, do so and exit.
if (bListOnly)
{
HashSet <Node> CompletedNodes = FindCompletedNodes(Graph, Storage);
Graph.Print(CompletedNodes, PrintOptions);
return(ExitCode.Success);
}
// Print out all the diagnostic messages which still apply, unless we're running a step as part of a build system or just listing the contents of the file.
if (SingleNode == null)
{
IEnumerable <GraphDiagnostic> Diagnostics = Graph.Diagnostics.Where(x => x.EnclosingTrigger == Trigger);
foreach (GraphDiagnostic Diagnostic in Diagnostics)
{
if (Diagnostic.EventType == LogEventType.Warning)
{
CommandUtils.LogWarning(Diagnostic.Message);
}
else
{
CommandUtils.LogError(Diagnostic.Message);
}
}
if (Diagnostics.Any(x => x.EventType == LogEventType.Error))
{
return(ExitCode.Error_Unknown);
}
}
// Execute the command
if (ExportFileName != null)
{
HashSet <Node> CompletedNodes = FindCompletedNodes(Graph, Storage);
Graph.Print(CompletedNodes, PrintOptions);
Graph.Export(new FileReference(ExportFileName), Trigger, CompletedNodes);
}
else if (SingleNode != null)
{
if (!BuildNode(new JobContext(this), Graph, SingleNode, Storage, bWithBanner: true))
{
return(ExitCode.Error_Unknown);
}
}
else
{
if (!BuildAllNodes(new JobContext(this), Graph, Storage))
{
return(ExitCode.Error_Unknown);
}
}
return(ExitCode.Success);
}
/// <summary>
/// Returns a path to the client binaries folder.
/// </summary>
/// <param name="RawProjectPath">Full project path.</param>
/// <param name="Platform">Platform type.</param>
/// <returns>Path to the binaries folder.</returns>
public static DirectoryReference GetProjectClientBinariesFolder(DirectoryReference ProjectClientBinariesPath, UnrealTargetPlatform Platform)
{
ProjectClientBinariesPath = DirectoryReference.Combine(ProjectClientBinariesPath, Platform.ToString());
return(ProjectClientBinariesPath);
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
public override void Execute(JobContext Job, HashSet <FileReference
> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Set the Engine directory
DirectoryReference EngineDir = DirectoryReference.Combine(CommandUtils.RootDirectory, "Engine");
if (!String.IsNullOrEmpty(Parameters.EngineDir))
{
EngineDir = DirectoryReference.Combine(CommandUtils.RootDirectory, Parameters.EngineDir);
}
// Set the Project directory
DirectoryReference ProjectDir = DirectoryReference.Combine(CommandUtils.RootDirectory, "Engine");
if (!String.IsNullOrEmpty(Parameters.ProjectDir))
{
ProjectDir = DirectoryReference.Combine(CommandUtils.RootDirectory, Parameters.ProjectDir);
}
// Resolve the input list
IEnumerable <FileReference> TargetFiles = ResolveFilespec(CommandUtils.RootDirectory, Parameters.Files, TagNameToFileSet);
HashSet <FileReference> Files = new HashSet <FileReference>();
foreach (FileReference TargetFile in TargetFiles)
{
// check all files are .target files
if (TargetFile.GetExtension() != ".target")
{
throw new AutomationException("Invalid file passed to TagReceipt task ({0})", TargetFile.FullName);
}
// Read the receipt
TargetReceipt Receipt;
if (!TargetReceipt.TryRead(TargetFile, EngineDir, ProjectDir, out Receipt))
{
CommandUtils.LogWarning("Unable to load file using TagReceipt task ({0})", TargetFile.FullName);
continue;
}
if (Parameters.BuildProducts)
{
foreach (BuildProduct BuildProduct in Receipt.BuildProducts)
{
if ((String.IsNullOrEmpty(Parameters.BuildProductType) && TargetReceipt.GetStageTypeFromBuildProductType(BuildProduct) == StagedFileType) || BuildProduct.Type == BuildProductType)
{
Files.Add(BuildProduct.Path);
}
}
}
if (Parameters.RuntimeDependencies)
{
foreach (RuntimeDependency RuntimeDependency in Receipt.RuntimeDependencies)
{
if (String.IsNullOrEmpty(Parameters.StagedFileType) || RuntimeDependency.Type == StagedFileType)
{
// Only add files that exist as dependencies are assumed to always exist
FileReference DependencyPath = RuntimeDependency.Path;
if (FileReference.Exists(DependencyPath))
{
Files.Add(DependencyPath);
}
else
{
CommandUtils.LogWarning("File listed as RuntimeDependency in {0} does not exist ({1})", TargetFile.FullName, DependencyPath.FullName);
}
}
}
}
if (Parameters.PrecompiledBuildDependencies)
{
foreach (FileReference PrecompiledBuildDependency in Receipt.PrecompiledBuildDependencies)
{
// Only add files that exist as dependencies are assumed to always exist
FileReference DependencyPath = PrecompiledBuildDependency;
if (FileReference.Exists(DependencyPath))
{
Files.Add(DependencyPath);
}
else
{
CommandUtils.LogWarning("File listed as PrecompiledBuildDependency in {0} does not exist ({1})", TargetFile.FullName, DependencyPath.FullName);
}
}
}
if (Parameters.PrecompiledRuntimeDependencies)
{
foreach (FileReference PrecompiledRuntimeDependency in Receipt.PrecompiledRuntimeDependencies)
{
// Only add files that exist as dependencies are assumed to always exist
FileReference DependencyPath = PrecompiledRuntimeDependency;
if (FileReference.Exists(DependencyPath))
{
Files.Add(DependencyPath);
}
else
{
CommandUtils.LogWarning("File listed as PrecompiledRuntimeDependency in {0} does not exist ({1})", TargetFile.FullName, DependencyPath.FullName);
}
}
}
}
// Apply the tag to all the matching files
FindOrAddTagSet(TagNameToFileSet, Parameters.With).UnionWith(Files);
}
private static void GenerateTempTarget(FileReference RawProjectPath)
{
DirectoryReference TempDir = DirectoryReference.Combine(RawProjectPath.Directory, "Intermediate", "Source");
DirectoryReference.CreateDirectory(TempDir);
// Get the project name for use in temporary files
string ProjectName = RawProjectPath.GetFileNameWithoutExtension();
// Create a target.cs file
MemoryStream TargetStream = new MemoryStream();
using (StreamWriter Writer = new StreamWriter(TargetStream))
{
Writer.WriteLine("using UnrealBuildTool;");
Writer.WriteLine();
Writer.WriteLine("public class {0}Target : TargetRules", ProjectName);
Writer.WriteLine("{");
Writer.WriteLine("\tpublic {0}Target(TargetInfo Target) : base(Target)", ProjectName);
Writer.WriteLine("\t{");
Writer.WriteLine("\t\tType = TargetType.Game;");
Writer.WriteLine("\t\tExtraModuleNames.Add(\"{0}\");", ProjectName);
Writer.WriteLine("\t}");
Writer.WriteLine("}");
}
FileReference TargetLocation = FileReference.Combine(TempDir, ProjectName + ".Target.cs");
FileReference.WriteAllBytesIfDifferent(TargetLocation, TargetStream.ToArray());
// Create a build.cs file
MemoryStream ModuleStream = new MemoryStream();
using (StreamWriter Writer = new StreamWriter(ModuleStream))
{
Writer.WriteLine("using UnrealBuildTool;");
Writer.WriteLine();
Writer.WriteLine("public class {0} : ModuleRules", ProjectName);
Writer.WriteLine("{");
Writer.WriteLine("\tpublic {0}(ReadOnlyTargetRules Target) : base(Target)", ProjectName);
Writer.WriteLine("\t{");
Writer.WriteLine("\t\tPCHUsage = PCHUsageMode.UseExplicitOrSharedPCHs;");
Writer.WriteLine();
Writer.WriteLine("\t\tPrivateDependencyModuleNames.Add(\"Core\");");
Writer.WriteLine("\t\tPrivateDependencyModuleNames.Add(\"Core\");");
Writer.WriteLine("\t}");
Writer.WriteLine("}");
}
FileReference ModuleLocation = FileReference.Combine(TempDir, ProjectName + ".Build.cs");
FileReference.WriteAllBytesIfDifferent(ModuleLocation, ModuleStream.ToArray());
// Create a main module cpp file
MemoryStream SourceFileStream = new MemoryStream();
using (StreamWriter Writer = new StreamWriter(SourceFileStream))
{
Writer.WriteLine("#include \"CoreTypes.h\"");
Writer.WriteLine("#include \"Modules/ModuleManager.h\"");
Writer.WriteLine();
Writer.WriteLine("IMPLEMENT_PRIMARY_GAME_MODULE(FDefaultModuleImpl, {0}, \"{0}\");", ProjectName);
}
FileReference SourceFileLocation = FileReference.Combine(TempDir, ProjectName + ".cpp");
FileReference.WriteAllBytesIfDifferent(SourceFileLocation, SourceFileStream.ToArray());
}
/// <summary>
/// Main entry point for the command
/// </summary>
public override void ExecuteBuild()
{
// Build a lookup of flags to set and clear for each identifier
Dictionary <string, int> IdentifierToIndex = new Dictionary <string, int>();
for (int Idx = 0; Idx < MacroPairs.GetLength(0); Idx++)
{
IdentifierToIndex[MacroPairs[Idx, 0]] = Idx;
IdentifierToIndex[MacroPairs[Idx, 1]] = ~Idx;
}
// Check if we want to just parse a single file
string FileParam = ParseParamValue("File");
if (FileParam != null)
{
// Check the file exists
FileReference File = new FileReference(FileParam);
if (!FileReference.Exists(File))
{
throw new AutomationException("File '{0}' does not exist", File);
}
CheckSourceFile(File, IdentifierToIndex, new object());
}
else
{
// Add the additional files to be ignored
foreach (string IgnoreFileName in ParseParamValues("Ignore"))
{
IgnoreFileNames.Add(IgnoreFileName);
}
// Create a list of all the root directories
HashSet <DirectoryReference> RootDirs = new HashSet <DirectoryReference>();
RootDirs.Add(EngineDirectory);
// Add the enterprise directory
DirectoryReference EnterpriseDirectory = DirectoryReference.Combine(RootDirectory, "Enterprise");
if (DirectoryReference.Exists(EnterpriseDirectory))
{
RootDirs.Add(EnterpriseDirectory);
}
// Add the project directories
string[] ProjectParams = ParseParamValues("Project");
foreach (string ProjectParam in ProjectParams)
{
FileReference ProjectFile = new FileReference(ProjectParam);
if (!FileReference.Exists(ProjectFile))
{
throw new AutomationException("Unable to find project '{0}'", ProjectFile);
}
RootDirs.Add(ProjectFile.Directory);
}
// Recurse through the tree
LogInformation("Finding source files...");
List <FileReference> SourceFiles = new List <FileReference>();
using (ThreadPoolWorkQueue Queue = new ThreadPoolWorkQueue())
{
foreach (DirectoryReference RootDir in RootDirs)
{
DirectoryInfo PluginsDir = new DirectoryInfo(Path.Combine(RootDir.FullName, "Plugins"));
if (PluginsDir.Exists)
{
Queue.Enqueue(() => FindSourceFiles(PluginsDir, SourceFiles, Queue));
}
DirectoryInfo SourceDir = new DirectoryInfo(Path.Combine(RootDir.FullName, "Source"));
if (SourceDir.Exists)
{
Queue.Enqueue(() => FindSourceFiles(SourceDir, SourceFiles, Queue));
}
}
Queue.Wait();
}
// Loop through all the source files
using (ThreadPoolWorkQueue Queue = new ThreadPoolWorkQueue())
{
object LogLock = new object();
foreach (FileReference SourceFile in SourceFiles)
{
Queue.Enqueue(() => CheckSourceFile(SourceFile, IdentifierToIndex, LogLock));
}
using (LogStatusScope Scope = new LogStatusScope("Checking source files..."))
{
while (!Queue.Wait(10 * 1000))
{
Scope.SetProgress("{0}/{1}", SourceFiles.Count - Queue.NumRemaining, SourceFiles.Count);
}
}
}
}
}
/// <summary>
/// Gets the directory used to store data for the given node
/// </summary>
/// <param name="BaseDir">A local or shared temp storage root directory.</param>
/// <param name="NodeName">Name of the node</param>
/// <returns>Directory to contain a node's data</returns>
static DirectoryReference GetDirectoryForNode(DirectoryReference BaseDir, string NodeName)
{
return(DirectoryReference.Combine(BaseDir, NodeName));
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
/// <returns>True if the task succeeded</returns>
public override bool Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Set the Engine directory
DirectoryReference EngineDir = DirectoryReference.Combine(CommandUtils.RootDirectory, "Engine");
if (!String.IsNullOrEmpty(Parameters.EngineDir))
{
EngineDir = DirectoryReference.Combine(CommandUtils.RootDirectory, Parameters.EngineDir);
}
// Set the Project directory
DirectoryReference ProjectDir = DirectoryReference.Combine(CommandUtils.RootDirectory, "Engine");
if (!String.IsNullOrEmpty(Parameters.ProjectDir))
{
ProjectDir = DirectoryReference.Combine(CommandUtils.RootDirectory, Parameters.ProjectDir);
}
// Resolve the input list
IEnumerable <FileReference> TargetFiles = ResolveFilespec(CommandUtils.RootDirectory, Parameters.Files, TagNameToFileSet);
HashSet <FileReference> Files = new HashSet <FileReference>();
HashSet <string> WildcardDependencies = new HashSet <string>();
foreach (FileReference TargetFile in TargetFiles)
{
// check all files are .target files
if (TargetFile.GetExtension() != ".target")
{
CommandUtils.LogError("Invalid file passed to TagReceipt task ({0})", TargetFile.FullName);
continue;
}
// Read the receipt
TargetReceipt Receipt;
if (!TargetReceipt.TryRead(TargetFile.FullName, out Receipt))
{
CommandUtils.LogWarning("Unable to load file using TagReceipt task ({0})", TargetFile.FullName);
continue;
}
// Convert the paths to absolute
Receipt.ExpandPathVariables(EngineDir, ProjectDir);
if (Parameters.BuildProducts)
{
foreach (BuildProduct BuildProduct in Receipt.BuildProducts)
{
if (String.IsNullOrEmpty(Parameters.BuildProductType) || BuildProduct.Type == BuildProductType)
{
Files.Add(new FileReference(BuildProduct.Path));
}
}
}
if (Parameters.RuntimeDependencies)
{
foreach (RuntimeDependency RuntimeDependency in Receipt.RuntimeDependencies)
{
if (String.IsNullOrEmpty(Parameters.StagedFileType) || RuntimeDependency.Type == StagedFileType)
{
// If it doesn't contain any wildcards, just add the pattern directly
if (FileFilter.FindWildcardIndex(RuntimeDependency.Path) == -1)
{
// Only add files that exist as dependencies are assumed to always exist
FileReference DependencyPath = new FileReference(RuntimeDependency.Path);
if (DependencyPath.Exists())
{
Files.Add(DependencyPath);
}
else
{
// Give a warning about files that don't exist so that we can clean up build.cs files
CommandUtils.LogWarning("File listed as RuntimeDependency in {0} does not exist ({1})", TargetFile.FullName, DependencyPath.FullName);
}
}
else
{
WildcardDependencies.Add(RuntimeDependency.Path);
}
}
}
}
if (Parameters.PrecompiledBuildDependencies)
{
foreach (string PrecompiledBuildDependency in Receipt.PrecompiledBuildDependencies)
{
// If it doesn't contain any wildcards, just add the pattern directly
if (FileFilter.FindWildcardIndex(PrecompiledBuildDependency) == -1)
{
// Only add files that exist as dependencies are assumed to always exist
FileReference DependencyPath = new FileReference(PrecompiledBuildDependency);
if (DependencyPath.Exists())
{
Files.Add(DependencyPath);
}
else
{
// Give a warning about files that don't exist so that we can clean up build.cs files
CommandUtils.LogWarning("File listed as PrecompiledBuildDependency in {0} does not exist ({1})", TargetFile.FullName, DependencyPath.FullName);
}
}
else
{
WildcardDependencies.Add(PrecompiledBuildDependency);
}
}
}
if (Parameters.PrecompiledRuntimeDependencies)
{
foreach (string PrecompiledRuntimeDependency in Receipt.PrecompiledRuntimeDependencies)
{
// If it doesn't contain any wildcards, just add the pattern directly
if (FileFilter.FindWildcardIndex(PrecompiledRuntimeDependency) == -1)
{
// Only add files that exist as dependencies are assumed to always exist
FileReference DependencyPath = new FileReference(PrecompiledRuntimeDependency);
if (DependencyPath.Exists())
{
Files.Add(DependencyPath);
}
else
{
// Give a warning about files that don't exist so that we can clean up build.cs files
CommandUtils.LogWarning("File listed as PrecompiledRuntimeDependency in {0} does not exist ({1})", TargetFile.FullName, DependencyPath.FullName);
}
}
else
{
WildcardDependencies.Add(PrecompiledRuntimeDependency);
}
}
}
}
// Turn any wildcards into a file list
Files.UnionWith(ResolveFilespecWithExcludePatterns(CommandUtils.RootDirectory, WildcardDependencies.ToList(), new List <string>(), TagNameToFileSet));
// Apply the tag to all the matching files
FindOrAddTagSet(TagNameToFileSet, Parameters.With).UnionWith(Files);
return(true);
}
/// <summary>
/// Run the automated tests
/// </summary>
public override void ExecuteBuild()
{
// Get all the shared directories
DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);
DirectoryReference LocalDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Local");
CommandUtils.CreateDirectory_NoExceptions(LocalDir.FullName);
CommandUtils.DeleteDirectoryContents(LocalDir.FullName);
DirectoryReference SharedDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Shared");
CommandUtils.CreateDirectory_NoExceptions(SharedDir.FullName);
CommandUtils.DeleteDirectoryContents(SharedDir.FullName);
DirectoryReference WorkingDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Working");
CommandUtils.CreateDirectory_NoExceptions(WorkingDir.FullName);
CommandUtils.DeleteDirectoryContents(WorkingDir.FullName);
// Create the temp storage object
TempStorage TempStore = new TempStorage(WorkingDir, LocalDir, SharedDir, true);
// Create a working directory, and copy some source files into it
DirectoryReference SourceDir = DirectoryReference.Combine(RootDir, "Engine", "Source", "Runtime");
if (!CommandUtils.CopyDirectory_NoExceptions(SourceDir.FullName, WorkingDir.FullName, true))
{
throw new AutomationException("Couldn't copy {0} to {1}", SourceDir.FullName, WorkingDir.FullName);
}
// Save the default output
Dictionary <FileReference, DateTime> DefaultOutput = SelectFiles(WorkingDir, 'a', 'f');
TempStore.Archive("TestNode", null, DefaultOutput.Keys.ToArray(), false);
Dictionary <FileReference, DateTime> NamedOutput = SelectFiles(WorkingDir, 'g', 'i');
TempStore.Archive("TestNode", "NamedOutput", NamedOutput.Keys.ToArray(), true);
// Check both outputs are still ok
TempStorageManifest DefaultManifest = TempStore.Retreive("TestNode", null);
CheckManifest(WorkingDir, DefaultManifest, DefaultOutput);
TempStorageManifest NamedManifest = TempStore.Retreive("TestNode", "NamedOutput");
CheckManifest(WorkingDir, NamedManifest, NamedOutput);
// Delete local temp storage and the working directory and try again
CommandUtils.Log("Clearing local folders...");
CommandUtils.DeleteDirectoryContents(WorkingDir.FullName);
CommandUtils.DeleteDirectoryContents(LocalDir.FullName);
// First output should fail
CommandUtils.Log("Checking default manifest is now unavailable...");
bool bGotManifest = false;
try
{
TempStore.Retreive("TestNode", null);
}
catch
{
bGotManifest = false;
}
if (bGotManifest)
{
throw new AutomationException("Did not expect shared temp storage manifest to exist");
}
// Second one should be fine
TempStorageManifest NamedManifestFromShared = TempStore.Retreive("TestNode", "NamedOutput");
CheckManifest(WorkingDir, NamedManifestFromShared, NamedOutput);
}
private static bool RequiresTempTarget(FileReference RawProjectPath, List <UnrealTargetPlatform> ClientTargetPlatforms, List <UnrealTargetConfiguration> ClientTargetConfigurations, bool AssetNativizationRequested)
{
// check to see if we already have a Target.cs file
if (File.Exists(Path.Combine(Path.GetDirectoryName(RawProjectPath.FullName), "Source", RawProjectPath.GetFileNameWithoutExtension() + ".Target.cs")))
{
return(false);
}
else if (Directory.Exists(Path.Combine(Path.GetDirectoryName(RawProjectPath.FullName), "Source")))
{
// wasn't one in the main Source directory, let's check all sub-directories
//@todo: may want to read each target.cs to see if it has a target corresponding to the project name as a final check
FileInfo[] Files = (new DirectoryInfo(Path.Combine(Path.GetDirectoryName(RawProjectPath.FullName), "Source")).GetFiles("*.Target.cs", SearchOption.AllDirectories));
if (Files.Length > 0)
{
return(false);
}
}
//
// once we reach this point, we can surmise that this is an asset-
// only (code free) project
if (AssetNativizationRequested)
{
// we're going to be converting some of the project's assets
// into native code, so we require a distinct target (executable)
// be generated for this project
return(true);
}
if (ClientTargetPlatforms != null)
{
foreach (UnrealTargetPlatform ClientPlatform in ClientTargetPlatforms)
{
EncryptionAndSigning.CryptoSettings Settings = EncryptionAndSigning.ParseCryptoSettings(RawProjectPath.Directory, ClientPlatform);
if (Settings.IsAnyEncryptionEnabled() || Settings.bEnablePakSigning)
{
return(true);
}
}
}
// no Target file, now check to see if build settings have changed
List <UnrealTargetPlatform> TargetPlatforms = ClientTargetPlatforms;
if (ClientTargetPlatforms == null || ClientTargetPlatforms.Count < 1)
{
// No client target platforms, add all in
TargetPlatforms = new List <UnrealTargetPlatform>();
foreach (UnrealTargetPlatform TargetPlatformType in Enum.GetValues(typeof(UnrealTargetPlatform)))
{
if (TargetPlatformType != UnrealTargetPlatform.Unknown)
{
TargetPlatforms.Add(TargetPlatformType);
}
}
}
List <UnrealTargetConfiguration> TargetConfigurations = ClientTargetConfigurations;
if (TargetConfigurations == null || TargetConfigurations.Count < 1)
{
// No client target configurations, add all in
TargetConfigurations = new List <UnrealTargetConfiguration>();
foreach (UnrealTargetConfiguration TargetConfigurationType in Enum.GetValues(typeof(UnrealTargetConfiguration)))
{
if (TargetConfigurationType != UnrealTargetConfiguration.Unknown)
{
TargetConfigurations.Add(TargetConfigurationType);
}
}
}
// Change the working directory to be the Engine/Source folder. We are running from Engine/Binaries/DotNET
DirectoryReference oldCWD = DirectoryReference.GetCurrentDirectory();
try
{
DirectoryReference EngineSourceDirectory = DirectoryReference.Combine(CommandUtils.EngineDirectory, "Source");
if (!DirectoryReference.Exists(EngineSourceDirectory)) // only set the directory if it exists, this should only happen if we are launching the editor from an artist sync
{
EngineSourceDirectory = DirectoryReference.Combine(CommandUtils.EngineDirectory, "Binaries");
}
Directory.SetCurrentDirectory(EngineSourceDirectory.FullName);
// Read the project descriptor, and find all the plugins available to this project
ProjectDescriptor Project = ProjectDescriptor.FromFile(RawProjectPath);
List <PluginInfo> AvailablePlugins = Plugins.ReadAvailablePlugins(CommandUtils.EngineDirectory, RawProjectPath, Project.AdditionalPluginDirectories);
// check the target platforms for any differences in build settings or additional plugins
bool RetVal = false;
foreach (UnrealTargetPlatform TargetPlatformType in TargetPlatforms)
{
if (!Automation.IsEngineInstalled() && !PlatformExports.HasDefaultBuildConfig(RawProjectPath, TargetPlatformType))
{
RetVal = true;
break;
}
// find if there are any plugins enabled or disabled which differ from the default
foreach (PluginInfo Plugin in AvailablePlugins)
{
bool bPluginEnabledForProject = false;
foreach (UnrealTargetConfiguration TargetConfigType in TargetConfigurations)
{
bPluginEnabledForProject |= Plugins.IsPluginEnabledForProject(Plugin, Project, TargetPlatformType, TargetConfigType, TargetRules.TargetType.Game);
}
if ((bPluginEnabledForProject != Plugin.EnabledByDefault) || (bPluginEnabledForProject && Plugin.Descriptor.bInstalled))
{
// NOTE: this code was only marking plugins that compiled for the platform to upgrade to code project, however
// this doesn't work in practice, because the runtime code will look for the plugin, without a .uplugin file,
// and will fail. This is the safest way to make sure all platforms are acting the same. However, if you
// whitelist the plugin in the .uproject file, the above UProjectInfo.IsPluginEnabledForProject check won't pass
// so you won't get in here. Leaving this commented out code in there, because someone is bound to come looking
// for why a non-whitelisted platform module is causing a project to convert to code-based.
// As an aside, if you run the project with UE4Game (not your Project's binary) from the debugger, it will work
// _in this case_ because the .uplugin file will have been staged, and there is no needed library
// if(Plugin.Descriptor.Modules.Any(Module => Module.IsCompiledInConfiguration(TargetPlatformType, TargetType.Game, bBuildDeveloperTools: false, bBuildEditor: false)))
{
RetVal = true;
break;
}
}
}
}
return(RetVal);
}
finally
{
// Change back to the original directory
Directory.SetCurrentDirectory(oldCWD.FullName);
}
}
public override void GetFilesToDeployOrStage(ProjectParams Params, DeploymentContext SC)
{
// Stage all the build products
foreach (StageTarget Target in SC.StageTargets)
{
SC.StageBuildProductsFromReceipt(Target.Receipt, Target.RequireFilesExist, Params.bTreatNonShippingBinariesAsDebugFiles);
}
if (SC.bStageCrashReporter)
{
StagedDirectoryReference CrashReportClientPath = StagedDirectoryReference.Combine("Engine/Binaries", SC.PlatformDir, "CrashReportClient.app");
StageAppBundle(SC, DirectoryReference.Combine(SC.LocalRoot, "Engine/Binaries", SC.PlatformDir, "CrashReportClient.app"), CrashReportClientPath);
}
// Find the app bundle path
List <FileReference> Exes = GetExecutableNames(SC);
foreach (var Exe in Exes)
{
StagedDirectoryReference AppBundlePath = null;
if (Exe.IsUnderDirectory(DirectoryReference.Combine(SC.RuntimeProjectRootDir, "Binaries", SC.PlatformDir)))
{
AppBundlePath = StagedDirectoryReference.Combine(SC.ShortProjectName, "Binaries", SC.PlatformDir, Path.GetFileNameWithoutExtension(Exe.FullName) + ".app");
}
else if (Exe.IsUnderDirectory(DirectoryReference.Combine(SC.RuntimeRootDir, "Engine/Binaries", SC.PlatformDir)))
{
AppBundlePath = StagedDirectoryReference.Combine("Engine/Binaries", SC.PlatformDir, Path.GetFileNameWithoutExtension(Exe.FullName) + ".app");
}
// Copy the custom icon and Steam dylib, if needed
if (AppBundlePath != null)
{
FileReference AppIconsFile = FileReference.Combine(SC.ProjectRoot, "Build", "Mac", "Application.icns");
if (FileReference.Exists(AppIconsFile))
{
SC.StageFile(StagedFileType.NonUFS, AppIconsFile, StagedFileReference.Combine(AppBundlePath, "Contents", "Resources", "Application.icns"));
}
}
}
// Copy the splash screen, Mac specific
FileReference SplashImage = FileReference.Combine(SC.ProjectRoot, "Content", "Splash", "Splash.bmp");
if (FileReference.Exists(SplashImage))
{
SC.StageFile(StagedFileType.NonUFS, SplashImage);
}
// Stage the bootstrap executable
if (!Params.NoBootstrapExe)
{
foreach (StageTarget Target in SC.StageTargets)
{
BuildProduct Executable = Target.Receipt.BuildProducts.FirstOrDefault(x => x.Type == BuildProductType.Executable);
if (Executable != null)
{
// only create bootstraps for executables
List <StagedFileReference> StagedFiles = SC.FilesToStage.NonUFSFiles.Where(x => x.Value == Executable.Path).Select(x => x.Key).ToList();
if (StagedFiles.Count > 0 && Executable.Path.FullName.Replace("\\", "/").Contains("/" + TargetPlatformType.ToString() + "/"))
{
string BootstrapArguments = "";
if (!ShouldStageCommandLine(Params, SC))
{
if (!SC.IsCodeBasedProject)
{
BootstrapArguments = String.Format("../../../{0}/{0}.uproject", SC.ShortProjectName);
}
else
{
BootstrapArguments = SC.ShortProjectName;
}
}
string BootstrapExeName;
if (SC.StageTargetConfigurations.Count > 1)
{
BootstrapExeName = Path.GetFileName(Executable.Path.FullName) + ".app";
}
else if (Params.IsCodeBasedProject)
{
BootstrapExeName = Target.Receipt.TargetName + ".app";
}
else
{
BootstrapExeName = SC.ShortProjectName + ".app";
}
string AppPath = Executable.Path.FullName.Substring(0, Executable.Path.FullName.LastIndexOf(".app/") + 4);
foreach (var DestPath in StagedFiles)
{
string AppRelativePath = DestPath.Name.Substring(0, DestPath.Name.LastIndexOf(".app/") + 4);
StageBootstrapExecutable(SC, BootstrapExeName, AppPath, AppRelativePath, BootstrapArguments);
}
}
}
}
}
// Copy the ShaderCache files, if they exist
FileReference DrawCacheFile = FileReference.Combine(SC.ProjectRoot, "Content", "DrawCache.ushadercache");
if (FileReference.Exists(DrawCacheFile))
{
SC.StageFile(StagedFileType.UFS, DrawCacheFile);
}
FileReference ByteCodeCacheFile = FileReference.Combine(SC.ProjectRoot, "Content", "ByteCodeCache.ushadercode");
if (FileReference.Exists(ByteCodeCacheFile))
{
SC.StageFile(StagedFileType.UFS, ByteCodeCacheFile);
}
{
// Stage any *.metallib files as NonUFS.
// Get the final output directory for cooked data
DirectoryReference CookOutputDir;
if (!String.IsNullOrEmpty(Params.CookOutputDir))
{
CookOutputDir = DirectoryReference.Combine(new DirectoryReference(Params.CookOutputDir), SC.CookPlatform);
}
else if (Params.CookInEditor)
{
CookOutputDir = DirectoryReference.Combine(SC.ProjectRoot, "Saved", "EditorCooked", SC.CookPlatform);
}
else
{
CookOutputDir = DirectoryReference.Combine(SC.ProjectRoot, "Saved", "Cooked", SC.CookPlatform);
}
if (DirectoryReference.Exists(CookOutputDir))
{
List <FileReference> CookedFiles = DirectoryReference.EnumerateFiles(CookOutputDir, "*.metallib", SearchOption.AllDirectories).ToList();
foreach (FileReference CookedFile in CookedFiles)
{
SC.StageFile(StagedFileType.NonUFS, CookedFile, new StagedFileReference(CookedFile.MakeRelativeTo(CookOutputDir)));
}
}
}
}
void StageBootstrapExecutable(DeploymentContext SC, string ExeName, string TargetFile, string StagedRelativeTargetPath, string StagedArguments)
{
DirectoryReference InputApp = DirectoryReference.Combine(SC.LocalRoot, "Engine", "Binaries", SC.PlatformDir, "BootstrapPackagedGame.app");
if (InternalUtils.SafeDirectoryExists(InputApp.FullName))
{
// Create the new bootstrap program
DirectoryReference IntermediateDir = DirectoryReference.Combine(SC.ProjectRoot, "Intermediate", "Staging");
InternalUtils.SafeCreateDirectory(IntermediateDir.FullName);
DirectoryReference IntermediateApp = DirectoryReference.Combine(IntermediateDir, ExeName);
if (DirectoryReference.Exists(IntermediateApp))
{
DirectoryReference.Delete(IntermediateApp, true);
}
CloneDirectory(InputApp.FullName, IntermediateApp.FullName);
// Rename the executable
string GameName = Path.GetFileNameWithoutExtension(ExeName);
FileReference.Move(FileReference.Combine(IntermediateApp, "Contents", "MacOS", "BootstrapPackagedGame"), FileReference.Combine(IntermediateApp, "Contents", "MacOS", GameName));
// Copy the icon
string SrcInfoPlistPath = CombinePaths(TargetFile, "Contents", "Info.plist");
string SrcInfoPlist = File.ReadAllText(SrcInfoPlistPath);
string IconName = GetValueFromInfoPlist(SrcInfoPlist, "CFBundleIconFile");
if (!string.IsNullOrEmpty(IconName))
{
string IconPath = CombinePaths(TargetFile, "Contents", "Resources", IconName + ".icns");
InternalUtils.SafeCreateDirectory(CombinePaths(IntermediateApp.FullName, "Contents", "Resources"));
File.Copy(IconPath, CombinePaths(IntermediateApp.FullName, "Contents", "Resources", IconName + ".icns"));
}
// Update Info.plist contents
string DestInfoPlistPath = CombinePaths(IntermediateApp.FullName, "Contents", "Info.plist");
string DestInfoPlist = File.ReadAllText(DestInfoPlistPath);
string AppIdentifier = GetValueFromInfoPlist(SrcInfoPlist, "CFBundleIdentifier");
if (AppIdentifier == "com.epicgames.UE4Game")
{
AppIdentifier = "";
}
string Copyright = GetValueFromInfoPlist(SrcInfoPlist, "NSHumanReadableCopyright");
string BundleVersion = GetValueFromInfoPlist(SrcInfoPlist, "CFBundleVersion", "1");
string ShortVersion = GetValueFromInfoPlist(SrcInfoPlist, "CFBundleShortVersionString", "1.0");
DestInfoPlist = DestInfoPlist.Replace("com.epicgames.BootstrapPackagedGame", string.IsNullOrEmpty(AppIdentifier) ? "com.epicgames." + GameName + "_bootstrap" : AppIdentifier + "_bootstrap");
DestInfoPlist = DestInfoPlist.Replace("BootstrapPackagedGame", GameName);
DestInfoPlist = DestInfoPlist.Replace("__UE4_ICON_FILE__", IconName);
DestInfoPlist = DestInfoPlist.Replace("__UE4_APP_TO_LAUNCH__", StagedRelativeTargetPath);
DestInfoPlist = DestInfoPlist.Replace("__UE4_COMMANDLINE__", StagedArguments);
DestInfoPlist = DestInfoPlist.Replace("__UE4_COPYRIGHT__", Copyright);
DestInfoPlist = DestInfoPlist.Replace("__UE4_BUNDLE_VERSION__", BundleVersion);
DestInfoPlist = DestInfoPlist.Replace("__UE4_SHORT_VERSION__", ShortVersion);
File.WriteAllText(DestInfoPlistPath, DestInfoPlist);
StageAppBundle(SC, IntermediateApp, new StagedDirectoryReference(ExeName));
}
}
public override void GetFilesToDeployOrStage(ProjectParams Params, DeploymentContext SC)
{
// Engine non-ufs (binaries)
if (SC.bStageCrashReporter)
{
FileReference ReceiptFileName = TargetReceipt.GetDefaultPath(CommandUtils.EngineDirectory, "CrashReportClient", SC.StageTargetPlatform.PlatformType, UnrealTargetConfiguration.Shipping, null);
if (FileReference.Exists(ReceiptFileName))
{
DirectoryReference EngineDir = CommandUtils.EngineDirectory;
DirectoryReference ProjectDir = DirectoryReference.FromFile(Params.RawProjectPath);
TargetReceipt Receipt = TargetReceipt.Read(ReceiptFileName, EngineDir, ProjectDir);
SC.StageBuildProductsFromReceipt(Receipt, true, false);
}
}
// NVCHANGE_BEGIN: Add VXGI
SC.StageFiles(StagedFileType.NonUFS, DirectoryReference.Combine(SC.LocalRoot, "Engine/Binaries/ThirdParty/GameWorks/VXGI"), "*.dll", StageFilesSearch.TopDirectoryOnly);
// NVCHANGE_END: Add VXGI
// NVCHANGE_BEGIN: Add TXAA
SC.StageFiles(StagedFileType.NonUFS, DirectoryReference.Combine(SC.LocalRoot, "Engine/Binaries/ThirdParty/NVIDIA/TXAA"), "*.dll", StageFilesSearch.TopDirectoryOnly);
// NVCHANGE_END: Add TXAA
// NVCHANGE_BEGIN: Add HBAO+
SC.StageFiles(StagedFileType.NonUFS, DirectoryReference.Combine(SC.LocalRoot, "Engine/Binaries/ThirdParty/GameWorks/GFSDK_SSAO"), "*.dll", StageFilesSearch.TopDirectoryOnly);
// NVCHANGE_END: Add HBAO+
// Stage all the build products
foreach (StageTarget Target in SC.StageTargets)
{
SC.StageBuildProductsFromReceipt(Target.Receipt, Target.RequireFilesExist, Params.bTreatNonShippingBinariesAsDebugFiles);
}
// Copy the splash screen, windows specific
FileReference SplashImage = FileReference.Combine(SC.ProjectRoot, "Content", "Splash", "Splash.bmp");
if (FileReference.Exists(SplashImage))
{
SC.StageFile(StagedFileType.NonUFS, SplashImage);
}
// Stage the bootstrap executable
if (!Params.NoBootstrapExe)
{
foreach (StageTarget Target in SC.StageTargets)
{
BuildProduct Executable = Target.Receipt.BuildProducts.FirstOrDefault(x => x.Type == BuildProductType.Executable);
if (Executable != null)
{
// only create bootstraps for executables
List <StagedFileReference> StagedFiles = SC.FilesToStage.NonUFSFiles.Where(x => x.Value == Executable.Path).Select(x => x.Key).ToList();
if (StagedFiles.Count > 0 && Executable.Path.HasExtension(".exe"))
{
string BootstrapArguments = "";
if (!ShouldStageCommandLine(Params, SC))
{
if (!SC.IsCodeBasedProject)
{
BootstrapArguments = String.Format("..\\..\\..\\{0}\\{0}.uproject", SC.ShortProjectName);
}
else
{
BootstrapArguments = SC.ShortProjectName;
}
}
string BootstrapExeName;
if (SC.StageTargetConfigurations.Count > 1)
{
BootstrapExeName = Executable.Path.GetFileName();
}
else if (Params.IsCodeBasedProject)
{
BootstrapExeName = Target.Receipt.TargetName + ".exe";
}
else
{
BootstrapExeName = SC.ShortProjectName + ".exe";
}
foreach (StagedFileReference StagePath in StagedFiles)
{
StageBootstrapExecutable(SC, BootstrapExeName, Executable.Path, StagePath, BootstrapArguments);
}
}
}
}
}
}
