/// <summary> /// Main entry point for the command /// </summary> public override void ExecuteBuild() { string FileSpec = ParseRequiredStringParam("Files"); // Make sure the patterns are a valid syntax if (!FileSpec.StartsWith("//")) { throw new AutomationException("Files must be specified as full depot paths"); } // Pick out the source and target prefixes string Prefix; if (FileSpec.EndsWith("*")) { Prefix = FileSpec.Substring(0, FileSpec.Length - 1); } else if (FileSpec.EndsWith("...")) { Prefix = FileSpec.Substring(0, FileSpec.Length - 3); } else { Prefix = FileSpec; } // Make sure there aren't any other wildcards in the pattern if (Prefix.Contains("?") || Prefix.Contains("*") || Prefix.Contains("...")) { throw new AutomationException("Wildcards are only permitted at the end of filespecs"); } // Find all the source files List <string> SourceFiles = P4.Files(String.Format("-e {0}", FileSpec)); if (SourceFiles.Count == 0) { throw new AutomationException("No files found matching {0}", FileSpec); } SourceFiles.RemoveAll(x => x.StartsWith(Prefix, StringComparison.Ordinal)); // Error if we didn't find anything if (SourceFiles.Count == 0) { throw new AutomationException("No files found matching spec"); } // Find all the target files List <string> TargetFiles = new List <string>(SourceFiles.Count); foreach (string SourceFile in SourceFiles) { if (SourceFile.StartsWith(Prefix, StringComparison.OrdinalIgnoreCase)) { TargetFiles.Add(Prefix + SourceFile.Substring(Prefix.Length)); } else { throw new AutomationException("Source file '{0}' does not start with '{1}'", SourceFile, Prefix); } } // Print what we're going to do LogInformation("Ready to rename {0} files:", SourceFiles.Count); for (int Idx = 0; Idx < SourceFiles.Count; Idx++) { LogInformation("{0,3}: {1}", Idx, SourceFiles[Idx]); LogInformation("{0,3} {1}", "", TargetFiles[Idx]); } // If we're not going through with it, print the renames if (!AllowSubmit) { LogWarning("Skipping due to no -Submit option"); return; } // Force sync all the old files foreach (string OldFile in SourceFiles) { P4.LogP4(String.Format("sync -f {0}", OldFile)); } // Delete all the old files int DeleteChangeNumber = P4.CreateChange(Description: String.Format("Fixing case of {0} (1/2){1}", FileSpec, BoilerplateText)); foreach (string OldFile in SourceFiles) { P4.LogP4(String.Format("delete -k -c {0} {1}", DeleteChangeNumber, OldFile)); } P4.Submit(DeleteChangeNumber); // Re-add all the files in the new location int AddChangeNumber = P4.CreateChange(Description: String.Format("Fixing case of {0} (2/2){1}", FileSpec, BoilerplateText)); foreach (string NewFile in TargetFiles) { P4.LogP4(String.Format("add -c {0} {1}", AddChangeNumber, NewFile)); } P4.Submit(AddChangeNumber); }
public override void ExecuteBuild() { var UEProjectRoot = ParseParamValue("UEProjectRoot"); if (UEProjectRoot == null) { UEProjectRoot = CmdEnv.LocalRoot; } var UEProjectDirectory = ParseParamValue("UEProjectDirectory"); if (UEProjectDirectory == null) { throw new AutomationException("Missing required command line argument: 'UEProjectDirectory'"); } var UEProjectName = ParseParamValue("UEProjectName"); if (UEProjectName == null) { UEProjectName = ""; } var LocalizationProjectNames = new List <string>(); { var LocalizationProjectNamesStr = ParseParamValue("LocalizationProjectNames"); if (LocalizationProjectNamesStr != null) { foreach (var ProjectName in LocalizationProjectNamesStr.Split(',')) { LocalizationProjectNames.Add(ProjectName.Trim()); } } } var LocalizationProviderName = ParseParamValue("LocalizationProvider"); if (LocalizationProviderName == null) { LocalizationProviderName = "OneSky"; } var LocalizationSteps = new List <string>(); { var LocalizationStepsStr = ParseParamValue("LocalizationSteps"); if (LocalizationStepsStr == null) { LocalizationSteps.AddRange(new string[] { "Download", "Gather", "Import", "Export", "Compile", "GenerateReports", "Upload" }); } else { foreach (var StepName in LocalizationStepsStr.Split(',')) { LocalizationSteps.Add(StepName.Trim()); } } LocalizationSteps.Add("Monolithic"); // Always allow the monolithic scripts to run as we don't know which steps they do } var ShouldGatherPlugins = ParseParam("IncludePlugins"); var IncludePlugins = new List <string>(); var ExcludePlugins = new List <string>(); if (ShouldGatherPlugins) { var IncludePluginsStr = ParseParamValue("IncludePlugins"); if (IncludePluginsStr != null) { foreach (var PluginName in IncludePluginsStr.Split(',')) { IncludePlugins.Add(PluginName.Trim()); } } var ExcludePluginsStr = ParseParamValue("ExcludePlugins"); if (ExcludePluginsStr != null) { foreach (var PluginName in ExcludePluginsStr.Split(',')) { ExcludePlugins.Add(PluginName.Trim()); } } } var AdditionalCommandletArguments = ParseParamValue("AdditionalCommandletArguments"); if (AdditionalCommandletArguments == null) { AdditionalCommandletArguments = ""; } var LocalizationBatches = new List <LocalizationBatch>(); // Add the static set of localization projects as a batch if (LocalizationProjectNames.Count > 0) { LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, UEProjectDirectory, "", LocalizationProjectNames)); } // Build up any additional batches needed for plugins if (ShouldGatherPlugins) { var PluginsRootDirectory = CombinePaths(UEProjectRoot, UEProjectDirectory, "Plugins"); IReadOnlyList <PluginInfo> AllPlugins = Plugins.ReadPluginsFromDirectory(new DirectoryReference(PluginsRootDirectory), UEProjectName.Length == 0 ? PluginType.Engine : PluginType.Project); // Add a batch for each plugin that meets our criteria foreach (var PluginInfo in AllPlugins) { bool ShouldIncludePlugin = (IncludePlugins.Count == 0 || IncludePlugins.Contains(PluginInfo.Name)) && !ExcludePlugins.Contains(PluginInfo.Name); if (ShouldIncludePlugin && PluginInfo.Descriptor.LocalizationTargets != null && PluginInfo.Descriptor.LocalizationTargets.Length > 0) { var RootRelativePluginPath = PluginInfo.Directory.MakeRelativeTo(new DirectoryReference(UEProjectRoot)); RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4 var PluginTargetNames = new List <string>(); foreach (var LocalizationTarget in PluginInfo.Descriptor.LocalizationTargets) { PluginTargetNames.Add(LocalizationTarget.Name); } LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, PluginInfo.Name, PluginTargetNames)); } } } // Create a single changelist to use for all changes, and hash the current PO files on disk so we can work out whether they actually change int PendingChangeList = 0; Dictionary <string, byte[]> InitalPOFileHashes = null; if (P4Enabled) { PendingChangeList = P4.CreateChange(P4Env.Client, "Localization Automation"); InitalPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot); } // Process each localization batch foreach (var LocalizationBatch in LocalizationBatches) { ProcessLocalizationProjects(LocalizationBatch, PendingChangeList, UEProjectRoot, UEProjectName, LocalizationProviderName, LocalizationSteps, AdditionalCommandletArguments); } // Clean-up the changelist so it only contains the changed files, and then submit it (if we were asked to) if (P4Enabled) { // Revert any PO files that haven't changed aside from their header { var POFilesToRevert = new List <string>(); var CurrentPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot); foreach (var CurrentPOFileHashPair in CurrentPOFileHashes) { byte[] InitialPOFileHash; if (InitalPOFileHashes.TryGetValue(CurrentPOFileHashPair.Key, out InitialPOFileHash) && InitialPOFileHash.SequenceEqual(CurrentPOFileHashPair.Value)) { POFilesToRevert.Add(CurrentPOFileHashPair.Key); } } if (POFilesToRevert.Count > 0) { var P4RevertArgsFilename = CombinePaths(CmdEnv.LocalRoot, "Engine", "Intermediate", String.Format("LocalizationP4RevertArgs-{0}.txt", Guid.NewGuid().ToString())); using (StreamWriter P4RevertArgsWriter = File.CreateText(P4RevertArgsFilename)) { foreach (var POFileToRevert in POFilesToRevert) { P4RevertArgsWriter.WriteLine(POFileToRevert); } } P4.LogP4(String.Format("-x{0} revert", P4RevertArgsFilename)); DeleteFile_NoExceptions(P4RevertArgsFilename); } } // Revert any other unchanged files P4.RevertUnchanged(PendingChangeList); // Submit that single changelist now if (AllowSubmit) { int SubmittedChangeList; P4.Submit(PendingChangeList, out SubmittedChangeList); } } }
public override void ExecuteBuild() { var UEProjectRoot = ParseParamValue("UEProjectRoot"); if (UEProjectRoot == null) { UEProjectRoot = CmdEnv.LocalRoot; } var UEProjectDirectory = ParseParamValue("UEProjectDirectory"); if (UEProjectDirectory == null) { throw new AutomationException("Missing required command line argument: 'UEProjectDirectory'"); } var UEProjectName = ParseParamValue("UEProjectName"); if (UEProjectName == null) { UEProjectName = ""; } var LocalizationProjectNames = new List <string>(); { var LocalizationProjectNamesStr = ParseParamValue("LocalizationProjectNames"); if (LocalizationProjectNamesStr != null) { foreach (var ProjectName in LocalizationProjectNamesStr.Split(',')) { LocalizationProjectNames.Add(ProjectName.Trim()); } } } var LocalizationProviderName = ParseParamValue("LocalizationProvider"); if (LocalizationProviderName == null) { LocalizationProviderName = ""; } var LocalizationStepNames = new List <string>(); { var LocalizationStepNamesStr = ParseParamValue("LocalizationSteps"); if (LocalizationStepNamesStr == null) { LocalizationStepNames.AddRange(new string[] { "Download", "Gather", "Import", "Export", "Compile", "GenerateReports", "Upload" }); } else { foreach (var StepName in LocalizationStepNamesStr.Split(',')) { LocalizationStepNames.Add(StepName.Trim()); } } LocalizationStepNames.Add("Monolithic"); // Always allow the monolithic scripts to run as we don't know which steps they do } var ShouldGatherPlugins = ParseParam("IncludePlugins"); var IncludePlugins = new List <string>(); var ExcludePlugins = new List <string>(); if (ShouldGatherPlugins) { var IncludePluginsStr = ParseParamValue("IncludePlugins"); if (IncludePluginsStr != null) { foreach (var PluginName in IncludePluginsStr.Split(',')) { IncludePlugins.Add(PluginName.Trim()); } } var ExcludePluginsStr = ParseParamValue("ExcludePlugins"); if (ExcludePluginsStr != null) { foreach (var PluginName in ExcludePluginsStr.Split(',')) { ExcludePlugins.Add(PluginName.Trim()); } } } var ShouldGatherPlatforms = ParseParam("IncludePlatforms"); var AdditionalCommandletArguments = ParseParamValue("AdditionalCommandletArguments"); if (AdditionalCommandletArguments == null) { AdditionalCommandletArguments = ""; } var EnableParallelGather = ParseParam("ParallelGather"); var StartTime = DateTime.UtcNow; var LocalizationBatches = new List <LocalizationBatch>(); // Add the static set of localization projects as a batch if (LocalizationProjectNames.Count > 0) { LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, UEProjectDirectory, "", LocalizationProjectNames)); } // Build up any additional batches needed for platforms if (ShouldGatherPlatforms) { var PlatformsRootDirectory = new DirectoryReference(CombinePaths(UEProjectRoot, UEProjectDirectory, "Platforms")); if (DirectoryReference.Exists(PlatformsRootDirectory)) { foreach (DirectoryReference PlatformDirectory in DirectoryReference.EnumerateDirectories(PlatformsRootDirectory)) { // Find the localization targets defined for this platform var PlatformTargetNames = GetLocalizationTargetsFromDirectory(new DirectoryReference(CombinePaths(PlatformDirectory.FullName, "Config", "Localization"))); if (PlatformTargetNames.Count > 0) { var RootRelativePluginPath = PlatformDirectory.MakeRelativeTo(new DirectoryReference(UEProjectRoot)); RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4 LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, "", PlatformTargetNames)); } } } } // Build up any additional batches needed for plugins if (ShouldGatherPlugins) { var PluginsRootDirectory = new DirectoryReference(CombinePaths(UEProjectRoot, UEProjectDirectory)); IReadOnlyList <PluginInfo> AllPlugins = Plugins.ReadPluginsFromDirectory(PluginsRootDirectory, "Plugins", UEProjectName.Length == 0 ? PluginType.Engine : PluginType.Project); // Add a batch for each plugin that meets our criteria var AvailablePluginNames = new HashSet <string>(); foreach (var PluginInfo in AllPlugins) { AvailablePluginNames.Add(PluginInfo.Name); bool ShouldIncludePlugin = (IncludePlugins.Count == 0 || IncludePlugins.Contains(PluginInfo.Name)) && !ExcludePlugins.Contains(PluginInfo.Name); if (ShouldIncludePlugin && PluginInfo.Descriptor.LocalizationTargets != null && PluginInfo.Descriptor.LocalizationTargets.Length > 0) { var RootRelativePluginPath = PluginInfo.Directory.MakeRelativeTo(new DirectoryReference(UEProjectRoot)); RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4 var PluginTargetNames = new List <string>(); foreach (var LocalizationTarget in PluginInfo.Descriptor.LocalizationTargets) { PluginTargetNames.Add(LocalizationTarget.Name); } LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, PluginInfo.Name, PluginTargetNames)); } } // If we had an explicit list of plugins to include, warn if any were missing foreach (string PluginName in IncludePlugins) { if (!AvailablePluginNames.Contains(PluginName)) { LogWarning("The plugin '{0}' specified by -IncludePlugins wasn't found and will be skipped.", PluginName); } } } // Create a single changelist to use for all changes int PendingChangeList = 0; if (P4Enabled) { var ChangeListCommitMessage = String.Format("Localization Automation using CL {0}", P4Env.Changelist); if (File.Exists(CombinePaths(CmdEnv.LocalRoot, @"Engine/Restricted/NotForLicensees/Build/EpicInternal.txt"))) { ChangeListCommitMessage += "\n#okforgithub ignore"; } PendingChangeList = P4.CreateChange(P4Env.Client, ChangeListCommitMessage); } // Prepare to process each localization batch var LocalizationTasks = new List <LocalizationTask>(); foreach (var LocalizationBatch in LocalizationBatches) { var LocalizationTask = new LocalizationTask(LocalizationBatch, UEProjectRoot, LocalizationProviderName, PendingChangeList, this); LocalizationTasks.Add(LocalizationTask); // Make sure the Localization configs and content is up-to-date to ensure we don't get errors later on if (P4Enabled) { LogInformation("Sync necessary content to head revision"); P4.Sync(P4Env.Branch + "/" + LocalizationTask.Batch.LocalizationTargetDirectory + "/Config/Localization/..."); P4.Sync(P4Env.Branch + "/" + LocalizationTask.Batch.LocalizationTargetDirectory + "/Content/Localization/..."); } // Generate the info we need to gather for each project foreach (var ProjectName in LocalizationTask.Batch.LocalizationProjectNames) { LocalizationTask.ProjectInfos.Add(GenerateProjectInfo(LocalizationTask.RootLocalizationTargetDirectory, ProjectName, LocalizationStepNames)); } } // Hash the current PO files on disk so we can work out whether they actually change Dictionary <string, byte[]> InitalPOFileHashes = null; if (P4Enabled) { InitalPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot); } // Download the latest translations from our localization provider if (LocalizationStepNames.Contains("Download")) { foreach (var LocalizationTask in LocalizationTasks) { if (LocalizationTask.LocProvider != null) { foreach (var ProjectInfo in LocalizationTask.ProjectInfos) { LocalizationTask.LocProvider.DownloadProjectFromLocalizationProvider(ProjectInfo.ProjectName, ProjectInfo.ImportInfo); } } } } // Begin the gather command for each task // These can run in parallel when ParallelGather is enabled { var EditorExe = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/Win64/UE4Editor-Cmd.exe"); // Set the common basic editor arguments var EditorArguments = P4Enabled ? String.Format("-SCCProvider=Perforce -P4Port={0} -P4User={1} -P4Client={2} -P4Passwd={3} -P4Changelist={4} -EnableSCC -DisableSCCSubmit", P4Env.ServerAndPort, P4Env.User, P4Env.Client, P4.GetAuthenticationToken(), PendingChangeList) : "-SCCProvider=None"; if (IsBuildMachine) { EditorArguments += " -BuildMachine"; } EditorArguments += " -Unattended -LogLocalizationConflicts"; if (EnableParallelGather) { EditorArguments += " -multiprocess"; } if (!String.IsNullOrEmpty(AdditionalCommandletArguments)) { EditorArguments += " " + AdditionalCommandletArguments; } // Set the common process run options var CommandletRunOptions = ERunOptions.Default | ERunOptions.NoLoggingOfRunCommand; // Disable logging of the run command as it will print the exit code which GUBP can pick up as an error (we do that ourselves later) if (EnableParallelGather) { CommandletRunOptions |= ERunOptions.NoWaitForExit; } foreach (var LocalizationTask in LocalizationTasks) { var ProjectArgument = String.IsNullOrEmpty(UEProjectName) ? "" : String.Format("\"{0}\"", Path.Combine(LocalizationTask.RootWorkingDirectory, String.Format("{0}.uproject", UEProjectName))); foreach (var ProjectInfo in LocalizationTask.ProjectInfos) { var LocalizationConfigFiles = new List <string>(); foreach (var LocalizationStep in ProjectInfo.LocalizationSteps) { if (LocalizationStepNames.Contains(LocalizationStep.Name)) { LocalizationConfigFiles.Add(LocalizationStep.LocalizationConfigFile); } } if (LocalizationConfigFiles.Count > 0) { var Arguments = String.Format("{0} -run=GatherText -config=\"{1}\" {2}", ProjectArgument, String.Join(";", LocalizationConfigFiles), EditorArguments); LogInformation("Running localization commandlet for '{0}': {1}", ProjectInfo.ProjectName, Arguments); LocalizationTask.GatherProcessResults.Add(Run(EditorExe, Arguments, null, CommandletRunOptions)); } else { LocalizationTask.GatherProcessResults.Add(null); } } } } // Wait for each commandlet process to finish and report the result. // This runs even for non-parallel execution to log the exit state of the process. foreach (var LocalizationTask in LocalizationTasks) { for (int ProjectIndex = 0; ProjectIndex < LocalizationTask.ProjectInfos.Count; ++ProjectIndex) { var ProjectInfo = LocalizationTask.ProjectInfos[ProjectIndex]; var RunResult = LocalizationTask.GatherProcessResults[ProjectIndex]; if (RunResult != null) { RunResult.WaitForExit(); RunResult.OnProcessExited(); RunResult.DisposeProcess(); if (RunResult.ExitCode == 0) { LogInformation("The localization commandlet for '{0}' exited with code 0.", ProjectInfo.ProjectName); } else { LogWarning("The localization commandlet for '{0}' exited with code {1} which likely indicates a crash.", ProjectInfo.ProjectName, RunResult.ExitCode); } } } } // Upload the latest sources to our localization provider if (LocalizationStepNames.Contains("Upload")) { foreach (var LocalizationTask in LocalizationTasks) { if (LocalizationTask.LocProvider != null) { // Upload all text to our localization provider for (int ProjectIndex = 0; ProjectIndex < LocalizationTask.ProjectInfos.Count; ++ProjectIndex) { var ProjectInfo = LocalizationTask.ProjectInfos[ProjectIndex]; var RunResult = LocalizationTask.GatherProcessResults[ProjectIndex]; if (RunResult != null && RunResult.ExitCode == 0) { // Recalculate the split platform paths before doing the upload, as the export may have changed them ProjectInfo.ExportInfo.CalculateSplitPlatformNames(LocalizationTask.RootLocalizationTargetDirectory); LocalizationTask.LocProvider.UploadProjectToLocalizationProvider(ProjectInfo.ProjectName, ProjectInfo.ExportInfo); } else { LogWarning("Skipping upload to the localization provider for '{0}' due to an earlier commandlet failure.", ProjectInfo.ProjectName); } } } } } // Clean-up the changelist so it only contains the changed files, and then submit it (if we were asked to) if (P4Enabled) { // Revert any PO files that haven't changed aside from their header { var POFilesToRevert = new List <string>(); var CurrentPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot); foreach (var CurrentPOFileHashPair in CurrentPOFileHashes) { byte[] InitialPOFileHash; if (InitalPOFileHashes.TryGetValue(CurrentPOFileHashPair.Key, out InitialPOFileHash) && InitialPOFileHash.SequenceEqual(CurrentPOFileHashPair.Value)) { POFilesToRevert.Add(CurrentPOFileHashPair.Key); } } if (POFilesToRevert.Count > 0) { var P4RevertArgsFilename = CombinePaths(CmdEnv.LocalRoot, "Engine", "Intermediate", String.Format("LocalizationP4RevertArgs-{0}.txt", Guid.NewGuid().ToString())); using (StreamWriter P4RevertArgsWriter = File.CreateText(P4RevertArgsFilename)) { foreach (var POFileToRevert in POFilesToRevert) { P4RevertArgsWriter.WriteLine(POFileToRevert); } } P4.LogP4(String.Format("-x{0} revert", P4RevertArgsFilename)); DeleteFile_NoExceptions(P4RevertArgsFilename); } } // Revert any other unchanged files P4.RevertUnchanged(PendingChangeList); // Submit that single changelist now if (AllowSubmit) { int SubmittedChangeList; P4.Submit(PendingChangeList, out SubmittedChangeList); } } var RunDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds; LogInformation("Localize command finished in {0} seconds", RunDuration / 1000); }
public override void ExecuteBuild() { Log("************************* Build Third Party Libs"); // figure out what batch/script to run string CompileScriptName; switch (UnrealBuildTool.BuildHostPlatform.Current.Platform) { case UnrealTargetPlatform.Win64: CompileScriptName = WindowsCompileScript; break; case UnrealTargetPlatform.Mac: CompileScriptName = MacCompileScript; break; case UnrealTargetPlatform.Linux: CompileScriptName = LinuxCompileScript; break; default: throw new AutomationException("Unknown runtime platform!"); } // look for changelist on the command line int WorkingCL = Int32.Parse(ParseParamValue("Changelist", "-1")); // if not specified, make one if (WorkingCL == -1) { WorkingCL = P4.CreateChange(P4Env.Client, String.Format("Third party libs built from changelist {0}", P4Env.Changelist)); } Log("Build from {0} Working in {1}", P4Env.Changelist, WorkingCL); // go to the third party lib dir string SearchLibraryDir = ParseParamValue("SearchDir", DefaultLibraryDir); CommandUtils.PushDir(SearchLibraryDir); // figure out what libraries to compile string LibsToCompileString = ParseParamValue("Libs"); // hunt down build batch files if the caller didn't specify a list to compile List <string> LibsToCompile = new List <string>(); if (string.IsNullOrEmpty(LibsToCompileString)) { // loop over third party directories looking for the right batch files foreach (string Dir in Directory.EnumerateDirectories(".")) { if (File.Exists(Path.Combine(Dir, CompileScriptName))) { LibsToCompile.Add(Path.GetFileName(Dir)); } } } else { // just split up the param and make sure the batch file exists string[] Libs = LibsToCompileString.Split('+'); bool bHadError = false; foreach (string Dir in Libs) { if (File.Exists(Path.Combine(Dir, CompileScriptName))) { LibsToCompile.Add(Path.GetFileName(Dir)); } else { LogError("Error: Requested lib {0} does not have a {1}", Dir, CompileScriptName); bHadError = true; } } if (bHadError) { // error out so that we don't fail to build some and have it lost in the noise throw new AutomationException("One or more libs were not set up to compile."); } } // set an envvar so that the inner batch files can check out files into a shared changelist Environment.SetEnvironmentVariable("THIRD_PARTY_CHANGELIST", string.Format("-c {0}", WorkingCL)); // now go through and run each batch file, foreach (string Lib in LibsToCompile) { Log("Building {0}", Lib); // go into the lib dir CommandUtils.PushDir(Lib); // run the builder batch file CommandUtils.RunAndLog(CmdEnv, CompileScriptName, "", "ThirdPartyLib_" + Lib); // go back to ThirdParty dir CommandUtils.PopDir(); } // undo the SearchLibraryDir push CommandUtils.PopDir(); PrintRunTime(); // revert any unchanged files P4.RevertUnchanged(WorkingCL); if (AllowSubmit) { int SubmittedCL; P4.Submit(WorkingCL, out SubmittedCL, true, true); Log("Submitted changelist {0}", SubmittedCL); } }
public override void ExecuteBuild() { int WorkingCL = -1; if (P4Enabled && AllowSubmit) { string CmdLine = ""; foreach (var Arg in Params) { CmdLine += Arg.ToString() + " "; } WorkingCL = P4.CreateChange(P4Env.Client, String.Format("MegaXGE build from changelist {0} - Params: {1}", P4Env.Changelist, CmdLine)); } LogInformation("************************* MegaXGE"); bool Clean = ParseParam("Clean"); string CleanToolLocation = CombinePaths(CmdEnv.LocalRoot, "Engine", "Build", "Batchfiles", "Clean.bat"); bool ShowProgress = ParseParam("Progress"); var UE4Build = new UE4Build(this); var Agenda = new UE4Build.BuildAgenda(); // we need to always build UHT when we use mega XGE var ProgramTargets = new string[] { "UnrealHeaderTool", }; Agenda.AddTargets(ProgramTargets, UnrealTargetPlatform.Win64, UnrealTargetConfiguration.Development); if (Clean) { LogSetProgress(ShowProgress, "Cleaning previous builds..."); foreach (var CurTarget in ProgramTargets) { string Args = String.Format("{0} {1} {2}", CurTarget, UnrealTargetPlatform.Win64.ToString(), UnrealTargetConfiguration.Development.ToString()); RunAndLog(CmdEnv, CleanToolLocation, Args); } } LogInformation("*************************"); for (int Arg = 1; Arg < 100; Arg++) { string Parm = String.Format("Target{0}", Arg); string Target = ParseParamValue(Parm, ""); if (String.IsNullOrEmpty(Target)) { break; } FileReference ProjectFile = null; string ProjectFileParam = ParseParamValue(String.Format("Project{0}", Arg), null); if (ProjectFileParam != null) { ProjectFile = new FileReference(ProjectFileParam); if (!FileReference.Exists(ProjectFile)) { throw new AutomationException("Project file '{0}' could not be found"); } } var Parts = Target.Split(' '); string JustTarget = Parts[0]; if (String.IsNullOrEmpty(JustTarget)) { throw new AutomationException("BUILD FAILED target option '{0}' not parsed.", Target); } var Targets = JustTarget.Split('|'); if (Targets.Length < 1) { throw new AutomationException("BUILD FAILED target option '{0}' not parsed.", Target); } var Platforms = new List <UnrealTargetPlatform>(); var Configurations = new List <UnrealTargetConfiguration>(); for (int Part = 1; Part < Parts.Length; Part++) { if (!String.IsNullOrEmpty(Parts[Part])) { var SubParts = Parts[Part].Split('|'); foreach (var SubPart in SubParts) { UnrealTargetPlatform Platform; if (Enum.TryParse(SubPart, true, out Platform)) { Platforms.Add(Platform); } else { switch (SubPart.ToUpperInvariant()) { case "DEBUG": Configurations.Add(UnrealTargetConfiguration.Debug); break; case "DEBUGGAME": Configurations.Add(UnrealTargetConfiguration.DebugGame); break; case "DEVELOPMENT": Configurations.Add(UnrealTargetConfiguration.Development); break; case "SHIPPING": Configurations.Add(UnrealTargetConfiguration.Shipping); break; case "TEST": Configurations.Add(UnrealTargetConfiguration.Test); break; default: throw new AutomationException("BUILD FAILED target option {0} not recognized.", SubPart); } } } } } if (Platforms.Count < 1) { Platforms.Add(UnrealTargetPlatform.Win64); } if (Configurations.Count < 1) { Configurations.Add(UnrealTargetConfiguration.Development); } foreach (var Platform in Platforms) { foreach (var CurTarget in Targets) { foreach (var Configuration in Configurations) { Agenda.AddTargets(new string[] { CurTarget }, Platform, Configuration, ProjectFile); LogInformation("Target {0} {1} {2}", CurTarget, Platform.ToString(), Configuration.ToString()); if (Clean) { string Args = String.Format("{0} {1} {2}", CurTarget, Platform.ToString(), Configuration.ToString()); RunAndLog(CmdEnv, CleanToolLocation, Args); } } } } } LogInformation("*************************"); Agenda.DoRetries = ParseParam("Retry"); UE4Build.Build(Agenda, InUpdateVersionFiles: IsBuildMachine, InUseParallelExecutor: ParseParam("useparallelexecutor"), InShowProgress: ShowProgress); // if (WorkingCL > 0) // only move UAT files if we intend to check in some build products // { // UE4Build.CopyUATFilesAndAddToBuildProducts(); // } UE4Build.CheckBuildProducts(UE4Build.BuildProductFiles); if (WorkingCL > 0) { // Sign everything we built CodeSign.SignMultipleIfEXEOrDLL(this, UE4Build.BuildProductFiles); // Open files for add or edit UE4Build.AddBuildProductsToChangelist(WorkingCL, UE4Build.BuildProductFiles); int SubmittedCL; P4.Submit(WorkingCL, out SubmittedCL, true, true); } PrintRunTime(); }
public override void ExecuteBuild() { string BucketName = ParseRequiredStringParam("Bucket"); FileReference CredentialsFile = ParseRequiredFileReferenceParam("CredentialsFile"); string CredentialsKey = ParseRequiredStringParam("CredentialsKey"); DirectoryReference CacheDir = ParseRequiredDirectoryReferenceParam("CacheDir"); DirectoryReference FilterDir = ParseRequiredDirectoryReferenceParam("FilterDir"); int Days = ParseParamInt("Days", 7); int MaxFileSize = ParseParamInt("MaxFileSize", 0); string RootManifestPath = ParseRequiredStringParam("Manifest"); string KeyPrefix = ParseParamValue("KeyPrefix", ""); bool bReset = ParseParam("Reset"); // The credentials to upload with AWSCredentials Credentials; // Try to get the credentials by the key passed in from the script CredentialProfileStoreChain CredentialsChain = new CredentialProfileStoreChain(CredentialsFile.FullName); if (!CredentialsChain.TryGetAWSCredentials(CredentialsKey, out Credentials)) { throw new AutomationException("Unknown credentials key: {0}", CredentialsKey); } // Create the new client using (AmazonS3Client Client = new AmazonS3Client(Credentials, Region)) { using (SemaphoreSlim RequestSemaphore = new SemaphoreSlim(4)) { // Read the filters HashSet <string> Paths = new HashSet <string>(); foreach (FileInfo FilterFile in FilterDir.ToDirectoryInfo().EnumerateFiles("*.txt")) { TimeSpan Age = DateTime.UtcNow - FilterFile.LastWriteTimeUtc; if (Age < TimeSpan.FromDays(3)) { Log.TraceInformation("Reading {0}", FilterFile.FullName); string[] Lines = File.ReadAllLines(FilterFile.FullName); foreach (string Line in Lines) { string TrimLine = Line.Trim().Replace('\\', '/'); if (TrimLine.Length > 0) { Paths.Add(TrimLine); } } } else if (Age > TimeSpan.FromDays(5)) { try { Log.TraceInformation("Deleting {0}", FilterFile.FullName); FilterFile.Delete(); } catch (Exception Ex) { Log.TraceWarning("Unable to delete: {0}", Ex.Message); Log.TraceLog(ExceptionUtils.FormatExceptionDetails(Ex)); } } } Log.TraceInformation("Found {0:n0} files", Paths.Count); // Enumerate all the files that are in the network DDC Log.TraceInformation(""); Log.TraceInformation("Filtering files in {0}...", CacheDir); List <DerivedDataFile> Files = ParallelExecute <string, DerivedDataFile>(Paths, (Path, FilesBag) => ReadFileInfo(CacheDir, Path, FilesBag)); // Filter to the maximum size if (MaxFileSize != 0) { int NumRemovedMaxSize = Files.RemoveAll(x => x.Info.Length > MaxFileSize); Log.TraceInformation(""); Log.TraceInformation("Removed {0} files above size limit ({1:n0} bytes)", NumRemovedMaxSize, MaxFileSize); } // Create the working directory DirectoryReference WorkingDir = DirectoryReference.Combine(EngineDirectory, "Saved", "UploadDDC"); DirectoryReference.CreateDirectory(WorkingDir); // Get the path to the manifest FileReference RootManifestFile = FileReference.Combine(CommandUtils.RootDirectory, RootManifestPath); // Read the old root manifest RootManifest OldRootManifest = new RootManifest(); if (FileReference.Exists(RootManifestFile)) { OldRootManifest.Read(JsonObject.Read(RootManifestFile)); } // Read the old bundle manifest BundleManifest OldBundleManifest = new BundleManifest(); if (OldRootManifest.Entries.Count > 0) { FileReference LocalBundleManifest = FileReference.Combine(WorkingDir, "OldBundleManifest.json"); if (TryDownloadFile(Client, BucketName, OldRootManifest.Entries.Last().Key, LocalBundleManifest)) { OldBundleManifest.Read(JsonObject.Read(LocalBundleManifest)); } } // Create the new manifest BundleManifest NewBundleManifest = new BundleManifest(); // Try to download the old manifest, and add all the bundles we want to keep to the new manifest if (!bReset) { foreach (BundleManifest.Entry Bundle in OldBundleManifest.Entries) { FileReference BundleFile = FileReference.Combine(WorkingDir, Bundle.Name); if (!FileReference.Exists(BundleFile)) { Log.TraceInformation("Downloading {0}", BundleFile); FileReference TempCompressedFile = new FileReference(BundleFile.FullName + ".incoming.gz"); if (!TryDownloadFile(Client, BucketName, Bundle.ObjectKey, TempCompressedFile)) { Log.TraceWarning("Unable to download {0}", Bundle.ObjectKey); continue; } FileReference TempUncompressedFile = new FileReference(BundleFile.FullName + ".incoming"); try { DecompressFile(TempCompressedFile, TempUncompressedFile); } catch (Exception Ex) { Log.TraceWarning("Unable to uncompress {0}: {1}", Bundle.ObjectKey, Ex.ToString()); continue; } FileReference.Move(TempUncompressedFile, BundleFile); } NewBundleManifest.Entries.Add(Bundle); } } // Figure out all the item digests that we already have Dictionary <BundleManifest.Entry, HashSet <ContentHash> > BundleToKeyHashes = new Dictionary <BundleManifest.Entry, HashSet <ContentHash> >(); foreach (BundleManifest.Entry Bundle in NewBundleManifest.Entries) { HashSet <ContentHash> KeyHashes = new HashSet <ContentHash>(); FileReference BundleFile = FileReference.Combine(WorkingDir, Bundle.Name); using (FileStream Stream = FileReference.Open(BundleFile, FileMode.Open, FileAccess.Read, FileShare.Read)) { BinaryReader Reader = new BinaryReader(Stream); uint Signature = Reader.ReadUInt32(); if (Signature != BundleSignatureV1) { throw new Exception(String.Format("Invalid signature for {0}", BundleFile)); } int NumEntries = Reader.ReadInt32(); for (int EntryIdx = 0; EntryIdx < NumEntries; EntryIdx++) { byte[] Digest = new byte[ContentHash.LengthSHA1]; if (Reader.Read(Digest, 0, ContentHash.LengthSHA1) != ContentHash.LengthSHA1) { throw new Exception("Unexpected EOF"); } KeyHashes.Add(new ContentHash(Digest)); Stream.Seek(4, SeekOrigin.Current); } } BundleToKeyHashes[Bundle] = KeyHashes; } // Calculate the download size of the manifest long DownloadSize = NewBundleManifest.Entries.Sum(x => (long)x.CompressedLength); // Remove any bundles which have less than the minimum required size in valid data. We don't mark the manifest as dirty yet; these // files will only be rewritten if new content is added, to prevent the last bundle being rewritten multiple times. foreach (KeyValuePair <BundleManifest.Entry, HashSet <ContentHash> > Pair in BundleToKeyHashes) { long ValidBundleSize = Files.Where(x => Pair.Value.Contains(x.KeyHash)).Sum(x => (long)x.Info.Length); if (ValidBundleSize < MinBundleSize) { NewBundleManifest.Entries.Remove(Pair.Key); } } // Find all the valid digests HashSet <ContentHash> ReusedKeyHashes = new HashSet <ContentHash>(); foreach (BundleManifest.Entry Bundle in NewBundleManifest.Entries) { ReusedKeyHashes.UnionWith(BundleToKeyHashes[Bundle]); } // Remove all the files which already exist int NumRemovedExist = Files.RemoveAll(x => ReusedKeyHashes.Contains(x.KeyHash)); if (NumRemovedExist > 0) { Log.TraceInformation(""); Log.TraceInformation("Removed {0:n0} files which already exist", NumRemovedExist); } // Read all the files we want to include List <Tuple <DerivedDataFile, byte[]> > FilesToInclude = new List <Tuple <DerivedDataFile, byte[]> >(); if (Files.Count > 0) { Log.TraceInformation(""); Log.TraceInformation("Reading remaining {0:n0} files into memory ({1:n1}mb)...", Files.Count, (float)Files.Sum(x => (long)x.Info.Length) / (1024 * 1024)); FilesToInclude.AddRange(ParallelExecute <DerivedDataFile, Tuple <DerivedDataFile, byte[]> >(Files, (x, y) => ReadFileData(x, y))); } // Generate new data using (RNGCryptoServiceProvider Crypto = new RNGCryptoServiceProvider()) { // Flag for whether to update the manifest bool bUpdateManifest = false; // Upload the new bundle Log.TraceInformation(""); if (FilesToInclude.Count == 0) { Log.TraceInformation("No new files to add."); } else { // Sort the files to include by creation time. This will bias towards grouping older, more "permanent", items together. Log.TraceInformation("Sorting input files"); List <Tuple <DerivedDataFile, byte[]> > SortedFilesToInclude = FilesToInclude.OrderBy(x => x.Item1.Info.CreationTimeUtc).ToList(); // Get the target bundle size long TotalSize = SortedFilesToInclude.Sum(x => (long)x.Item2.Length); int NumBundles = (int)((TotalSize + (MaxBundleSize - 1)) / MaxBundleSize); long TargetBundleSize = TotalSize / NumBundles; // Split the input data into bundles List <List <Tuple <DerivedDataFile, byte[]> > > BundleFilesToIncludeList = new List <List <Tuple <DerivedDataFile, byte[]> > >(); long BundleSize = 0; for (int FileIdx = 0; FileIdx < SortedFilesToInclude.Count; BundleSize = BundleSize % TargetBundleSize) { List <Tuple <DerivedDataFile, byte[]> > BundleFilesToInclude = new List <Tuple <DerivedDataFile, byte[]> >(); for (; BundleSize < TargetBundleSize && FileIdx < SortedFilesToInclude.Count; FileIdx++) { BundleFilesToInclude.Add(SortedFilesToInclude[FileIdx]); BundleSize += SortedFilesToInclude[FileIdx].Item2.Length; } BundleFilesToIncludeList.Add(BundleFilesToInclude); } // Upload each bundle DateTime NewBundleTime = DateTime.UtcNow; for (int BundleIdx = 0; BundleIdx < BundleFilesToIncludeList.Count; BundleIdx++) { List <Tuple <DerivedDataFile, byte[]> > BundleFilesToInclude = BundleFilesToIncludeList[BundleIdx]; // Get the new bundle info string NewBundleSuffix = (BundleFilesToIncludeList.Count > 1) ? String.Format("-{0}_of_{1}", BundleIdx + 1, BundleFilesToIncludeList.Count) : ""; string NewBundleName = String.Format("Bundle-{0:yyyy.MM.dd-HH.mm}{1}.ddb", NewBundleTime.ToLocalTime(), NewBundleSuffix); // Create a random number for the object key string NewBundleObjectKey = KeyPrefix + "bulk/" + CreateObjectName(Crypto); // Create the bundle header byte[] Header; using (MemoryStream HeaderStream = new MemoryStream()) { BinaryWriter Writer = new BinaryWriter(HeaderStream); Writer.Write(BundleSignatureV1); Writer.Write(BundleFilesToInclude.Count); foreach (Tuple <DerivedDataFile, byte[]> FileToInclude in BundleFilesToInclude) { Writer.Write(FileToInclude.Item1.KeyHash.Bytes, 0, ContentHash.LengthSHA1); Writer.Write((int)FileToInclude.Item2.Length); } Header = HeaderStream.ToArray(); } // Create the output file FileReference NewBundleFile = FileReference.Combine(WorkingDir, NewBundleName + ".gz"); Log.TraceInformation("Writing {0}", NewBundleFile); using (FileStream BundleStream = FileReference.Open(NewBundleFile, FileMode.Create, FileAccess.Write, FileShare.Read)) { using (GZipStream ZipStream = new GZipStream(BundleStream, CompressionLevel.Optimal, true)) { ZipStream.Write(Header, 0, Header.Length); foreach (Tuple <DerivedDataFile, byte[]> FileToInclude in BundleFilesToInclude) { ZipStream.Write(FileToInclude.Item2, 0, FileToInclude.Item2.Length); } } } // Upload the file long NewBundleCompressedLength = NewBundleFile.ToFileInfo().Length; long NewBundleUncompressedLength = Header.Length + BundleFilesToInclude.Sum(x => (long)x.Item2.Length); Log.TraceInformation("Uploading bundle to {0} ({1:n1}mb)", NewBundleObjectKey, NewBundleCompressedLength / (1024.0f * 1024.0f)); UploadFile(Client, BucketName, NewBundleFile, 0, NewBundleObjectKey, RequestSemaphore, null); // Add the bundle to the new manifest BundleManifest.Entry Bundle = new BundleManifest.Entry(); Bundle.Name = NewBundleName; Bundle.ObjectKey = NewBundleObjectKey; Bundle.Time = NewBundleTime; Bundle.CompressedLength = (int)NewBundleCompressedLength; Bundle.UncompressedLength = (int)NewBundleUncompressedLength; NewBundleManifest.Entries.Add(Bundle); // Mark the manifest as requiring an update bUpdateManifest = true; } } // Update the manifest if (bUpdateManifest) { DateTime UtcNow = DateTime.UtcNow; DateTime RemoveBundleManifestsBefore = UtcNow - TimeSpan.FromDays(3.0); // Update the root manifest RootManifest NewRootManifest = new RootManifest(); NewRootManifest.AccessKey = OldRootManifest.AccessKey; NewRootManifest.SecretKey = OldRootManifest.SecretKey; foreach (RootManifest.Entry Entry in OldRootManifest.Entries) { if (Entry.CreateTime >= RemoveBundleManifestsBefore) { NewRootManifest.Entries.Add(Entry); } } // Make sure there's an entry for the last 24h DateTime RequireBundleManifestAfter = UtcNow - TimeSpan.FromDays(1.0); if (!NewRootManifest.Entries.Any(x => x.CreateTime > RequireBundleManifestAfter)) { RootManifest.Entry NewEntry = new RootManifest.Entry(); NewEntry.CreateTime = UtcNow; NewEntry.Key = KeyPrefix + CreateObjectName(Crypto); NewRootManifest.Entries.Add(NewEntry); } // Save out the new bundle manifest FileReference NewBundleManifestFile = FileReference.Combine(WorkingDir, "NewBundleManifest.json"); NewBundleManifest.Save(NewBundleManifestFile); // Update all the bundle manifests still valid foreach (RootManifest.Entry Entry in NewRootManifest.Entries) { Log.TraceInformation("Uploading bundle manifest to {0}", Entry.Key); UploadFile(Client, BucketName, NewBundleManifestFile, 0, Entry.Key, RequestSemaphore, null); } // Overwrite all the existing manifests if (AllowSubmit) { List <string> ExistingFiles = P4.Files(CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName)); // Create a changelist containing the new manifest int ChangeNumber = P4.CreateChange(Description: "Updating DDC bundle manifest"); if (ExistingFiles.Count > 0) { P4.Edit(ChangeNumber, CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName)); NewRootManifest.Save(RootManifestFile); } else { NewRootManifest.Save(RootManifestFile); P4.Add(ChangeNumber, CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName)); } // Submit it int SubmittedChangeNumber; P4.Submit(ChangeNumber, out SubmittedChangeNumber, true); if (SubmittedChangeNumber <= 0) { throw new AutomationException("Failed to submit change"); } // Delete any bundles that are no longer referenced HashSet <string> KeepObjectKeys = new HashSet <string>(NewBundleManifest.Entries.Select(x => x.ObjectKey)); foreach (BundleManifest.Entry OldEntry in OldBundleManifest.Entries) { if (!KeepObjectKeys.Contains(OldEntry.ObjectKey)) { Log.TraceInformation("Deleting unreferenced bundle {0}", OldEntry.ObjectKey); DeleteFile(Client, BucketName, OldEntry.ObjectKey, RequestSemaphore, null); } } // Delete any bundle manifests which are no longer referenced HashSet <string> KeepManifestKeys = new HashSet <string>(NewRootManifest.Entries.Select(x => x.Key)); foreach (RootManifest.Entry OldEntry in OldRootManifest.Entries) { if (!KeepManifestKeys.Contains(OldEntry.Key)) { Log.TraceInformation("Deleting unreferenced manifest {0}", OldEntry.Key); DeleteFile(Client, BucketName, OldEntry.Key, RequestSemaphore, null); } } } else { // Skip submitting Log.TraceWarning("Skipping manifest submit due to missing -Submit argument."); } // Update the new download size DownloadSize = NewBundleManifest.Entries.Sum(x => (long)x.CompressedLength); } } // Print some stats about the final manifest Log.TraceInformation(""); Log.TraceInformation("Total download size {0:n1}mb", DownloadSize / (1024.0 * 1024.0)); Log.TraceInformation(""); } } }
public override void ExecuteBuild() { SetupStaticBuildEnvironment(); bool bBuildSolutions = true; if (ParseParam("SkipBuildSolutions")) { bBuildSolutions = false; } bool bBuildLibraries = true; if (ParseParam("SkipBuild")) { bBuildLibraries = false; } bool bAutoCreateChangelist = true; if (ParseParam("SkipCreateChangelist")) { bAutoCreateChangelist = false; } bool bAutoSubmit = false; // bAutoCreateChangelist; if (ParseParam("SkipSubmit")) { bAutoSubmit = false; } // if we don't pass anything, we'll just merge by default string RobomergeCommand = ParseParamValue("Robomerge", "").ToLower(); if (!string.IsNullOrEmpty(RobomergeCommand)) { // for merge default action, add flag to make sure buildmachine commit isn't skipped if (RobomergeCommand == "merge") { RobomergeCommand = "#robomerge[all] #DisregardExcludedAuthors"; } // otherwise add hashtags else if (RobomergeCommand == "ignore") { RobomergeCommand = "#robomerge #ignore"; } else if (RobomergeCommand == "null") { RobomergeCommand = "#robomerge #null"; } // otherwise the submit will likely fail. else { throw new AutomationException("Invalid Robomerge param passed in {0}. Must be \"merge\", \"null\", or \"ignore\"", RobomergeCommand); } } // get the platforms we want to build for List <TargetPlatformData> TargetPlatforms = GetTargetPlatforms(); // get the platforms we want to build for List <WindowsCompiler> TargetWindowsCompilers = GetTargetWindowsCompilers(); // get the configurations we want to build for List <string> TargetConfigurations = GetTargetConfigurations(); if (bBuildSolutions) { // build target lib for all platforms foreach (TargetPlatformData TargetData in TargetPlatforms) { if (!PlatformSupportsTargetLib(TargetData)) { continue; } SetupBuildForTargetLibAndPlatform(TargetData, TargetConfigurations, TargetWindowsCompilers, false); } } HashSet <FileReference> FilesToReconcile = new HashSet <FileReference>(); if (bBuildLibraries) { // build target lib for all platforms foreach (TargetPlatformData TargetData in TargetPlatforms) { if (!PlatformSupportsTargetLib(TargetData)) { continue; } HashSet <FileReference> FilesToDelete = new HashSet <FileReference>(); foreach (string TargetConfiguration in TargetConfigurations) { // Delete output files before building them if (TargetData.Platform == UnrealTargetPlatform.Win64) { foreach (WindowsCompiler TargetCompiler in TargetWindowsCompilers) { FindOutputFiles(FilesToDelete, TargetData, TargetConfiguration, TargetCompiler); } } else { FindOutputFiles(FilesToDelete, TargetData, TargetConfiguration); } } foreach (FileReference FileToDelete in FilesToDelete) { FilesToReconcile.Add(FileToDelete); InternalUtils.SafeDeleteFile(FileToDelete.ToString()); } BuildTargetLibForPlatform(TargetData, TargetConfigurations, TargetWindowsCompilers); if (DoesPlatformUseMSBuild(TargetData)) { foreach (WindowsCompiler TargetWindowsCompiler in TargetWindowsCompilers) { CopyLibsToFinalDestination(TargetData, TargetConfigurations, TargetWindowsCompiler); } } else { CopyLibsToFinalDestination(TargetData, TargetConfigurations); } } } int P4ChangeList = InvalidChangeList; if (bAutoCreateChangelist) { string RobomergeLine = string.Empty; if (!string.IsNullOrEmpty(RobomergeCommand)) { RobomergeLine = Environment.NewLine + RobomergeCommand; } P4ChangeList = P4.CreateChange(P4Env.Client, "BuildHlslcc.Automation: Deploying hlslcc libs." + Environment.NewLine + "#rb none" + Environment.NewLine + "#lockdown Nick.Penwarden" + Environment.NewLine + "#tests none" + Environment.NewLine + "#jira none" + Environment.NewLine + "#okforgithub ignore" + RobomergeLine); } if (P4ChangeList != InvalidChangeList) { foreach (string TargetConfiguration in TargetConfigurations) { //Add any new files that p4 is not yet tracking. foreach (TargetPlatformData TargetData in TargetPlatforms) { if (!PlatformSupportsTargetLib(TargetData)) { continue; } if (TargetData.Platform == UnrealTargetPlatform.Win64) { foreach (WindowsCompiler TargetCompiler in TargetWindowsCompilers) { FindOutputFiles(FilesToReconcile, TargetData, TargetConfiguration, TargetCompiler); } } else { FindOutputFiles(FilesToReconcile, TargetData, TargetConfiguration); } } } foreach (FileReference FileToReconcile in FilesToReconcile) { P4.Reconcile(P4ChangeList, FileToReconcile.ToString()); } } if (bAutoSubmit && (P4ChangeList != InvalidChangeList)) { if (!P4.TryDeleteEmptyChange(P4ChangeList)) { LogInformation("Submitting changelist " + P4ChangeList.ToString()); int SubmittedChangeList = InvalidChangeList; P4.Submit(P4ChangeList, out SubmittedChangeList); } else { LogInformation("Nothing to submit!"); } } }