/// <summary>
/// Main entry point for the command
/// </summary>
public override void ExecuteBuild()
{
string FileSpec = ParseRequiredStringParam("Files");
// Make sure the patterns are a valid syntax
if (!FileSpec.StartsWith("//"))
{
throw new AutomationException("Files must be specified as full depot paths");
}
// Pick out the source and target prefixes
string Prefix;
if (FileSpec.EndsWith("*"))
{
Prefix = FileSpec.Substring(0, FileSpec.Length - 1);
}
else if (FileSpec.EndsWith("..."))
{
Prefix = FileSpec.Substring(0, FileSpec.Length - 3);
}
else
{
Prefix = FileSpec;
}
// Make sure there aren't any other wildcards in the pattern
if (Prefix.Contains("?") || Prefix.Contains("*") || Prefix.Contains("..."))
{
throw new AutomationException("Wildcards are only permitted at the end of filespecs");
}
// Find all the source files
List <string> SourceFiles = P4.Files(String.Format("-e {0}", FileSpec));
if (SourceFiles.Count == 0)
{
throw new AutomationException("No files found matching {0}", FileSpec);
}
SourceFiles.RemoveAll(x => x.StartsWith(Prefix, StringComparison.Ordinal));
// Error if we didn't find anything
if (SourceFiles.Count == 0)
{
throw new AutomationException("No files found matching spec");
}
// Find all the target files
List <string> TargetFiles = new List <string>(SourceFiles.Count);
foreach (string SourceFile in SourceFiles)
{
if (SourceFile.StartsWith(Prefix, StringComparison.OrdinalIgnoreCase))
{
TargetFiles.Add(Prefix + SourceFile.Substring(Prefix.Length));
}
else
{
throw new AutomationException("Source file '{0}' does not start with '{1}'", SourceFile, Prefix);
}
}
// Print what we're going to do
LogInformation("Ready to rename {0} files:", SourceFiles.Count);
for (int Idx = 0; Idx < SourceFiles.Count; Idx++)
{
LogInformation("{0,3}: {1}", Idx, SourceFiles[Idx]);
LogInformation("{0,3} {1}", "", TargetFiles[Idx]);
}
// If we're not going through with it, print the renames
if (!AllowSubmit)
{
LogWarning("Skipping due to no -Submit option");
return;
}
// Force sync all the old files
foreach (string OldFile in SourceFiles)
{
P4.LogP4(String.Format("sync -f {0}", OldFile));
}
// Delete all the old files
int DeleteChangeNumber = P4.CreateChange(Description: String.Format("Fixing case of {0} (1/2){1}", FileSpec, BoilerplateText));
foreach (string OldFile in SourceFiles)
{
P4.LogP4(String.Format("delete -k -c {0} {1}", DeleteChangeNumber, OldFile));
}
P4.Submit(DeleteChangeNumber);
// Re-add all the files in the new location
int AddChangeNumber = P4.CreateChange(Description: String.Format("Fixing case of {0} (2/2){1}", FileSpec, BoilerplateText));
foreach (string NewFile in TargetFiles)
{
P4.LogP4(String.Format("add -c {0} {1}", AddChangeNumber, NewFile));
}
P4.Submit(AddChangeNumber);
}
public override void ExecuteBuild()
{
var UEProjectRoot = ParseParamValue("UEProjectRoot");
if (UEProjectRoot == null)
{
UEProjectRoot = CmdEnv.LocalRoot;
}
var UEProjectDirectory = ParseParamValue("UEProjectDirectory");
if (UEProjectDirectory == null)
{
throw new AutomationException("Missing required command line argument: 'UEProjectDirectory'");
}
var UEProjectName = ParseParamValue("UEProjectName");
if (UEProjectName == null)
{
UEProjectName = "";
}
var LocalizationProjectNames = new List <string>();
{
var LocalizationProjectNamesStr = ParseParamValue("LocalizationProjectNames");
if (LocalizationProjectNamesStr != null)
{
foreach (var ProjectName in LocalizationProjectNamesStr.Split(','))
{
LocalizationProjectNames.Add(ProjectName.Trim());
}
}
}
var LocalizationProviderName = ParseParamValue("LocalizationProvider");
if (LocalizationProviderName == null)
{
LocalizationProviderName = "OneSky";
}
var LocalizationSteps = new List <string>();
{
var LocalizationStepsStr = ParseParamValue("LocalizationSteps");
if (LocalizationStepsStr == null)
{
LocalizationSteps.AddRange(new string[] { "Download", "Gather", "Import", "Export", "Compile", "GenerateReports", "Upload" });
}
else
{
foreach (var StepName in LocalizationStepsStr.Split(','))
{
LocalizationSteps.Add(StepName.Trim());
}
}
LocalizationSteps.Add("Monolithic"); // Always allow the monolithic scripts to run as we don't know which steps they do
}
var ShouldGatherPlugins = ParseParam("IncludePlugins");
var IncludePlugins = new List <string>();
var ExcludePlugins = new List <string>();
if (ShouldGatherPlugins)
{
var IncludePluginsStr = ParseParamValue("IncludePlugins");
if (IncludePluginsStr != null)
{
foreach (var PluginName in IncludePluginsStr.Split(','))
{
IncludePlugins.Add(PluginName.Trim());
}
}
var ExcludePluginsStr = ParseParamValue("ExcludePlugins");
if (ExcludePluginsStr != null)
{
foreach (var PluginName in ExcludePluginsStr.Split(','))
{
ExcludePlugins.Add(PluginName.Trim());
}
}
}
var AdditionalCommandletArguments = ParseParamValue("AdditionalCommandletArguments");
if (AdditionalCommandletArguments == null)
{
AdditionalCommandletArguments = "";
}
var LocalizationBatches = new List <LocalizationBatch>();
// Add the static set of localization projects as a batch
if (LocalizationProjectNames.Count > 0)
{
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, UEProjectDirectory, "", LocalizationProjectNames));
}
// Build up any additional batches needed for plugins
if (ShouldGatherPlugins)
{
var PluginsRootDirectory = CombinePaths(UEProjectRoot, UEProjectDirectory, "Plugins");
IReadOnlyList <PluginInfo> AllPlugins = Plugins.ReadPluginsFromDirectory(new DirectoryReference(PluginsRootDirectory), UEProjectName.Length == 0 ? PluginType.Engine : PluginType.Project);
// Add a batch for each plugin that meets our criteria
foreach (var PluginInfo in AllPlugins)
{
bool ShouldIncludePlugin = (IncludePlugins.Count == 0 || IncludePlugins.Contains(PluginInfo.Name)) && !ExcludePlugins.Contains(PluginInfo.Name);
if (ShouldIncludePlugin && PluginInfo.Descriptor.LocalizationTargets != null && PluginInfo.Descriptor.LocalizationTargets.Length > 0)
{
var RootRelativePluginPath = PluginInfo.Directory.MakeRelativeTo(new DirectoryReference(UEProjectRoot));
RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4
var PluginTargetNames = new List <string>();
foreach (var LocalizationTarget in PluginInfo.Descriptor.LocalizationTargets)
{
PluginTargetNames.Add(LocalizationTarget.Name);
}
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, PluginInfo.Name, PluginTargetNames));
}
}
}
// Create a single changelist to use for all changes, and hash the current PO files on disk so we can work out whether they actually change
int PendingChangeList = 0;
Dictionary <string, byte[]> InitalPOFileHashes = null;
if (P4Enabled)
{
PendingChangeList = P4.CreateChange(P4Env.Client, "Localization Automation");
InitalPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot);
}
// Process each localization batch
foreach (var LocalizationBatch in LocalizationBatches)
{
ProcessLocalizationProjects(LocalizationBatch, PendingChangeList, UEProjectRoot, UEProjectName, LocalizationProviderName, LocalizationSteps, AdditionalCommandletArguments);
}
// Clean-up the changelist so it only contains the changed files, and then submit it (if we were asked to)
if (P4Enabled)
{
// Revert any PO files that haven't changed aside from their header
{
var POFilesToRevert = new List <string>();
var CurrentPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot);
foreach (var CurrentPOFileHashPair in CurrentPOFileHashes)
{
byte[] InitialPOFileHash;
if (InitalPOFileHashes.TryGetValue(CurrentPOFileHashPair.Key, out InitialPOFileHash) && InitialPOFileHash.SequenceEqual(CurrentPOFileHashPair.Value))
{
POFilesToRevert.Add(CurrentPOFileHashPair.Key);
}
}
if (POFilesToRevert.Count > 0)
{
var P4RevertArgsFilename = CombinePaths(CmdEnv.LocalRoot, "Engine", "Intermediate", String.Format("LocalizationP4RevertArgs-{0}.txt", Guid.NewGuid().ToString()));
using (StreamWriter P4RevertArgsWriter = File.CreateText(P4RevertArgsFilename))
{
foreach (var POFileToRevert in POFilesToRevert)
{
P4RevertArgsWriter.WriteLine(POFileToRevert);
}
}
P4.LogP4(String.Format("-x{0} revert", P4RevertArgsFilename));
DeleteFile_NoExceptions(P4RevertArgsFilename);
}
}
// Revert any other unchanged files
P4.RevertUnchanged(PendingChangeList);
// Submit that single changelist now
if (AllowSubmit)
{
int SubmittedChangeList;
P4.Submit(PendingChangeList, out SubmittedChangeList);
}
}
}
public override void ExecuteBuild()
{
var UEProjectRoot = ParseParamValue("UEProjectRoot");
if (UEProjectRoot == null)
{
UEProjectRoot = CmdEnv.LocalRoot;
}
var UEProjectDirectory = ParseParamValue("UEProjectDirectory");
if (UEProjectDirectory == null)
{
throw new AutomationException("Missing required command line argument: 'UEProjectDirectory'");
}
var UEProjectName = ParseParamValue("UEProjectName");
if (UEProjectName == null)
{
UEProjectName = "";
}
var LocalizationProjectNames = new List <string>();
{
var LocalizationProjectNamesStr = ParseParamValue("LocalizationProjectNames");
if (LocalizationProjectNamesStr != null)
{
foreach (var ProjectName in LocalizationProjectNamesStr.Split(','))
{
LocalizationProjectNames.Add(ProjectName.Trim());
}
}
}
var LocalizationProviderName = ParseParamValue("LocalizationProvider");
if (LocalizationProviderName == null)
{
LocalizationProviderName = "";
}
var LocalizationStepNames = new List <string>();
{
var LocalizationStepNamesStr = ParseParamValue("LocalizationSteps");
if (LocalizationStepNamesStr == null)
{
LocalizationStepNames.AddRange(new string[] { "Download", "Gather", "Import", "Export", "Compile", "GenerateReports", "Upload" });
}
else
{
foreach (var StepName in LocalizationStepNamesStr.Split(','))
{
LocalizationStepNames.Add(StepName.Trim());
}
}
LocalizationStepNames.Add("Monolithic"); // Always allow the monolithic scripts to run as we don't know which steps they do
}
var ShouldGatherPlugins = ParseParam("IncludePlugins");
var IncludePlugins = new List <string>();
var ExcludePlugins = new List <string>();
if (ShouldGatherPlugins)
{
var IncludePluginsStr = ParseParamValue("IncludePlugins");
if (IncludePluginsStr != null)
{
foreach (var PluginName in IncludePluginsStr.Split(','))
{
IncludePlugins.Add(PluginName.Trim());
}
}
var ExcludePluginsStr = ParseParamValue("ExcludePlugins");
if (ExcludePluginsStr != null)
{
foreach (var PluginName in ExcludePluginsStr.Split(','))
{
ExcludePlugins.Add(PluginName.Trim());
}
}
}
var ShouldGatherPlatforms = ParseParam("IncludePlatforms");
var AdditionalCommandletArguments = ParseParamValue("AdditionalCommandletArguments");
if (AdditionalCommandletArguments == null)
{
AdditionalCommandletArguments = "";
}
var EnableParallelGather = ParseParam("ParallelGather");
var StartTime = DateTime.UtcNow;
var LocalizationBatches = new List <LocalizationBatch>();
// Add the static set of localization projects as a batch
if (LocalizationProjectNames.Count > 0)
{
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, UEProjectDirectory, "", LocalizationProjectNames));
}
// Build up any additional batches needed for platforms
if (ShouldGatherPlatforms)
{
var PlatformsRootDirectory = new DirectoryReference(CombinePaths(UEProjectRoot, UEProjectDirectory, "Platforms"));
if (DirectoryReference.Exists(PlatformsRootDirectory))
{
foreach (DirectoryReference PlatformDirectory in DirectoryReference.EnumerateDirectories(PlatformsRootDirectory))
{
// Find the localization targets defined for this platform
var PlatformTargetNames = GetLocalizationTargetsFromDirectory(new DirectoryReference(CombinePaths(PlatformDirectory.FullName, "Config", "Localization")));
if (PlatformTargetNames.Count > 0)
{
var RootRelativePluginPath = PlatformDirectory.MakeRelativeTo(new DirectoryReference(UEProjectRoot));
RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, "", PlatformTargetNames));
}
}
}
}
// Build up any additional batches needed for plugins
if (ShouldGatherPlugins)
{
var PluginsRootDirectory = new DirectoryReference(CombinePaths(UEProjectRoot, UEProjectDirectory));
IReadOnlyList <PluginInfo> AllPlugins = Plugins.ReadPluginsFromDirectory(PluginsRootDirectory, "Plugins", UEProjectName.Length == 0 ? PluginType.Engine : PluginType.Project);
// Add a batch for each plugin that meets our criteria
var AvailablePluginNames = new HashSet <string>();
foreach (var PluginInfo in AllPlugins)
{
AvailablePluginNames.Add(PluginInfo.Name);
bool ShouldIncludePlugin = (IncludePlugins.Count == 0 || IncludePlugins.Contains(PluginInfo.Name)) && !ExcludePlugins.Contains(PluginInfo.Name);
if (ShouldIncludePlugin && PluginInfo.Descriptor.LocalizationTargets != null && PluginInfo.Descriptor.LocalizationTargets.Length > 0)
{
var RootRelativePluginPath = PluginInfo.Directory.MakeRelativeTo(new DirectoryReference(UEProjectRoot));
RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4
var PluginTargetNames = new List <string>();
foreach (var LocalizationTarget in PluginInfo.Descriptor.LocalizationTargets)
{
PluginTargetNames.Add(LocalizationTarget.Name);
}
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, PluginInfo.Name, PluginTargetNames));
}
}
// If we had an explicit list of plugins to include, warn if any were missing
foreach (string PluginName in IncludePlugins)
{
if (!AvailablePluginNames.Contains(PluginName))
{
LogWarning("The plugin '{0}' specified by -IncludePlugins wasn't found and will be skipped.", PluginName);
}
}
}
// Create a single changelist to use for all changes
int PendingChangeList = 0;
if (P4Enabled)
{
var ChangeListCommitMessage = String.Format("Localization Automation using CL {0}", P4Env.Changelist);
if (File.Exists(CombinePaths(CmdEnv.LocalRoot, @"Engine/Restricted/NotForLicensees/Build/EpicInternal.txt")))
{
ChangeListCommitMessage += "\n#okforgithub ignore";
}
PendingChangeList = P4.CreateChange(P4Env.Client, ChangeListCommitMessage);
}
// Prepare to process each localization batch
var LocalizationTasks = new List <LocalizationTask>();
foreach (var LocalizationBatch in LocalizationBatches)
{
var LocalizationTask = new LocalizationTask(LocalizationBatch, UEProjectRoot, LocalizationProviderName, PendingChangeList, this);
LocalizationTasks.Add(LocalizationTask);
// Make sure the Localization configs and content is up-to-date to ensure we don't get errors later on
if (P4Enabled)
{
LogInformation("Sync necessary content to head revision");
P4.Sync(P4Env.Branch + "/" + LocalizationTask.Batch.LocalizationTargetDirectory + "/Config/Localization/...");
P4.Sync(P4Env.Branch + "/" + LocalizationTask.Batch.LocalizationTargetDirectory + "/Content/Localization/...");
}
// Generate the info we need to gather for each project
foreach (var ProjectName in LocalizationTask.Batch.LocalizationProjectNames)
{
LocalizationTask.ProjectInfos.Add(GenerateProjectInfo(LocalizationTask.RootLocalizationTargetDirectory, ProjectName, LocalizationStepNames));
}
}
// Hash the current PO files on disk so we can work out whether they actually change
Dictionary <string, byte[]> InitalPOFileHashes = null;
if (P4Enabled)
{
InitalPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot);
}
// Download the latest translations from our localization provider
if (LocalizationStepNames.Contains("Download"))
{
foreach (var LocalizationTask in LocalizationTasks)
{
if (LocalizationTask.LocProvider != null)
{
foreach (var ProjectInfo in LocalizationTask.ProjectInfos)
{
LocalizationTask.LocProvider.DownloadProjectFromLocalizationProvider(ProjectInfo.ProjectName, ProjectInfo.ImportInfo);
}
}
}
}
// Begin the gather command for each task
// These can run in parallel when ParallelGather is enabled
{
var EditorExe = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/Win64/UE4Editor-Cmd.exe");
// Set the common basic editor arguments
var EditorArguments = P4Enabled
? String.Format("-SCCProvider=Perforce -P4Port={0} -P4User={1} -P4Client={2} -P4Passwd={3} -P4Changelist={4} -EnableSCC -DisableSCCSubmit", P4Env.ServerAndPort, P4Env.User, P4Env.Client, P4.GetAuthenticationToken(), PendingChangeList)
: "-SCCProvider=None";
if (IsBuildMachine)
{
EditorArguments += " -BuildMachine";
}
EditorArguments += " -Unattended -LogLocalizationConflicts";
if (EnableParallelGather)
{
EditorArguments += " -multiprocess";
}
if (!String.IsNullOrEmpty(AdditionalCommandletArguments))
{
EditorArguments += " " + AdditionalCommandletArguments;
}
// Set the common process run options
var CommandletRunOptions = ERunOptions.Default | ERunOptions.NoLoggingOfRunCommand; // Disable logging of the run command as it will print the exit code which GUBP can pick up as an error (we do that ourselves later)
if (EnableParallelGather)
{
CommandletRunOptions |= ERunOptions.NoWaitForExit;
}
foreach (var LocalizationTask in LocalizationTasks)
{
var ProjectArgument = String.IsNullOrEmpty(UEProjectName) ? "" : String.Format("\"{0}\"", Path.Combine(LocalizationTask.RootWorkingDirectory, String.Format("{0}.uproject", UEProjectName)));
foreach (var ProjectInfo in LocalizationTask.ProjectInfos)
{
var LocalizationConfigFiles = new List <string>();
foreach (var LocalizationStep in ProjectInfo.LocalizationSteps)
{
if (LocalizationStepNames.Contains(LocalizationStep.Name))
{
LocalizationConfigFiles.Add(LocalizationStep.LocalizationConfigFile);
}
}
if (LocalizationConfigFiles.Count > 0)
{
var Arguments = String.Format("{0} -run=GatherText -config=\"{1}\" {2}", ProjectArgument, String.Join(";", LocalizationConfigFiles), EditorArguments);
LogInformation("Running localization commandlet for '{0}': {1}", ProjectInfo.ProjectName, Arguments);
LocalizationTask.GatherProcessResults.Add(Run(EditorExe, Arguments, null, CommandletRunOptions));
}
else
{
LocalizationTask.GatherProcessResults.Add(null);
}
}
}
}
// Wait for each commandlet process to finish and report the result.
// This runs even for non-parallel execution to log the exit state of the process.
foreach (var LocalizationTask in LocalizationTasks)
{
for (int ProjectIndex = 0; ProjectIndex < LocalizationTask.ProjectInfos.Count; ++ProjectIndex)
{
var ProjectInfo = LocalizationTask.ProjectInfos[ProjectIndex];
var RunResult = LocalizationTask.GatherProcessResults[ProjectIndex];
if (RunResult != null)
{
RunResult.WaitForExit();
RunResult.OnProcessExited();
RunResult.DisposeProcess();
if (RunResult.ExitCode == 0)
{
LogInformation("The localization commandlet for '{0}' exited with code 0.", ProjectInfo.ProjectName);
}
else
{
LogWarning("The localization commandlet for '{0}' exited with code {1} which likely indicates a crash.", ProjectInfo.ProjectName, RunResult.ExitCode);
}
}
}
}
// Upload the latest sources to our localization provider
if (LocalizationStepNames.Contains("Upload"))
{
foreach (var LocalizationTask in LocalizationTasks)
{
if (LocalizationTask.LocProvider != null)
{
// Upload all text to our localization provider
for (int ProjectIndex = 0; ProjectIndex < LocalizationTask.ProjectInfos.Count; ++ProjectIndex)
{
var ProjectInfo = LocalizationTask.ProjectInfos[ProjectIndex];
var RunResult = LocalizationTask.GatherProcessResults[ProjectIndex];
if (RunResult != null && RunResult.ExitCode == 0)
{
// Recalculate the split platform paths before doing the upload, as the export may have changed them
ProjectInfo.ExportInfo.CalculateSplitPlatformNames(LocalizationTask.RootLocalizationTargetDirectory);
LocalizationTask.LocProvider.UploadProjectToLocalizationProvider(ProjectInfo.ProjectName, ProjectInfo.ExportInfo);
}
else
{
LogWarning("Skipping upload to the localization provider for '{0}' due to an earlier commandlet failure.", ProjectInfo.ProjectName);
}
}
}
}
}
// Clean-up the changelist so it only contains the changed files, and then submit it (if we were asked to)
if (P4Enabled)
{
// Revert any PO files that haven't changed aside from their header
{
var POFilesToRevert = new List <string>();
var CurrentPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot);
foreach (var CurrentPOFileHashPair in CurrentPOFileHashes)
{
byte[] InitialPOFileHash;
if (InitalPOFileHashes.TryGetValue(CurrentPOFileHashPair.Key, out InitialPOFileHash) && InitialPOFileHash.SequenceEqual(CurrentPOFileHashPair.Value))
{
POFilesToRevert.Add(CurrentPOFileHashPair.Key);
}
}
if (POFilesToRevert.Count > 0)
{
var P4RevertArgsFilename = CombinePaths(CmdEnv.LocalRoot, "Engine", "Intermediate", String.Format("LocalizationP4RevertArgs-{0}.txt", Guid.NewGuid().ToString()));
using (StreamWriter P4RevertArgsWriter = File.CreateText(P4RevertArgsFilename))
{
foreach (var POFileToRevert in POFilesToRevert)
{
P4RevertArgsWriter.WriteLine(POFileToRevert);
}
}
P4.LogP4(String.Format("-x{0} revert", P4RevertArgsFilename));
DeleteFile_NoExceptions(P4RevertArgsFilename);
}
}
// Revert any other unchanged files
P4.RevertUnchanged(PendingChangeList);
// Submit that single changelist now
if (AllowSubmit)
{
int SubmittedChangeList;
P4.Submit(PendingChangeList, out SubmittedChangeList);
}
}
var RunDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds;
LogInformation("Localize command finished in {0} seconds", RunDuration / 1000);
}
public override void ExecuteBuild()
{
Log("************************* Build Third Party Libs");
// figure out what batch/script to run
string CompileScriptName;
switch (UnrealBuildTool.BuildHostPlatform.Current.Platform)
{
case UnrealTargetPlatform.Win64:
CompileScriptName = WindowsCompileScript;
break;
case UnrealTargetPlatform.Mac:
CompileScriptName = MacCompileScript;
break;
case UnrealTargetPlatform.Linux:
CompileScriptName = LinuxCompileScript;
break;
default:
throw new AutomationException("Unknown runtime platform!");
}
// look for changelist on the command line
int WorkingCL = Int32.Parse(ParseParamValue("Changelist", "-1"));
// if not specified, make one
if (WorkingCL == -1)
{
WorkingCL = P4.CreateChange(P4Env.Client, String.Format("Third party libs built from changelist {0}", P4Env.Changelist));
}
Log("Build from {0} Working in {1}", P4Env.Changelist, WorkingCL);
// go to the third party lib dir
string SearchLibraryDir = ParseParamValue("SearchDir", DefaultLibraryDir);
CommandUtils.PushDir(SearchLibraryDir);
// figure out what libraries to compile
string LibsToCompileString = ParseParamValue("Libs");
// hunt down build batch files if the caller didn't specify a list to compile
List <string> LibsToCompile = new List <string>();
if (string.IsNullOrEmpty(LibsToCompileString))
{
// loop over third party directories looking for the right batch files
foreach (string Dir in Directory.EnumerateDirectories("."))
{
if (File.Exists(Path.Combine(Dir, CompileScriptName)))
{
LibsToCompile.Add(Path.GetFileName(Dir));
}
}
}
else
{
// just split up the param and make sure the batch file exists
string[] Libs = LibsToCompileString.Split('+');
bool bHadError = false;
foreach (string Dir in Libs)
{
if (File.Exists(Path.Combine(Dir, CompileScriptName)))
{
LibsToCompile.Add(Path.GetFileName(Dir));
}
else
{
LogError("Error: Requested lib {0} does not have a {1}", Dir, CompileScriptName);
bHadError = true;
}
}
if (bHadError)
{
// error out so that we don't fail to build some and have it lost in the noise
throw new AutomationException("One or more libs were not set up to compile.");
}
}
// set an envvar so that the inner batch files can check out files into a shared changelist
Environment.SetEnvironmentVariable("THIRD_PARTY_CHANGELIST", string.Format("-c {0}", WorkingCL));
// now go through and run each batch file,
foreach (string Lib in LibsToCompile)
{
Log("Building {0}", Lib);
// go into the lib dir
CommandUtils.PushDir(Lib);
// run the builder batch file
CommandUtils.RunAndLog(CmdEnv, CompileScriptName, "", "ThirdPartyLib_" + Lib);
// go back to ThirdParty dir
CommandUtils.PopDir();
}
// undo the SearchLibraryDir push
CommandUtils.PopDir();
PrintRunTime();
// revert any unchanged files
P4.RevertUnchanged(WorkingCL);
if (AllowSubmit)
{
int SubmittedCL;
P4.Submit(WorkingCL, out SubmittedCL, true, true);
Log("Submitted changelist {0}", SubmittedCL);
}
}
public override void ExecuteBuild()
{
int WorkingCL = -1;
if (P4Enabled && AllowSubmit)
{
string CmdLine = "";
foreach (var Arg in Params)
{
CmdLine += Arg.ToString() + " ";
}
WorkingCL = P4.CreateChange(P4Env.Client, String.Format("MegaXGE build from changelist {0} - Params: {1}", P4Env.Changelist, CmdLine));
}
LogInformation("************************* MegaXGE");
bool Clean = ParseParam("Clean");
string CleanToolLocation = CombinePaths(CmdEnv.LocalRoot, "Engine", "Build", "Batchfiles", "Clean.bat");
bool ShowProgress = ParseParam("Progress");
var UE4Build = new UE4Build(this);
var Agenda = new UE4Build.BuildAgenda();
// we need to always build UHT when we use mega XGE
var ProgramTargets = new string[]
{
"UnrealHeaderTool",
};
Agenda.AddTargets(ProgramTargets, UnrealTargetPlatform.Win64, UnrealTargetConfiguration.Development);
if (Clean)
{
LogSetProgress(ShowProgress, "Cleaning previous builds...");
foreach (var CurTarget in ProgramTargets)
{
string Args = String.Format("{0} {1} {2}", CurTarget, UnrealTargetPlatform.Win64.ToString(), UnrealTargetConfiguration.Development.ToString());
RunAndLog(CmdEnv, CleanToolLocation, Args);
}
}
LogInformation("*************************");
for (int Arg = 1; Arg < 100; Arg++)
{
string Parm = String.Format("Target{0}", Arg);
string Target = ParseParamValue(Parm, "");
if (String.IsNullOrEmpty(Target))
{
break;
}
FileReference ProjectFile = null;
string ProjectFileParam = ParseParamValue(String.Format("Project{0}", Arg), null);
if (ProjectFileParam != null)
{
ProjectFile = new FileReference(ProjectFileParam);
if (!FileReference.Exists(ProjectFile))
{
throw new AutomationException("Project file '{0}' could not be found");
}
}
var Parts = Target.Split(' ');
string JustTarget = Parts[0];
if (String.IsNullOrEmpty(JustTarget))
{
throw new AutomationException("BUILD FAILED target option '{0}' not parsed.", Target);
}
var Targets = JustTarget.Split('|');
if (Targets.Length < 1)
{
throw new AutomationException("BUILD FAILED target option '{0}' not parsed.", Target);
}
var Platforms = new List <UnrealTargetPlatform>();
var Configurations = new List <UnrealTargetConfiguration>();
for (int Part = 1; Part < Parts.Length; Part++)
{
if (!String.IsNullOrEmpty(Parts[Part]))
{
var SubParts = Parts[Part].Split('|');
foreach (var SubPart in SubParts)
{
UnrealTargetPlatform Platform;
if (Enum.TryParse(SubPart, true, out Platform))
{
Platforms.Add(Platform);
}
else
{
switch (SubPart.ToUpperInvariant())
{
case "DEBUG":
Configurations.Add(UnrealTargetConfiguration.Debug);
break;
case "DEBUGGAME":
Configurations.Add(UnrealTargetConfiguration.DebugGame);
break;
case "DEVELOPMENT":
Configurations.Add(UnrealTargetConfiguration.Development);
break;
case "SHIPPING":
Configurations.Add(UnrealTargetConfiguration.Shipping);
break;
case "TEST":
Configurations.Add(UnrealTargetConfiguration.Test);
break;
default:
throw new AutomationException("BUILD FAILED target option {0} not recognized.", SubPart);
}
}
}
}
}
if (Platforms.Count < 1)
{
Platforms.Add(UnrealTargetPlatform.Win64);
}
if (Configurations.Count < 1)
{
Configurations.Add(UnrealTargetConfiguration.Development);
}
foreach (var Platform in Platforms)
{
foreach (var CurTarget in Targets)
{
foreach (var Configuration in Configurations)
{
Agenda.AddTargets(new string[] { CurTarget }, Platform, Configuration, ProjectFile);
LogInformation("Target {0} {1} {2}", CurTarget, Platform.ToString(), Configuration.ToString());
if (Clean)
{
string Args = String.Format("{0} {1} {2}", CurTarget, Platform.ToString(), Configuration.ToString());
RunAndLog(CmdEnv, CleanToolLocation, Args);
}
}
}
}
}
LogInformation("*************************");
Agenda.DoRetries = ParseParam("Retry");
UE4Build.Build(Agenda, InUpdateVersionFiles: IsBuildMachine, InUseParallelExecutor: ParseParam("useparallelexecutor"), InShowProgress: ShowProgress);
// if (WorkingCL > 0) // only move UAT files if we intend to check in some build products
// {
// UE4Build.CopyUATFilesAndAddToBuildProducts();
// }
UE4Build.CheckBuildProducts(UE4Build.BuildProductFiles);
if (WorkingCL > 0)
{
// Sign everything we built
CodeSign.SignMultipleIfEXEOrDLL(this, UE4Build.BuildProductFiles);
// Open files for add or edit
UE4Build.AddBuildProductsToChangelist(WorkingCL, UE4Build.BuildProductFiles);
int SubmittedCL;
P4.Submit(WorkingCL, out SubmittedCL, true, true);
}
PrintRunTime();
}
public override void ExecuteBuild()
{
string BucketName = ParseRequiredStringParam("Bucket");
FileReference CredentialsFile = ParseRequiredFileReferenceParam("CredentialsFile");
string CredentialsKey = ParseRequiredStringParam("CredentialsKey");
DirectoryReference CacheDir = ParseRequiredDirectoryReferenceParam("CacheDir");
DirectoryReference FilterDir = ParseRequiredDirectoryReferenceParam("FilterDir");
int Days = ParseParamInt("Days", 7);
int MaxFileSize = ParseParamInt("MaxFileSize", 0);
string RootManifestPath = ParseRequiredStringParam("Manifest");
string KeyPrefix = ParseParamValue("KeyPrefix", "");
bool bReset = ParseParam("Reset");
// The credentials to upload with
AWSCredentials Credentials;
// Try to get the credentials by the key passed in from the script
CredentialProfileStoreChain CredentialsChain = new CredentialProfileStoreChain(CredentialsFile.FullName);
if (!CredentialsChain.TryGetAWSCredentials(CredentialsKey, out Credentials))
{
throw new AutomationException("Unknown credentials key: {0}", CredentialsKey);
}
// Create the new client
using (AmazonS3Client Client = new AmazonS3Client(Credentials, Region))
{
using (SemaphoreSlim RequestSemaphore = new SemaphoreSlim(4))
{
// Read the filters
HashSet <string> Paths = new HashSet <string>();
foreach (FileInfo FilterFile in FilterDir.ToDirectoryInfo().EnumerateFiles("*.txt"))
{
TimeSpan Age = DateTime.UtcNow - FilterFile.LastWriteTimeUtc;
if (Age < TimeSpan.FromDays(3))
{
Log.TraceInformation("Reading {0}", FilterFile.FullName);
string[] Lines = File.ReadAllLines(FilterFile.FullName);
foreach (string Line in Lines)
{
string TrimLine = Line.Trim().Replace('\\', '/');
if (TrimLine.Length > 0)
{
Paths.Add(TrimLine);
}
}
}
else if (Age > TimeSpan.FromDays(5))
{
try
{
Log.TraceInformation("Deleting {0}", FilterFile.FullName);
FilterFile.Delete();
}
catch (Exception Ex)
{
Log.TraceWarning("Unable to delete: {0}", Ex.Message);
Log.TraceLog(ExceptionUtils.FormatExceptionDetails(Ex));
}
}
}
Log.TraceInformation("Found {0:n0} files", Paths.Count);
// Enumerate all the files that are in the network DDC
Log.TraceInformation("");
Log.TraceInformation("Filtering files in {0}...", CacheDir);
List <DerivedDataFile> Files = ParallelExecute <string, DerivedDataFile>(Paths, (Path, FilesBag) => ReadFileInfo(CacheDir, Path, FilesBag));
// Filter to the maximum size
if (MaxFileSize != 0)
{
int NumRemovedMaxSize = Files.RemoveAll(x => x.Info.Length > MaxFileSize);
Log.TraceInformation("");
Log.TraceInformation("Removed {0} files above size limit ({1:n0} bytes)", NumRemovedMaxSize, MaxFileSize);
}
// Create the working directory
DirectoryReference WorkingDir = DirectoryReference.Combine(EngineDirectory, "Saved", "UploadDDC");
DirectoryReference.CreateDirectory(WorkingDir);
// Get the path to the manifest
FileReference RootManifestFile = FileReference.Combine(CommandUtils.RootDirectory, RootManifestPath);
// Read the old root manifest
RootManifest OldRootManifest = new RootManifest();
if (FileReference.Exists(RootManifestFile))
{
OldRootManifest.Read(JsonObject.Read(RootManifestFile));
}
// Read the old bundle manifest
BundleManifest OldBundleManifest = new BundleManifest();
if (OldRootManifest.Entries.Count > 0)
{
FileReference LocalBundleManifest = FileReference.Combine(WorkingDir, "OldBundleManifest.json");
if (TryDownloadFile(Client, BucketName, OldRootManifest.Entries.Last().Key, LocalBundleManifest))
{
OldBundleManifest.Read(JsonObject.Read(LocalBundleManifest));
}
}
// Create the new manifest
BundleManifest NewBundleManifest = new BundleManifest();
// Try to download the old manifest, and add all the bundles we want to keep to the new manifest
if (!bReset)
{
foreach (BundleManifest.Entry Bundle in OldBundleManifest.Entries)
{
FileReference BundleFile = FileReference.Combine(WorkingDir, Bundle.Name);
if (!FileReference.Exists(BundleFile))
{
Log.TraceInformation("Downloading {0}", BundleFile);
FileReference TempCompressedFile = new FileReference(BundleFile.FullName + ".incoming.gz");
if (!TryDownloadFile(Client, BucketName, Bundle.ObjectKey, TempCompressedFile))
{
Log.TraceWarning("Unable to download {0}", Bundle.ObjectKey);
continue;
}
FileReference TempUncompressedFile = new FileReference(BundleFile.FullName + ".incoming");
try
{
DecompressFile(TempCompressedFile, TempUncompressedFile);
}
catch (Exception Ex)
{
Log.TraceWarning("Unable to uncompress {0}: {1}", Bundle.ObjectKey, Ex.ToString());
continue;
}
FileReference.Move(TempUncompressedFile, BundleFile);
}
NewBundleManifest.Entries.Add(Bundle);
}
}
// Figure out all the item digests that we already have
Dictionary <BundleManifest.Entry, HashSet <ContentHash> > BundleToKeyHashes = new Dictionary <BundleManifest.Entry, HashSet <ContentHash> >();
foreach (BundleManifest.Entry Bundle in NewBundleManifest.Entries)
{
HashSet <ContentHash> KeyHashes = new HashSet <ContentHash>();
FileReference BundleFile = FileReference.Combine(WorkingDir, Bundle.Name);
using (FileStream Stream = FileReference.Open(BundleFile, FileMode.Open, FileAccess.Read, FileShare.Read))
{
BinaryReader Reader = new BinaryReader(Stream);
uint Signature = Reader.ReadUInt32();
if (Signature != BundleSignatureV1)
{
throw new Exception(String.Format("Invalid signature for {0}", BundleFile));
}
int NumEntries = Reader.ReadInt32();
for (int EntryIdx = 0; EntryIdx < NumEntries; EntryIdx++)
{
byte[] Digest = new byte[ContentHash.LengthSHA1];
if (Reader.Read(Digest, 0, ContentHash.LengthSHA1) != ContentHash.LengthSHA1)
{
throw new Exception("Unexpected EOF");
}
KeyHashes.Add(new ContentHash(Digest));
Stream.Seek(4, SeekOrigin.Current);
}
}
BundleToKeyHashes[Bundle] = KeyHashes;
}
// Calculate the download size of the manifest
long DownloadSize = NewBundleManifest.Entries.Sum(x => (long)x.CompressedLength);
// Remove any bundles which have less than the minimum required size in valid data. We don't mark the manifest as dirty yet; these
// files will only be rewritten if new content is added, to prevent the last bundle being rewritten multiple times.
foreach (KeyValuePair <BundleManifest.Entry, HashSet <ContentHash> > Pair in BundleToKeyHashes)
{
long ValidBundleSize = Files.Where(x => Pair.Value.Contains(x.KeyHash)).Sum(x => (long)x.Info.Length);
if (ValidBundleSize < MinBundleSize)
{
NewBundleManifest.Entries.Remove(Pair.Key);
}
}
// Find all the valid digests
HashSet <ContentHash> ReusedKeyHashes = new HashSet <ContentHash>();
foreach (BundleManifest.Entry Bundle in NewBundleManifest.Entries)
{
ReusedKeyHashes.UnionWith(BundleToKeyHashes[Bundle]);
}
// Remove all the files which already exist
int NumRemovedExist = Files.RemoveAll(x => ReusedKeyHashes.Contains(x.KeyHash));
if (NumRemovedExist > 0)
{
Log.TraceInformation("");
Log.TraceInformation("Removed {0:n0} files which already exist", NumRemovedExist);
}
// Read all the files we want to include
List <Tuple <DerivedDataFile, byte[]> > FilesToInclude = new List <Tuple <DerivedDataFile, byte[]> >();
if (Files.Count > 0)
{
Log.TraceInformation("");
Log.TraceInformation("Reading remaining {0:n0} files into memory ({1:n1}mb)...", Files.Count, (float)Files.Sum(x => (long)x.Info.Length) / (1024 * 1024));
FilesToInclude.AddRange(ParallelExecute <DerivedDataFile, Tuple <DerivedDataFile, byte[]> >(Files, (x, y) => ReadFileData(x, y)));
}
// Generate new data
using (RNGCryptoServiceProvider Crypto = new RNGCryptoServiceProvider())
{
// Flag for whether to update the manifest
bool bUpdateManifest = false;
// Upload the new bundle
Log.TraceInformation("");
if (FilesToInclude.Count == 0)
{
Log.TraceInformation("No new files to add.");
}
else
{
// Sort the files to include by creation time. This will bias towards grouping older, more "permanent", items together.
Log.TraceInformation("Sorting input files");
List <Tuple <DerivedDataFile, byte[]> > SortedFilesToInclude = FilesToInclude.OrderBy(x => x.Item1.Info.CreationTimeUtc).ToList();
// Get the target bundle size
long TotalSize = SortedFilesToInclude.Sum(x => (long)x.Item2.Length);
int NumBundles = (int)((TotalSize + (MaxBundleSize - 1)) / MaxBundleSize);
long TargetBundleSize = TotalSize / NumBundles;
// Split the input data into bundles
List <List <Tuple <DerivedDataFile, byte[]> > > BundleFilesToIncludeList = new List <List <Tuple <DerivedDataFile, byte[]> > >();
long BundleSize = 0;
for (int FileIdx = 0; FileIdx < SortedFilesToInclude.Count; BundleSize = BundleSize % TargetBundleSize)
{
List <Tuple <DerivedDataFile, byte[]> > BundleFilesToInclude = new List <Tuple <DerivedDataFile, byte[]> >();
for (; BundleSize < TargetBundleSize && FileIdx < SortedFilesToInclude.Count; FileIdx++)
{
BundleFilesToInclude.Add(SortedFilesToInclude[FileIdx]);
BundleSize += SortedFilesToInclude[FileIdx].Item2.Length;
}
BundleFilesToIncludeList.Add(BundleFilesToInclude);
}
// Upload each bundle
DateTime NewBundleTime = DateTime.UtcNow;
for (int BundleIdx = 0; BundleIdx < BundleFilesToIncludeList.Count; BundleIdx++)
{
List <Tuple <DerivedDataFile, byte[]> > BundleFilesToInclude = BundleFilesToIncludeList[BundleIdx];
// Get the new bundle info
string NewBundleSuffix = (BundleFilesToIncludeList.Count > 1) ? String.Format("-{0}_of_{1}", BundleIdx + 1, BundleFilesToIncludeList.Count) : "";
string NewBundleName = String.Format("Bundle-{0:yyyy.MM.dd-HH.mm}{1}.ddb", NewBundleTime.ToLocalTime(), NewBundleSuffix);
// Create a random number for the object key
string NewBundleObjectKey = KeyPrefix + "bulk/" + CreateObjectName(Crypto);
// Create the bundle header
byte[] Header;
using (MemoryStream HeaderStream = new MemoryStream())
{
BinaryWriter Writer = new BinaryWriter(HeaderStream);
Writer.Write(BundleSignatureV1);
Writer.Write(BundleFilesToInclude.Count);
foreach (Tuple <DerivedDataFile, byte[]> FileToInclude in BundleFilesToInclude)
{
Writer.Write(FileToInclude.Item1.KeyHash.Bytes, 0, ContentHash.LengthSHA1);
Writer.Write((int)FileToInclude.Item2.Length);
}
Header = HeaderStream.ToArray();
}
// Create the output file
FileReference NewBundleFile = FileReference.Combine(WorkingDir, NewBundleName + ".gz");
Log.TraceInformation("Writing {0}", NewBundleFile);
using (FileStream BundleStream = FileReference.Open(NewBundleFile, FileMode.Create, FileAccess.Write, FileShare.Read))
{
using (GZipStream ZipStream = new GZipStream(BundleStream, CompressionLevel.Optimal, true))
{
ZipStream.Write(Header, 0, Header.Length);
foreach (Tuple <DerivedDataFile, byte[]> FileToInclude in BundleFilesToInclude)
{
ZipStream.Write(FileToInclude.Item2, 0, FileToInclude.Item2.Length);
}
}
}
// Upload the file
long NewBundleCompressedLength = NewBundleFile.ToFileInfo().Length;
long NewBundleUncompressedLength = Header.Length + BundleFilesToInclude.Sum(x => (long)x.Item2.Length);
Log.TraceInformation("Uploading bundle to {0} ({1:n1}mb)", NewBundleObjectKey, NewBundleCompressedLength / (1024.0f * 1024.0f));
UploadFile(Client, BucketName, NewBundleFile, 0, NewBundleObjectKey, RequestSemaphore, null);
// Add the bundle to the new manifest
BundleManifest.Entry Bundle = new BundleManifest.Entry();
Bundle.Name = NewBundleName;
Bundle.ObjectKey = NewBundleObjectKey;
Bundle.Time = NewBundleTime;
Bundle.CompressedLength = (int)NewBundleCompressedLength;
Bundle.UncompressedLength = (int)NewBundleUncompressedLength;
NewBundleManifest.Entries.Add(Bundle);
// Mark the manifest as requiring an update
bUpdateManifest = true;
}
}
// Update the manifest
if (bUpdateManifest)
{
DateTime UtcNow = DateTime.UtcNow;
DateTime RemoveBundleManifestsBefore = UtcNow - TimeSpan.FromDays(3.0);
// Update the root manifest
RootManifest NewRootManifest = new RootManifest();
NewRootManifest.AccessKey = OldRootManifest.AccessKey;
NewRootManifest.SecretKey = OldRootManifest.SecretKey;
foreach (RootManifest.Entry Entry in OldRootManifest.Entries)
{
if (Entry.CreateTime >= RemoveBundleManifestsBefore)
{
NewRootManifest.Entries.Add(Entry);
}
}
// Make sure there's an entry for the last 24h
DateTime RequireBundleManifestAfter = UtcNow - TimeSpan.FromDays(1.0);
if (!NewRootManifest.Entries.Any(x => x.CreateTime > RequireBundleManifestAfter))
{
RootManifest.Entry NewEntry = new RootManifest.Entry();
NewEntry.CreateTime = UtcNow;
NewEntry.Key = KeyPrefix + CreateObjectName(Crypto);
NewRootManifest.Entries.Add(NewEntry);
}
// Save out the new bundle manifest
FileReference NewBundleManifestFile = FileReference.Combine(WorkingDir, "NewBundleManifest.json");
NewBundleManifest.Save(NewBundleManifestFile);
// Update all the bundle manifests still valid
foreach (RootManifest.Entry Entry in NewRootManifest.Entries)
{
Log.TraceInformation("Uploading bundle manifest to {0}", Entry.Key);
UploadFile(Client, BucketName, NewBundleManifestFile, 0, Entry.Key, RequestSemaphore, null);
}
// Overwrite all the existing manifests
if (AllowSubmit)
{
List <string> ExistingFiles = P4.Files(CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName));
// Create a changelist containing the new manifest
int ChangeNumber = P4.CreateChange(Description: "Updating DDC bundle manifest");
if (ExistingFiles.Count > 0)
{
P4.Edit(ChangeNumber, CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName));
NewRootManifest.Save(RootManifestFile);
}
else
{
NewRootManifest.Save(RootManifestFile);
P4.Add(ChangeNumber, CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName));
}
// Submit it
int SubmittedChangeNumber;
P4.Submit(ChangeNumber, out SubmittedChangeNumber, true);
if (SubmittedChangeNumber <= 0)
{
throw new AutomationException("Failed to submit change");
}
// Delete any bundles that are no longer referenced
HashSet <string> KeepObjectKeys = new HashSet <string>(NewBundleManifest.Entries.Select(x => x.ObjectKey));
foreach (BundleManifest.Entry OldEntry in OldBundleManifest.Entries)
{
if (!KeepObjectKeys.Contains(OldEntry.ObjectKey))
{
Log.TraceInformation("Deleting unreferenced bundle {0}", OldEntry.ObjectKey);
DeleteFile(Client, BucketName, OldEntry.ObjectKey, RequestSemaphore, null);
}
}
// Delete any bundle manifests which are no longer referenced
HashSet <string> KeepManifestKeys = new HashSet <string>(NewRootManifest.Entries.Select(x => x.Key));
foreach (RootManifest.Entry OldEntry in OldRootManifest.Entries)
{
if (!KeepManifestKeys.Contains(OldEntry.Key))
{
Log.TraceInformation("Deleting unreferenced manifest {0}", OldEntry.Key);
DeleteFile(Client, BucketName, OldEntry.Key, RequestSemaphore, null);
}
}
}
else
{
// Skip submitting
Log.TraceWarning("Skipping manifest submit due to missing -Submit argument.");
}
// Update the new download size
DownloadSize = NewBundleManifest.Entries.Sum(x => (long)x.CompressedLength);
}
}
// Print some stats about the final manifest
Log.TraceInformation("");
Log.TraceInformation("Total download size {0:n1}mb", DownloadSize / (1024.0 * 1024.0));
Log.TraceInformation("");
}
}
}
public override void ExecuteBuild()
{
SetupStaticBuildEnvironment();
bool bBuildSolutions = true;
if (ParseParam("SkipBuildSolutions"))
{
bBuildSolutions = false;
}
bool bBuildLibraries = true;
if (ParseParam("SkipBuild"))
{
bBuildLibraries = false;
}
bool bAutoCreateChangelist = true;
if (ParseParam("SkipCreateChangelist"))
{
bAutoCreateChangelist = false;
}
bool bAutoSubmit = false; // bAutoCreateChangelist;
if (ParseParam("SkipSubmit"))
{
bAutoSubmit = false;
}
// if we don't pass anything, we'll just merge by default
string RobomergeCommand = ParseParamValue("Robomerge", "").ToLower();
if (!string.IsNullOrEmpty(RobomergeCommand))
{
// for merge default action, add flag to make sure buildmachine commit isn't skipped
if (RobomergeCommand == "merge")
{
RobomergeCommand = "#robomerge[all] #DisregardExcludedAuthors";
}
// otherwise add hashtags
else if (RobomergeCommand == "ignore")
{
RobomergeCommand = "#robomerge #ignore";
}
else if (RobomergeCommand == "null")
{
RobomergeCommand = "#robomerge #null";
}
// otherwise the submit will likely fail.
else
{
throw new AutomationException("Invalid Robomerge param passed in {0}. Must be \"merge\", \"null\", or \"ignore\"", RobomergeCommand);
}
}
// get the platforms we want to build for
List <TargetPlatformData> TargetPlatforms = GetTargetPlatforms();
// get the platforms we want to build for
List <WindowsCompiler> TargetWindowsCompilers = GetTargetWindowsCompilers();
// get the configurations we want to build for
List <string> TargetConfigurations = GetTargetConfigurations();
if (bBuildSolutions)
{
// build target lib for all platforms
foreach (TargetPlatformData TargetData in TargetPlatforms)
{
if (!PlatformSupportsTargetLib(TargetData))
{
continue;
}
SetupBuildForTargetLibAndPlatform(TargetData, TargetConfigurations, TargetWindowsCompilers, false);
}
}
HashSet <FileReference> FilesToReconcile = new HashSet <FileReference>();
if (bBuildLibraries)
{
// build target lib for all platforms
foreach (TargetPlatformData TargetData in TargetPlatforms)
{
if (!PlatformSupportsTargetLib(TargetData))
{
continue;
}
HashSet <FileReference> FilesToDelete = new HashSet <FileReference>();
foreach (string TargetConfiguration in TargetConfigurations)
{
// Delete output files before building them
if (TargetData.Platform == UnrealTargetPlatform.Win64)
{
foreach (WindowsCompiler TargetCompiler in TargetWindowsCompilers)
{
FindOutputFiles(FilesToDelete, TargetData, TargetConfiguration, TargetCompiler);
}
}
else
{
FindOutputFiles(FilesToDelete, TargetData, TargetConfiguration);
}
}
foreach (FileReference FileToDelete in FilesToDelete)
{
FilesToReconcile.Add(FileToDelete);
InternalUtils.SafeDeleteFile(FileToDelete.ToString());
}
BuildTargetLibForPlatform(TargetData, TargetConfigurations, TargetWindowsCompilers);
if (DoesPlatformUseMSBuild(TargetData))
{
foreach (WindowsCompiler TargetWindowsCompiler in TargetWindowsCompilers)
{
CopyLibsToFinalDestination(TargetData, TargetConfigurations, TargetWindowsCompiler);
}
}
else
{
CopyLibsToFinalDestination(TargetData, TargetConfigurations);
}
}
}
int P4ChangeList = InvalidChangeList;
if (bAutoCreateChangelist)
{
string RobomergeLine = string.Empty;
if (!string.IsNullOrEmpty(RobomergeCommand))
{
RobomergeLine = Environment.NewLine + RobomergeCommand;
}
P4ChangeList = P4.CreateChange(P4Env.Client, "BuildHlslcc.Automation: Deploying hlslcc libs." + Environment.NewLine + "#rb none" + Environment.NewLine + "#lockdown Nick.Penwarden" + Environment.NewLine + "#tests none" + Environment.NewLine + "#jira none" + Environment.NewLine + "#okforgithub ignore" + RobomergeLine);
}
if (P4ChangeList != InvalidChangeList)
{
foreach (string TargetConfiguration in TargetConfigurations)
{
//Add any new files that p4 is not yet tracking.
foreach (TargetPlatformData TargetData in TargetPlatforms)
{
if (!PlatformSupportsTargetLib(TargetData))
{
continue;
}
if (TargetData.Platform == UnrealTargetPlatform.Win64)
{
foreach (WindowsCompiler TargetCompiler in TargetWindowsCompilers)
{
FindOutputFiles(FilesToReconcile, TargetData, TargetConfiguration, TargetCompiler);
}
}
else
{
FindOutputFiles(FilesToReconcile, TargetData, TargetConfiguration);
}
}
}
foreach (FileReference FileToReconcile in FilesToReconcile)
{
P4.Reconcile(P4ChangeList, FileToReconcile.ToString());
}
}
if (bAutoSubmit && (P4ChangeList != InvalidChangeList))
{
if (!P4.TryDeleteEmptyChange(P4ChangeList))
{
LogInformation("Submitting changelist " + P4ChangeList.ToString());
int SubmittedChangeList = InvalidChangeList;
P4.Submit(P4ChangeList, out SubmittedChangeList);
}
else
{
LogInformation("Nothing to submit!");
}
}
}
