/// <summary>
/// Execute the task and run the cooker.
/// </summary>
/// <param name="BuildProducts">List of build products for the current node. Cooking build products will be appended to this list.</param>
/// <returns>True if the node succeeded</returns>
public override bool Execute(List <string> BuildProducts)
{
// Figure out the project that this target belongs to
FileReference ProjectFile;
if (UProjectInfo.TryGetProjectForTarget(Target, out ProjectFile))
{
ProjectFile = null;
}
// Execute the cooker
using (TelemetryStopwatch CookStopwatch = new TelemetryStopwatch("Cook.{0}.{1}", ProjectFile.GetFileNameWithoutExtension(), CookPlatform))
{
CommandUtils.CookCommandlet(ProjectFile, "UE4Editor-Cmd.exe", Maps, null, null, null, CookPlatform, Arguments);
}
// Find all the cooked files
DirectoryReference CookedDirectory = DirectoryReference.Combine(ProjectFile.Directory, "Saved", "Cooked", CookPlatform);
List <FileReference> CookedFiles = CookedDirectory.EnumerateFileReferences().ToList();
if (CookedFiles.Count == 0)
{
throw new AutomationException("Cooking did not produce any files in {0}", CookedDirectory.FullName);
}
BuildProducts.AddRange(CookedFiles.Select(x => x.FullName));
return(true);
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
public override void Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Figure out the project that this target belongs to
FileReference ProjectFile = null;
if (Parameters.Project != null)
{
ProjectFile = new FileReference(Parameters.Project);
if (!FileReference.Exists(ProjectFile))
{
throw new AutomationException("Missing project file - {0}", ProjectFile.FullName);
}
}
// Execute the cooker
using (TelemetryStopwatch CookStopwatch = new TelemetryStopwatch("Cook.{0}.{1}", (ProjectFile == null)? "UE4" : ProjectFile.GetFileNameWithoutExtension(), Parameters.Platform))
{
string[] Maps = (Parameters.Maps == null)? null : Parameters.Maps.Split(new char[] { '+' });
string Arguments = (Parameters.Versioned ? "" : "-Unversioned ") + "-LogCmds=\"LogSavePackage Warning\" " + Parameters.Arguments;
CommandUtils.CookCommandlet(ProjectFile, "UE4Editor-Cmd.exe", Maps, null, null, null, Parameters.Platform, Arguments);
}
// Find all the cooked files
List <FileReference> CookedFiles = new List <FileReference>();
foreach (string Platform in Parameters.Platform.Split('+'))
{
DirectoryReference PlatformCookedDirectory = DirectoryReference.Combine(ProjectFile.Directory, "Saved", "Cooked", Platform);
if (!DirectoryReference.Exists(PlatformCookedDirectory))
{
throw new AutomationException("Cook output directory not found ({0})", PlatformCookedDirectory.FullName);
}
List <FileReference> PlatformCookedFiles = DirectoryReference.EnumerateFiles(PlatformCookedDirectory, "*", System.IO.SearchOption.AllDirectories).ToList();
if (PlatformCookedFiles.Count == 0)
{
throw new AutomationException("Cooking did not produce any files in {0}", PlatformCookedDirectory.FullName);
}
CookedFiles.AddRange(PlatformCookedFiles);
DirectoryReference PackagingFilesDirectory = DirectoryReference.Combine(ProjectFile.Directory, "Saved", "TmpPackaging", Platform);
if (DirectoryReference.Exists(PackagingFilesDirectory))
{
List <FileReference> PackagingFiles = DirectoryReference.EnumerateFiles(PackagingFilesDirectory, "*", System.IO.SearchOption.AllDirectories).ToList();
CookedFiles.AddRange(PackagingFiles);
}
}
// Apply the optional tag to the build products
foreach (string TagName in FindTagNamesFromList(Parameters.Tag))
{
FindOrAddTagSet(TagNameToFileSet, TagName).UnionWith(CookedFiles);
}
// Add them to the set of build products
BuildProducts.UnionWith(CookedFiles);
}
public BranchInfo(List <UnrealTargetPlatform> InHostPlatforms)
{
BaseEngineProject = new BranchUProject();
var AllProjects = UnrealBuildTool.UProjectInfo.AllProjectFiles;
using (TelemetryStopwatch SortProjectsStopwatch = new TelemetryStopwatch("SortProjects"))
{
foreach (var InfoEntry in AllProjects)
{
var UProject = new BranchUProject(InfoEntry);
if (UProject.Properties.bIsCodeBasedProject)
{
CodeProjects.Add(UProject);
}
else
{
NonCodeProjects.Add(UProject);
// the base project uses BlankProject if it really needs a .uproject file
if (BaseEngineProject.FilePath == null && UProject.GameName == "BlankProject")
{
BaseEngineProject.FilePath = UProject.FilePath;
}
}
}
}
/* if (String.IsNullOrEmpty(BaseEngineProject.FilePath))
* {
* throw new AutomationException("All branches must have the blank project /Samples/Sandbox/BlankProject");
* }*/
using (TelemetryStopwatch ProjectDumpStopwatch = new TelemetryStopwatch("Project Dump"))
{
CommandUtils.LogVerbose(" Base Engine:");
BaseEngineProject.Dump(InHostPlatforms);
CommandUtils.LogVerbose(" {0} Code projects:", CodeProjects.Count);
foreach (var Proj in CodeProjects)
{
Proj.Dump(InHostPlatforms);
}
CommandUtils.LogVerbose(" {0} Non-Code projects:", CodeProjects.Count);
foreach (var Proj in NonCodeProjects)
{
Proj.Dump(InHostPlatforms);
}
}
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
/// <returns>True if the task succeeded</returns>
public override bool Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Figure out the project that this target belongs to
FileReference ProjectFile = null;
if (Parameters.Project != null)
{
ProjectFile = new FileReference(Parameters.Project);
if (!ProjectFile.Exists())
{
CommandUtils.LogError("Missing project file - {0}", ProjectFile.FullName);
return(false);
}
}
// Execute the cooker
using (TelemetryStopwatch CookStopwatch = new TelemetryStopwatch("Cook.{0}.{1}", (ProjectFile == null)? "UE4" : ProjectFile.GetFileNameWithoutExtension(), Parameters.Platform))
{
string[] Maps = (Parameters.Maps == null)? null : Parameters.Maps.Split(new char[] { '+' });
CommandUtils.CookCommandlet(ProjectFile, "UE4Editor-Cmd.exe", Maps, null, null, null, Parameters.Platform, (Parameters.Versioned? "" : "-Unversioned ") + Parameters.Arguments);
}
// Find all the cooked files
DirectoryReference CookedDirectory = DirectoryReference.Combine(ProjectFile.Directory, "Saved", "Cooked", Parameters.Platform);
if (!CookedDirectory.Exists())
{
CommandUtils.LogError("Cook output directory not found ({0})", CookedDirectory.FullName);
return(false);
}
List <FileReference> CookedFiles = CookedDirectory.EnumerateFileReferences("*", System.IO.SearchOption.AllDirectories).ToList();
if (CookedFiles.Count == 0)
{
CommandUtils.LogError("Cooking did not produce any files in {0}", CookedDirectory.FullName);
return(false);
}
// Apply the optional tag to the build products
if (!String.IsNullOrEmpty(Parameters.Tag))
{
FindOrAddTagSet(TagNameToFileSet, Parameters.Tag).UnionWith(CookedFiles);
}
// Add them to the set of build products
BuildProducts.UnionWith(CookedFiles);
return(true);
}
/// <summary>
/// Builds a Visual Studio solution with MsBuild (using msbuild.exe rather than devenv.com can help circumvent issues with expired Visual Studio licenses).
/// Automatically creates a logfile. When no LogName is specified, the executable name is used as logfile base name.
/// </summary>
/// <param name="Env">BuildEnvironment to use.</param>
/// <param name="SolutionFile">Path to the solution file</param>
/// <param name="BuildConfig">Configuration to build.</param>
/// <param name="BuildPlatform">Platform to build.</param>
/// <param name="LogName">Optional logfile name.</param>
public static void BuildSolution(CommandEnvironment Env, string SolutionFile, string BuildConfig, string BuildPlatform, string LogName = null)
{
if (!FileExists(SolutionFile))
{
throw new AutomationException(String.Format("Unabled to build Solution {0}. Solution file not found.", SolutionFile));
}
if (String.IsNullOrEmpty(Env.MsBuildExe))
{
throw new AutomationException("Unable to find msbuild.exe at: \"{0}\"", Env.MsBuildExe);
}
string CmdLine = String.Format("\"{0}\" /m /t:Build /p:Configuration=\"{1}\" /p:Platform=\"{2}\" /verbosity:minimal /nologo", SolutionFile, BuildConfig, BuildPlatform);
using (TelemetryStopwatch CompileStopwatch = new TelemetryStopwatch("Compile.{0}.{1}.{2}", Path.GetFileName(SolutionFile), "WinC#", BuildConfig))
{
RunAndLog(Env, Env.MsBuildExe, CmdLine, LogName);
}
}
/// <summary>
/// Builds a Visual Studio solution with MsDevEnv. Automatically creates a logfile. When
/// no LogName is specified, the executable name is used as logfile base name.
/// </summary>
/// <param name="Env">BuildEnvironment to use.</param>
/// <param name="SolutionFile">Path to the solution file</param>
/// <param name="BuildConfig">Configuration to build.</param>
/// <param name="LogName">Optional logfile name.</param>
public static void BuildSolution(CommandEnvironment Env, string SolutionFile, string BuildConfig = "Development", string LogName = null)
{
if (!FileExists(SolutionFile))
{
throw new AutomationException(String.Format("Unabled to build Solution {0}. Solution file not found.", SolutionFile));
}
if (String.IsNullOrEmpty(Env.MsDevExe))
{
throw new AutomationException(String.Format("Unabled to build Solution {0}. devenv.com not found.", SolutionFile));
}
string CmdLine = String.Format("\"{0}\" /build \"{1}\"", SolutionFile, BuildConfig);
using(TelemetryStopwatch CompileStopwatch = new TelemetryStopwatch("Compile.{0}.{1}.{2}", Path.GetFileName(SolutionFile), "WinC#", BuildConfig))
{
RunAndLog(Env, Env.MsDevExe, CmdLine, LogName);
}
}
/// <summary>
/// Builds a Visual Studio solution with MsBuild (using msbuild.exe rather than devenv.com can help circumvent issues with expired Visual Studio licenses).
/// Automatically creates a logfile. When no LogName is specified, the executable name is used as logfile base name.
/// </summary>
/// <param name="Env">BuildEnvironment to use.</param>
/// <param name="SolutionFile">Path to the solution file</param>
/// <param name="BuildConfig">Configuration to build.</param>
/// <param name="BuildPlatform">Platform to build.</param>
/// <param name="LogName">Optional logfile name.</param>
public static void BuildSolution(CommandEnvironment Env, string SolutionFile, string BuildConfig, string BuildPlatform, string LogName = null)
{
if (!FileExists(SolutionFile))
{
throw new AutomationException(String.Format("Unabled to build Solution {0}. Solution file not found.", SolutionFile));
}
if (String.IsNullOrEmpty(Env.MsBuildExe))
{
throw new AutomationException("Unable to find msbuild.exe at: \"{0}\"", Env.MsBuildExe);
}
string CmdLine = String.Format("\"{0}\" /m /t:Build /p:Configuration=\"{1}\" /p:Platform=\"{2}\" /verbosity:minimal /nologo", SolutionFile, BuildConfig, BuildPlatform);
using (TelemetryStopwatch CompileStopwatch = new TelemetryStopwatch("Compile.{0}.{1}.{2}", Path.GetFileName(SolutionFile), "WinC#", BuildConfig))
{
RunAndLog(Env, Env.MsBuildExe, CmdLine, LogName);
}
}
/// <summary>
/// Builds a Visual Studio solution with MsDevEnv. Automatically creates a logfile. When
/// no LogName is specified, the executable name is used as logfile base name.
/// </summary>
/// <param name="Env">BuildEnvironment to use.</param>
/// <param name="SolutionFile">Path to the solution file</param>
/// <param name="BuildConfig">Configuration to build.</param>
/// <param name="LogName">Optional logfile name.</param>
public static void BuildSolution(CommandEnvironment Env, string SolutionFile, string BuildConfig = "Development", string LogName = null)
{
if (!FileExists(SolutionFile))
{
throw new AutomationException(String.Format("Unabled to build Solution {0}. Solution file not found.", SolutionFile));
}
if (String.IsNullOrEmpty(Env.MsDevExe))
{
throw new AutomationException(String.Format("Unabled to build Solution {0}. devenv.com not found.", SolutionFile));
}
string CmdLine = String.Format("\"{0}\" /build \"{1}\"", SolutionFile, BuildConfig);
using (TelemetryStopwatch CompileStopwatch = new TelemetryStopwatch("Compile.{0}.{1}.{2}", Path.GetFileName(SolutionFile), "WinC#", BuildConfig))
{
RunAndLog(Env, Env.MsDevExe, CmdLine, LogName);
}
}
/// <summary>
/// Saves the given files (that should be rooted at the branch root) to a shared temp storage manifest with the given temp storage node and game.
/// </summary>
/// <param name="NodeName">The node which these build products belong to</param>
/// <param name="OutputName">The output name of the node.</param>
/// <param name="BuildProducts">Array of build products to be archived</param>
/// <param name="bPushToRemote">Allow skipping the copying of this manifest to shared storage, because it's not required by any other agent</param>
/// <returns>The created manifest instance (which has already been saved to disk).</returns>
public TempStorageManifest Archive(string NodeName, string OutputName, FileReference[] BuildProducts, bool bPushToRemote = true)
{
using (TelemetryStopwatch TelemetryStopwatch = new TelemetryStopwatch("StoreToTempStorage"))
{
// Create a manifest for the given build products
FileInfo[] Files = BuildProducts.Select(x => new FileInfo(x.FullName)).ToArray();
TempStorageManifest Manifest = new TempStorageManifest(Files, RootDir);
// Create the local directory for this node
DirectoryReference LocalNodeDir = GetDirectoryForNode(LocalDir, NodeName);
LocalNodeDir.CreateDirectory();
// Compress the files and copy to shared storage if necessary
bool bRemote = SharedDir != null && bPushToRemote && bWriteToSharedStorage;
if (bRemote)
{
// Create the shared directory for this node
DirectoryReference SharedNodeDir = GetDirectoryForNode(SharedDir, NodeName);
SharedNodeDir.CreateDirectory();
// Zip all the build products
FileInfo[] ZipFiles = ParallelZipFiles(Files, RootDir, SharedNodeDir, LocalNodeDir, OutputName);
Manifest.ZipFiles = ZipFiles.Select(x => new TempStorageZipFile(x)).ToArray();
// Save the shared manifest
FileReference SharedManifestFile = GetManifestFile(SharedDir, NodeName, OutputName);
CommandUtils.Log("Saving shared manifest to {0}", SharedManifestFile.FullName);
Manifest.Save(SharedManifestFile);
}
// Save the local manifest
FileReference LocalManifestFile = GetManifestFile(LocalDir, NodeName, OutputName);
CommandUtils.Log("Saving local manifest to {0}", LocalManifestFile.FullName);
Manifest.Save(LocalManifestFile);
// Update the stats
long ZipFilesTotalSize = (Manifest.ZipFiles == null)? 0 : Manifest.ZipFiles.Sum(x => x.Length);
TelemetryStopwatch.Finish(string.Format("StoreToTempStorage.{0}.{1}.{2}.{3}.{4}.{5}.{6}", Files.Length, Manifest.GetTotalSize(), ZipFilesTotalSize, bRemote? "Remote" : "Local", 0, 0, OutputName));
return(Manifest);
}
}
/// <summary>
/// Main method.
/// </summary>
/// <param name="Arguments">Command line</param>
public static ExitCode Process(string[] Arguments)
{
// Initial check for local or build machine runs BEFORE we parse the command line (We need this value set
// in case something throws the exception while parsing the command line)
IsBuildMachine = !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("uebp_LOCAL_ROOT"));
// Scan the command line for commands to execute.
var CommandsToExecute = new List <CommandInfo>();
string OutScriptsForProjectFileName;
var AdditionalScriptsFolders = new List <string>();
ParseCommandLine(Arguments, CommandsToExecute, out OutScriptsForProjectFileName, AdditionalScriptsFolders);
// Get the path to the telemetry file, if present
string TelemetryFile = CommandUtils.ParseParamValue(Arguments, "-Telemetry");
// Check for build machine override (force local)
IsBuildMachine = GlobalCommandLine.ForceLocal ? false : IsBuildMachine;
Log.TraceVerbose("IsBuildMachine={0}", IsBuildMachine);
Environment.SetEnvironmentVariable("IsBuildMachine", IsBuildMachine ? "1" : "0");
// should we kill processes on exit
ShouldKillProcesses = !GlobalCommandLine.NoKill;
Log.TraceVerbose("ShouldKillProcesses={0}", ShouldKillProcesses);
if (CommandsToExecute.Count == 0 && GlobalCommandLine.Help)
{
DisplayHelp();
return(ExitCode.Success);
}
// Disable AutoSDKs if specified on the command line
if (GlobalCommandLine.NoAutoSDK)
{
UEBuildPlatformSDK.bAllowAutoSDKSwitching = false;
}
// Setup environment
Log.TraceInformation("Setting up command environment.");
CommandUtils.InitCommandEnvironment();
// Change CWD to UE4 root.
Environment.CurrentDirectory = CommandUtils.CmdEnv.LocalRoot;
// Fill in the project info
UnrealBuildTool.UProjectInfo.FillProjectInfo();
// Clean rules folders up
ProjectUtils.CleanupFolders();
// Compile scripts.
ScriptCompiler Compiler = new ScriptCompiler();
using (TelemetryStopwatch ScriptCompileStopwatch = new TelemetryStopwatch("ScriptCompile"))
{
Compiler.FindAndCompileAllScripts(OutScriptsForProjectFileName, AdditionalScriptsFolders);
}
if (GlobalCommandLine.CompileOnly)
{
Log.TraceInformation("Compilation successful, exiting (CompileOnly)");
return(ExitCode.Success);
}
if (GlobalCommandLine.List)
{
ListAvailableCommands(Compiler.Commands);
return(ExitCode.Success);
}
if (GlobalCommandLine.Help)
{
DisplayHelp(CommandsToExecute, Compiler.Commands);
return(ExitCode.Success);
}
// Enable or disable P4 support
CommandUtils.InitP4Support(CommandsToExecute, Compiler.Commands);
if (CommandUtils.P4Enabled)
{
Log.TraceInformation("Setting up Perforce environment.");
CommandUtils.InitP4Environment();
CommandUtils.InitDefaultP4Connection();
}
// Find and execute commands.
ExitCode Result = Execute(CommandsToExecute, Compiler.Commands);
if (TelemetryFile != null)
{
Directory.CreateDirectory(Path.GetDirectoryName(TelemetryFile));
CommandUtils.Telemetry.Write(TelemetryFile);
}
return(Result);
}
public bool ProcessXGEItems(List<XGEItem> Actions, string XGETool, string Args, string TaskFilePath, bool DoRetries, bool SpecialTestFlag, bool ShowProgress)
{
TelemetryStopwatch CombineXGEStopwatch = new TelemetryStopwatch("CombineXGEItemFiles.{0}", Path.GetFileNameWithoutExtension(XGETool));
XmlDocument XGETaskDocument;
if (!CombineXGEItemFiles(Actions, TaskFilePath, out XGETaskDocument))
{
if (DeleteBuildProducts)
{
throw new AutomationException("Unable to find xge xmls: " + CombinePaths(CmdEnv.LogFolder, "*.xge.xml"));
}
else
{
LogVerbose("Incremental build, apparently everything was up to date, no XGE jobs produced.");
}
}
else
{
XmlElement EnvironmentsElement = XGETaskDocument.CreateElement("Environments");// Make sure all the tasks have a unique prefix
if(ShowProgress)
{
List<XmlElement> AllToolElements = new List<XmlElement>();
foreach(XmlElement EnvironmentElement in EnvironmentsElement.GetElementsByTagName("Environment"))
{
foreach(XmlElement ToolsElement in EnvironmentElement.GetElementsByTagName("Tools"))
{
foreach(XmlElement ToolElement in ToolsElement.GetElementsByTagName("Tool"))
{
AllToolElements.Add(ToolElement);
}
}
}
for(int Idx = 0; Idx < AllToolElements.Count; Idx++)
{
XmlElement ToolElement = AllToolElements[Idx];
if (ToolElement.HasAttribute("OutputPrefix"))
{
ToolElement.SetAttribute("OutputPrefix", ToolElement.Attributes["OutputPrefix"].Value + String.Format(" [@progress increment 1/{0}]", AllToolElements.Count));
}
else
{
ToolElement.SetAttribute("OutputPrefix", String.Format(" [@progress increment 1/{0} skipline]", AllToolElements.Count));
}
}
}
// Write the XGE task XML to a temporary file.
using (FileStream OutputFileStream = new FileStream(TaskFilePath, FileMode.Create, FileAccess.Write))
{
XGETaskDocument.Save(OutputFileStream);
}
if (!FileExists(TaskFilePath))
{
throw new AutomationException("Unable to find xge xml: " + TaskFilePath);
}
CombineXGEStopwatch.Finish();
if(XGETool == null)
{
PushDir(CombinePaths(CmdEnv.LocalRoot, @"\Engine\Source"));
try
{
int ExitCode = ParallelExecutor.Execute(TaskFilePath);
if(ExitCode != 0)
{
return false;
}
}
finally
{
PopDir();
}
}
else
{
using (TelemetryStopwatch ProcessXGEStopwatch = new TelemetryStopwatch("ProcessXGE.{0}", Path.GetFileNameWithoutExtension(XGETool)))
{
int Retries = DoRetries ? 2 : 1;
int ConnectionRetries = 4;
for (int i = 0; i < Retries; i++)
{
try
{
while (true)
{
LogVerbose("Running {0} *******", XGETool);
PushDir(CombinePaths(CmdEnv.LocalRoot, @"\Engine\Source"));
int SuccesCode;
string LogFile = GetRunAndLogOnlyName(CmdEnv, XGETool);
string Output = RunAndLog(XGETool, Args, out SuccesCode, LogFile);
bool bOutputContainsProject = Output.Contains("------Project:");
PopDir();
if (ConnectionRetries > 0 && (SuccesCode == 4 || SuccesCode == 2) && !bOutputContainsProject && XGETool != null)
{
LogWarning(String.Format("{0} failure on the local connection timeout", XGETool));
if (ConnectionRetries < 2)
{
System.Threading.Thread.Sleep(60000);
}
ConnectionRetries--;
continue;
}
else if (SuccesCode != 0)
{
throw new AutomationException("Command failed (Result:{3}): {0} {1}. See logfile for details: '{2}' ",
XGETool, Args, Path.GetFileName(LogFile), SuccesCode);
}
LogVerbose("{0} {1} Done *******", XGETool, Args);
break;
}
break;
}
catch (Exception Ex)
{
if (i + 1 >= Retries)
{
throw;
}
LogWarning("{0} failed on try {1}, deleting products to force a rebuild: {2}", XGETool, i + 1, Ex.ToString());
foreach (var Item in Actions)
{
XGEDeleteBuildProducts(Item.Manifest);
}
}
}
}
}
}
foreach (var Item in Actions)
{
XGEFinishBuildWithUBT(Item);
}
return true;
}
BuildManifest BuildWithUBT(string TargetName, UnrealBuildTool.UnrealTargetPlatform TargetPlatform, string Config, FileReference UprojectPath, bool ForceMonolithic = false, bool ForceNonUnity = false, bool ForceDebugInfo = false, bool ForceFlushMac = false, bool DisableXGE = false, string InAddArgs = "", bool ForceUnity = false, Dictionary<string, string> EnvVars = null)
{
string AddArgs = "";
if (UprojectPath != null)
{
AddArgs += " " + CommandUtils.MakePathSafeToUseWithCommandLine(UprojectPath.FullName);
}
AddArgs += " " + InAddArgs;
if (ForceMonolithic)
{
AddArgs += " -monolithic";
}
if (ForceNonUnity)
{
AddArgs += " -disableunity";
}
if (ForceUnity)
{
AddArgs += " -forceunity";
}
if (ForceDebugInfo)
{
AddArgs += " -forcedebuginfo";
}
if (ForceFlushMac)
{
AddArgs += " -flushmac";
}
if (DisableXGE)
{
AddArgs += " -noxge";
}
PrepareUBT();
// let the platform determine when to use the manifest
bool UseManifest = Platform.GetPlatform(TargetPlatform).ShouldUseManifestForUBTBuilds(AddArgs);
BuildManifest Manifest = null;
if (UseManifest)
{
string UBTManifest = GetUBTManifest(UprojectPath, AddArgs);
DeleteFile(UBTManifest);
using(TelemetryStopwatch PrepareManifestStopwatch = new TelemetryStopwatch("PrepareUBTManifest.{0}.{1}.{2}", TargetName, TargetPlatform.ToString(), Config))
{
RunUBT(CmdEnv, UBTExecutable: UBTExecutable, Project: UprojectPath, Target: TargetName, Platform: TargetPlatform.ToString(), Config: Config, AdditionalArgs: AddArgs + " -generatemanifest", EnvVars: EnvVars);
}
Manifest = PrepareManifest(UBTManifest, false);
}
using(TelemetryStopwatch CompileStopwatch = new TelemetryStopwatch("Compile.{0}.{1}.{2}", TargetName, TargetPlatform.ToString(), Config))
{
RunUBT(CmdEnv, UBTExecutable: UBTExecutable, Project: UprojectPath, Target: TargetName, Platform: TargetPlatform.ToString(), Config: Config, AdditionalArgs: AddArgs, EnvVars: EnvVars);
}
if(!AddArgs.Contains("-nolink"))
{
// allow the platform to perform any actions after building a target (seems almost like this should be done in UBT)
Platform.GetPlatform(TargetPlatform).PostBuildTarget(this, UprojectPath, TargetName, Config);
}
if (UseManifest)
{
string UBTManifest = GetUBTManifest(UprojectPath, AddArgs);
AddBuildProductsFromManifest(UBTManifest);
DeleteFile(UBTManifest);
}
return Manifest;
}
/// <summary>
/// Saves the given files (that should be rooted at the branch root) to a shared temp storage manifest with the given temp storage node and game.
/// </summary>
/// <param name="NodeName">The node which created the storage block</param>
/// <param name="BlockName">Name of the block to retrieve. May be null or empty.</param>
/// <param name="BuildProducts">Array of build products to be archived</param>
/// <param name="bPushToRemote">Allow skipping the copying of this manifest to shared storage, because it's not required by any other agent</param>
/// <returns>The created manifest instance (which has already been saved to disk).</returns>
public TempStorageManifest Archive(string NodeName, string BlockName, FileReference[] BuildProducts, bool bPushToRemote = true)
{
using(TelemetryStopwatch TelemetryStopwatch = new TelemetryStopwatch("StoreToTempStorage"))
{
// Create a manifest for the given build products
FileInfo[] Files = BuildProducts.Select(x => new FileInfo(x.FullName)).ToArray();
TempStorageManifest Manifest = new TempStorageManifest(Files, RootDir);
// Create the local directory for this node
DirectoryReference LocalNodeDir = GetDirectoryForNode(LocalDir, NodeName);
LocalNodeDir.CreateDirectory();
// Compress the files and copy to shared storage if necessary
bool bRemote = SharedDir != null && bPushToRemote && bWriteToSharedStorage;
if(bRemote)
{
// Create the shared directory for this node
FileReference SharedManifestFile = GetManifestLocation(SharedDir, NodeName, BlockName);
SharedManifestFile.Directory.CreateDirectory();
// Zip all the build products
FileInfo[] ZipFiles = ParallelZipFiles(Files, RootDir, SharedManifestFile.Directory, LocalNodeDir, SharedManifestFile.GetFileNameWithoutExtension());
Manifest.ZipFiles = ZipFiles.Select(x => new TempStorageZipFile(x)).ToArray();
// Save the shared manifest
CommandUtils.Log("Saving shared manifest to {0}", SharedManifestFile.FullName);
Manifest.Save(SharedManifestFile);
}
// Save the local manifest
FileReference LocalManifestFile = GetManifestLocation(LocalDir, NodeName, BlockName);
CommandUtils.Log("Saving local manifest to {0}", LocalManifestFile.FullName);
Manifest.Save(LocalManifestFile);
// Update the stats
long ZipFilesTotalSize = (Manifest.ZipFiles == null)? 0 : Manifest.ZipFiles.Sum(x => x.Length);
TelemetryStopwatch.Finish(string.Format("StoreToTempStorage.{0}.{1}.{2}.{3}.{4}.{5}.{6}", Files.Length, Manifest.GetTotalSize(), ZipFilesTotalSize, bRemote? "Remote" : "Local", 0, 0, BlockName));
return Manifest;
}
}
/// <summary>
/// Saves the list of fully qualified files (that should be rooted at the shared temp storage location for the game) to a shared temp storage manifest with the given temp storage node and game.
/// </summary>
/// <param name="TempStorageNodeInfo">Node info descibing the block of temp storage (essentially used to identify a subdirectory insides the game's temp storage folder).</param>
/// <param name="Files">Fully qualified names of files to reference in the manifest file.</param>
/// <param name="bLocalOnly">If true, only ensures that the local temp storage manifest exists. Otherwise, checks if the manifest exists in either local or shared storage.</param>
/// <param name="GameName">game name to determine the temp storage folder for. Empty is equivalent to "UE4".</param>
/// <param name="RootDir">Folder that all the given files are rooted from. If null or empty, CmdEnv.LocalRoot is used.</param>
/// <returns>The created manifest instance (which has already been saved to disk).</returns>
public static void StoreToTempStorage(TempStorageNodeInfo TempStorageNodeInfo, List<string> Files, bool bLocalOnly, string GameName, string RootDir)
{
using (var TelemetryStopwatch = new TelemetryStopwatch("StoreToTempStorage"))
{
// use LocalRoot if one is not specified
if (string.IsNullOrEmpty(RootDir))
{
RootDir = CommandUtils.CmdEnv.LocalRoot;
}
// save the manifest to local temp storage.
var Local = SaveLocalTempStorageManifest(RootDir, TempStorageNodeInfo, Files);
var LocalTotalSize = Local.GetTotalSize();
if (!bLocalOnly)
{
var BlockPath = SharedTempStorageDirectory(TempStorageNodeInfo, GameName);
CommandUtils.LogConsole("Storing to {0}", BlockPath);
if (CommandUtils.DirectoryExists_NoExceptions(BlockPath))
{
throw new AutomationException("Storage Block Already Exists! {0}", BlockPath);
}
CommandUtils.CreateDirectory(true, BlockPath);
if (!CommandUtils.DirectoryExists_NoExceptions(BlockPath))
{
throw new AutomationException("Storage Block Could Not Be Created! {0}", BlockPath);
}
// We know the source files exist and are under RootDir because we created the manifest, which verifies it.
// Now create the list of target files
var DestFiles = Files.Select(Filename => CommandUtils.MakeRerootedFilePath(Filename, RootDir, BlockPath)).ToList();
// make sure the target file doesn't exist, as we never expect this.
foreach (var DestFile in DestFiles)
{
if (CommandUtils.FileExists_NoExceptions(true, DestFile))
{
throw new AutomationException("Dest file {0} already exists.", DestFile);
}
}
if (bZipTempStorage)
{
var ZipTimer = DateTimeStopwatch.Start();
var Zip = new Ionic.Zip.ZipFile
{
UseZip64WhenSaving = Ionic.Zip.Zip64Option.Always
};
foreach (string File in Files)
{
// strip the root dir off each file so we get a proper relative path.
Zip.AddFile(Path.Combine(RootDir, File), Path.GetDirectoryName(CommandUtils.StripBaseDirectory(File, RootDir)));
}
var LocalManifest = LocalTempStorageManifestFilename(TempStorageNodeInfo);
var SharedManifest = SharedTempStorageManifestFilename(TempStorageNodeInfo, GameName);
var ZipFilename = Path.ChangeExtension(LocalManifest, "zip");
Zip.Save(ZipFilename);
var ZipFileSize = new FileInfo(ZipFilename).Length;
var ZipTimeMS = (long)ZipTimer.ElapsedTime.TotalMilliseconds;
var CopyTimer = DateTimeStopwatch.Start();
InternalUtils.Robust_CopyFile(ZipFilename, CommandUtils.CombinePaths(BlockPath, Path.GetFileName(ZipFilename)));
var CopyTimeMS = (long)CopyTimer.ElapsedTime.TotalMilliseconds;
CommandUtils.DeleteFile(ZipFilename);
// copy the local manifest to the shared location. We have to assume the zip is a good copy.
CommandUtils.CopyFile(LocalManifest, SharedManifest);
TelemetryStopwatch.Finish(string.Format("StoreToTempStorage.{0}.{1}.{2}.Remote.{3}.{4}.{5}", Files.Count, LocalTotalSize, ZipFileSize, CopyTimeMS, ZipTimeMS, TempStorageNodeInfo.NodeStorageName));
}
else
{
var CopyTimer = DateTimeStopwatch.Start();
CommandUtils.ThreadedCopyFiles(Files, DestFiles, ThreadsToCopyWith());
var CopyTimeMS = (long)CopyTimer.ElapsedTime.TotalMilliseconds;
// save the shared temp storage manifest file.
var Shared = SaveSharedTempStorageManifest(TempStorageNodeInfo, GameName, DestFiles);
// These manifests better have the same files, timestamps, and sizes, else something failed in the copy.
if (!Local.Compare(Shared))
{
// we will rename this so it can't be used, but leave it around for inspection
var SharedTempManifestFilename = SharedTempStorageManifestFilename(TempStorageNodeInfo, GameName);
CommandUtils.RenameFile_NoExceptions(SharedTempManifestFilename, SharedTempManifestFilename + ".broken");
throw new AutomationException("Shared and Local manifest mismatch. Wrote out {0} for inspection.", SharedTempManifestFilename + ".broken");
}
TelemetryStopwatch.Finish(string.Format("StoreToTempStorage.{0}.{1}.{2}.Remote.{3}.{4}.{5}", Files.Count, LocalTotalSize, 0L, CopyTimeMS, 0L, TempStorageNodeInfo.NodeStorageName));
}
}
else
{
TelemetryStopwatch.Finish(string.Format("StoreToTempStorage.{0}.{1}.{2}.Local.{3}.{4}.{5}", Files.Count, LocalTotalSize, 0L, 0L, 0L, TempStorageNodeInfo.NodeStorageName));
}
}
}
public override void DoBuild(GUBP bp)
{
if (HostPlatform == UnrealTargetPlatform.Mac)
{
// not sure if we need something here or if the cook commandlet will automatically convert the exe name
}
using(TelemetryStopwatch CookStopwatch = new TelemetryStopwatch("Cook.{0}.{1}", GameProj.GameName, CookPlatform))
{
CommandUtils.CookCommandlet(GameProj.FilePath, "UE4Editor-Cmd.exe", null, null, null, null, CookPlatform);
}
var CookedPath = RootIfAnyForTempStorage();
var CookedFiles = CommandUtils.FindFiles("*", true, CookedPath);
if (CookedFiles.GetLength(0) < 1)
{
throw new AutomationException("CookedPath {1} did not produce any files.", CookedPath);
}
BuildProducts = new List<string>();
foreach (var CookedFile in CookedFiles)
{
AddBuildProduct(CookedFile);
}
}
/// <summary>
/// Main method.
/// </summary>
/// <param name="Arguments">Command line</param>
public static ExitCode Process(string[] Arguments, StartupTraceListener StartupListener)
{
// Initial check for local or build machine runs BEFORE we parse the command line (We need this value set
// in case something throws the exception while parsing the command line)
IsBuildMachine = !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("uebp_LOCAL_ROOT")) || Arguments.Any(x => x.Equals("-BuildMachine", StringComparison.InvariantCultureIgnoreCase));
// Scan the command line for commands to execute.
var CommandsToExecute = new List <CommandInfo>();
string OutScriptsForProjectFileName;
var AdditionalScriptsFolders = new List <string>();
ParseCommandLine(Arguments, CommandsToExecute, out OutScriptsForProjectFileName, AdditionalScriptsFolders);
// Get the path to the telemetry file, if present
string TelemetryFile = CommandUtils.ParseParamValue(Arguments, "-Telemetry");
Log.TraceVerbose("IsBuildMachine={0}", IsBuildMachine);
Environment.SetEnvironmentVariable("IsBuildMachine", IsBuildMachine ? "1" : "0");
// should we kill processes on exit
ShouldKillProcesses = !GlobalCommandLine.NoKill;
Log.TraceVerbose("ShouldKillProcesses={0}", ShouldKillProcesses);
if (CommandsToExecute.Count == 0 && GlobalCommandLine.Help)
{
DisplayHelp();
return(ExitCode.Success);
}
// Disable AutoSDKs if specified on the command line
if (GlobalCommandLine.NoAutoSDK)
{
PlatformExports.PreventAutoSDKSwitching();
}
// Setup environment
Log.TraceLog("Setting up command environment.");
CommandUtils.InitCommandEnvironment();
// Create the log file, and flush the startup listener to it
TraceListener LogTraceListener = LogUtils.AddLogFileListener(CommandUtils.CmdEnv.LogFolder, CommandUtils.CmdEnv.FinalLogFolder);
StartupListener.CopyTo(LogTraceListener);
Trace.Listeners.Remove(StartupListener);
// Initialize UBT
if (!UnrealBuildTool.PlatformExports.Initialize())
{
Log.TraceInformation("Failed to initialize UBT");
return(ExitCode.Error_Unknown);
}
// Clean rules folders up
ProjectUtils.CleanupFolders();
// Compile scripts.
Compiler = new ScriptCompiler();
using (TelemetryStopwatch ScriptCompileStopwatch = new TelemetryStopwatch("ScriptCompile"))
{
Compiler.FindAndCompileAllScripts(OutScriptsForProjectFileName, AdditionalScriptsFolders);
}
if (GlobalCommandLine.CompileOnly)
{
Log.TraceInformation("Compilation successful, exiting (CompileOnly)");
return(ExitCode.Success);
}
if (GlobalCommandLine.List)
{
ListAvailableCommands(Compiler.Commands);
return(ExitCode.Success);
}
if (GlobalCommandLine.Help)
{
DisplayHelp(CommandsToExecute, Compiler.Commands);
return(ExitCode.Success);
}
// Enable or disable P4 support
CommandUtils.InitP4Support(CommandsToExecute, Compiler.Commands);
if (CommandUtils.P4Enabled)
{
Log.TraceLog("Setting up Perforce environment.");
CommandUtils.InitP4Environment();
CommandUtils.InitDefaultP4Connection();
}
// Find and execute commands.
ExitCode Result = Execute(CommandsToExecute, Compiler.Commands);
if (TelemetryFile != null)
{
Directory.CreateDirectory(Path.GetDirectoryName(TelemetryFile));
CommandUtils.Telemetry.Write(TelemetryFile);
}
return(Result);
}
private void WriteJobSteps(List <BuildNode> OrderedToDo, bool bSkipTriggers)
{
BuildNode LastSticky = null;
bool HitNonSticky = false;
bool bHaveECNodes = false;
// sticky nodes are ones that we run on the main agent. We run then first and they must not be intermixed with parallel jobs
foreach (BuildNode NodeToDo in OrderedToDo)
{
if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC
{
bHaveECNodes = true;
if (NodeToDo.IsSticky)
{
LastSticky = NodeToDo;
if (HitNonSticky && !bSkipTriggers)
{
throw new AutomationException("Sticky and non-sticky jobs did not sort right.");
}
}
else
{
HitNonSticky = true;
}
}
}
List <JobStep> Steps = new List <JobStep>();
using (TelemetryStopwatch PerlOutputStopwatch = new TelemetryStopwatch("PerlOutput"))
{
string ParentPath = Command.ParseParamValue("ParentPath");
bool bHasNoop = false;
if (LastSticky == null && bHaveECNodes)
{
// if we don't have any sticky nodes and we have other nodes, we run a fake noop just to release the resource
JobStep NoopStep = new JobStep(ParentPath, "Noop", "GUBP_UAT_Node", false, null, null, JobStepReleaseMode.Release);
NoopStep.ActualParameters.Add("NodeName", "Noop");
NoopStep.ActualParameters.Add("Sticky", "1");
Steps.Add(NoopStep);
bHasNoop = true;
}
Dictionary <string, List <BuildNode> > AgentGroupChains = new Dictionary <string, List <BuildNode> >();
List <BuildNode> StickyChain = new List <BuildNode>();
foreach (BuildNode NodeToDo in OrderedToDo)
{
if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC
{
string MyAgentGroup = NodeToDo.AgentSharingGroup;
if (MyAgentGroup != "")
{
if (!AgentGroupChains.ContainsKey(MyAgentGroup))
{
AgentGroupChains.Add(MyAgentGroup, new List <BuildNode> {
NodeToDo
});
}
else
{
AgentGroupChains[MyAgentGroup].Add(NodeToDo);
}
}
}
if (NodeToDo.IsSticky)
{
if (!StickyChain.Contains(NodeToDo))
{
StickyChain.Add(NodeToDo);
}
}
}
foreach (BuildNode NodeToDo in OrderedToDo)
{
if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC
{
bool Sticky = NodeToDo.IsSticky;
if (NodeToDo.IsSticky)
{
if (NodeToDo.AgentSharingGroup != "")
{
throw new AutomationException("Node {0} is both agent sharing and sitcky.", NodeToDo.Name);
}
if (NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64)
{
throw new AutomationException("Node {0} is sticky, but {1} hosted. Sticky nodes must be PC hosted.", NodeToDo.Name, NodeToDo.AgentPlatform);
}
if (NodeToDo.AgentRequirements != "")
{
throw new AutomationException("Node {0} is sticky but has agent requirements.", NodeToDo.Name);
}
}
string ProcedureInfix = "";
if (NodeToDo.AgentPlatform != UnrealTargetPlatform.Unknown && NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64)
{
ProcedureInfix = "_" + NodeToDo.AgentPlatform.ToString();
}
bool DoParallel = !Sticky || NodeToDo.IsParallelAgentShareEditor;
List <BuildNode> UncompletedEcDeps = new List <BuildNode>();
foreach (BuildNode Dep in FindDirectOrderDependencies(NodeToDo))
{
if (!Dep.IsComplete && OrderedToDo.Contains(Dep)) // if something is already finished, we don't put it into EC
{
if (OrderedToDo.IndexOf(Dep) > OrderedToDo.IndexOf(NodeToDo))
{
throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", NodeToDo.Name, Dep.Name);
}
UncompletedEcDeps.Add(Dep);
}
}
string PreCondition = GetPreConditionForNode(OrderedToDo, ParentPath, bHasNoop, AgentGroupChains, StickyChain, NodeToDo, UncompletedEcDeps);
string RunCondition = GetRunConditionForNode(UncompletedEcDeps, ParentPath);
// Create the job steps for this node
TriggerNode TriggerNodeToDo = NodeToDo as TriggerNode;
if (TriggerNodeToDo == null)
{
// Create the jobs to setup the agent sharing group if necessary
string NodeParentPath = ParentPath;
if (NodeToDo.AgentSharingGroup != "")
{
NodeParentPath = String.Format("{0}/jobSteps[{1}]", NodeParentPath, NodeToDo.AgentSharingGroup);
List <BuildNode> MyChain = AgentGroupChains[NodeToDo.AgentSharingGroup];
if (MyChain.IndexOf(NodeToDo) <= 0)
{
// Create the parent job step for this group
JobStep ParentStep = new JobStep(ParentPath, NodeToDo.AgentSharingGroup, null, true, PreCondition, null, JobStepReleaseMode.Keep);
Steps.Add(ParentStep);
// Get the resource pool
JobStep GetPoolStep = new JobStep(NodeParentPath, String.Format("{0}_GetPool", NodeToDo.AgentSharingGroup), String.Format("GUBP{0}_AgentShare_GetPool", ProcedureInfix), true, PreCondition, null, JobStepReleaseMode.Keep);
GetPoolStep.ActualParameters.Add("AgentSharingGroup", NodeToDo.AgentSharingGroup);
GetPoolStep.ActualParameters.Add("NodeName", NodeToDo.Name);
Steps.Add(GetPoolStep);
// Get the agent for this sharing group
JobStep GetAgentStep = new JobStep(NodeParentPath, String.Format("{0}_GetAgent", NodeToDo.AgentSharingGroup), String.Format("GUBP{0}_AgentShare_GetAgent", ProcedureInfix), true, GetPoolStep.GetCompletedCondition(), null, JobStepReleaseMode.Keep);
GetAgentStep.Exclusive = JobStepExclusiveMode.Call;
GetAgentStep.ResourceName = String.Format("$[/myJob/jobSteps[{0}]/ResourcePool]", NodeToDo.AgentSharingGroup);
GetAgentStep.ActualParameters.Add("AgentSharingGroup", NodeToDo.AgentSharingGroup);
GetAgentStep.ActualParameters.Add("NodeName", NodeToDo.Name);
Steps.Add(GetAgentStep);
// Set the precondition from this point onwards to be whether the group was set up, since it can't succeed unless the original precondition succeeded
PreCondition = GetAgentStep.GetCompletedCondition();
}
}
// Get the procedure name
string Procedure;
if (NodeToDo.IsParallelAgentShareEditor)
{
if (NodeToDo.AgentSharingGroup == "")
{
Procedure = "GUBP_UAT_Node_Parallel_AgentShare_Editor";
}
else
{
Procedure = "GUBP_UAT_Node_Parallel_AgentShare3_Editor";
}
}
else
{
if (NodeToDo.IsSticky)
{
Procedure = "GUBP" + ProcedureInfix + "_UAT_Node";
}
else if (NodeToDo.AgentSharingGroup == "")
{
Procedure = String.Format("GUBP{0}_UAT_Node_Parallel", ProcedureInfix);
}
else
{
Procedure = String.Format("GUBP{0}_UAT_Node_Parallel_AgentShare3", ProcedureInfix);
}
}
if (NodeToDo.IsSticky && NodeToDo == LastSticky)
{
Procedure += "_Release";
}
// Build the job step for this node
JobStep MainStep = new JobStep(NodeParentPath, NodeToDo.Name, Procedure, DoParallel, PreCondition, RunCondition, (NodeToDo.IsSticky && NodeToDo == LastSticky) ? JobStepReleaseMode.Release : JobStepReleaseMode.Keep);
MainStep.ActualParameters.Add("NodeName", NodeToDo.Name);
MainStep.ActualParameters.Add("Sticky", NodeToDo.IsSticky ? "1" : "0");
if (NodeToDo.AgentSharingGroup != "")
{
MainStep.ActualParameters.Add("AgentSharingGroup", NodeToDo.AgentSharingGroup);
}
Steps.Add(MainStep);
}
else
{
// Get the procedure name
string Procedure;
if (TriggerNodeToDo.IsTriggered)
{
Procedure = String.Format("GUBP{0}_UAT_Node{1}", ProcedureInfix, (NodeToDo == LastSticky) ? "_Release" : "");
}
else if (TriggerNodeToDo.RequiresRecursiveWorkflow)
{
Procedure = "GUBP_UAT_Trigger"; //here we run a recursive workflow to wait for the trigger
}
else
{
Procedure = "GUBP_Hardcoded_Trigger"; //here we advance the state in the hardcoded workflow so folks can approve
}
// Create the job step
JobStep TriggerStep = new JobStep(ParentPath, NodeToDo.Name, Procedure, DoParallel, PreCondition, RunCondition, (NodeToDo.IsSticky && NodeToDo == LastSticky) ? JobStepReleaseMode.Release : JobStepReleaseMode.Keep);
TriggerStep.ActualParameters.Add("NodeName", NodeToDo.Name);
TriggerStep.ActualParameters.Add("Sticky", NodeToDo.IsSticky ? "1" : "0");
if (!TriggerNodeToDo.IsTriggered)
{
TriggerStep.ActualParameters.Add("TriggerState", TriggerNodeToDo.StateName);
TriggerStep.ActualParameters.Add("ActionText", TriggerNodeToDo.ActionText);
TriggerStep.ActualParameters.Add("DescText", TriggerNodeToDo.DescriptionText);
if (NodeToDo.RecipientsForFailureEmails.Length > 0)
{
TriggerStep.ActualParameters.Add("EmailsForTrigger", String.Join(" ", NodeToDo.RecipientsForFailureEmails));
}
}
Steps.Add(TriggerStep);
}
}
}
WriteECPerl(Steps);
}
}
void ExecuteNodes(ElectricCommander EC, List<BuildNode> OrderedToDo, bool bFake, bool bFakeEC, bool bSaveSharedTempStorage, JobInfo JobInfo, string FakeFail)
{
Dictionary<string, BuildNode> BuildProductToNodeMap = new Dictionary<string, BuildNode>();
foreach (BuildNode NodeToDo in OrderedToDo)
{
if (NodeToDo.Node.BuildProducts != null)
{
throw new AutomationException("topological sort error");
}
var TempStorageNodeInfo = new TempStorageNodeInfo(JobInfo, NodeToDo.Name);
string GameNameIfAny = NodeToDo.GameNameIfAnyForTempStorage;
string StorageRootIfAny = NodeToDo.RootIfAnyForTempStorage;
if (bFake)
{
StorageRootIfAny = ""; // we don't rebase fake runs since those are entirely "records of success", which are always in the logs folder
}
if (string.IsNullOrEmpty(StorageRootIfAny))
{
StorageRootIfAny = CmdEnv.LocalRoot;
}
// this is kinda complicated
bool SaveSuccessRecords = (IsBuildMachine || bFakeEC) && // no real reason to make these locally except for fakeEC tests
(!(NodeToDo is TriggerNode) || NodeToDo.IsSticky); // trigger nodes are run twice, one to start the new workflow and once when it is actually triggered, we will save reconds for the latter
LogConsole("***** Running GUBP Node {0} -> {1} : {2}", NodeToDo.Name, GameNameIfAny, TempStorageNodeInfo.GetRelativeDirectory());
if (NodeToDo.IsComplete)
{
if (NodeToDo.Name == VersionFilesNode.StaticGetFullName() && !IsBuildMachine)
{
LogConsole("***** NOT ****** Retrieving GUBP Node {0} from {1}; it is the version files.", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory());
NodeToDo.Node.BuildProducts = new List<string>();
}
else
{
LogConsole("***** Retrieving GUBP Node {0} from {1}", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory());
bool WasLocal;
try
{
NodeToDo.Node.BuildProducts = TempStorage.RetrieveFromTempStorage(TempStorageNodeInfo, out WasLocal, GameNameIfAny, StorageRootIfAny);
}
catch (Exception Ex)
{
if(GameNameIfAny != "")
{
NodeToDo.Node.BuildProducts = TempStorage.RetrieveFromTempStorage(TempStorageNodeInfo, out WasLocal, "", StorageRootIfAny);
}
else
{
throw new AutomationException(Ex, "Build Products cannot be found for node {0}", NodeToDo.Name);
}
}
if (!WasLocal)
{
NodeToDo.Node.PostLoadFromSharedTempStorage(this);
}
}
}
else
{
if (SaveSuccessRecords)
{
// We save our status to a new temp storage location specifically named with a suffix so we can find it later.
EC.SaveStatus(new TempStorageNodeInfo(JobInfo, NodeToDo.Name + StartedTempStorageSuffix), bSaveSharedTempStorage, GameNameIfAny);
}
double BuildDuration = 0.0;
try
{
if (!String.IsNullOrEmpty(FakeFail) && FakeFail.Equals(NodeToDo.Name, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Failing node {0} by request.", NodeToDo.Name);
}
if (bFake)
{
LogConsole("***** FAKE!! Building GUBP Node {0} for {1}", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory());
NodeToDo.DoFakeBuild();
}
else
{
LogConsole("***** Building GUBP Node {0} for {1}", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory());
DateTime StartTime = DateTime.UtcNow;
using(TelemetryStopwatch DoBuildStopwatch = new TelemetryStopwatch("DoBuild.{0}", NodeToDo.Name))
{
NodeToDo.DoBuild();
}
BuildDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds / 1000;
}
TempStorage.StoreToTempStorage(TempStorageNodeInfo, NodeToDo.Node.BuildProducts, !bSaveSharedTempStorage, GameNameIfAny, StorageRootIfAny);
}
catch (Exception Ex)
{
//@todo: This is a dup of non-exception code. Consolidate this!!
NodeHistory History = null;
if (SaveSuccessRecords)
{
using(TelemetryStopwatch UpdateNodeHistoryStopwatch = new TelemetryStopwatch("UpdateNodeHistory"))
{
History = FindNodeHistory(NodeToDo, JobInfo);
}
using(TelemetryStopwatch SaveNodeStatusStopwatch = new TelemetryStopwatch("SaveNodeStatus"))
{
// We save our status to a new temp storage location specifically named with a suffix so we can find it later.
EC.SaveStatus(new TempStorageNodeInfo(JobInfo, NodeToDo.Name + FailedTempStorageSuffix), bSaveSharedTempStorage, GameNameIfAny, ParseParamValue("MyJobStepId"));
}
using(TelemetryStopwatch UpdateECPropsStopwatch = new TelemetryStopwatch("UpdateECProps"))
{
EC.UpdateECProps(NodeToDo);
}
if (IsBuildMachine)
{
using(TelemetryStopwatch GetFailEmailsStopwatch = new TelemetryStopwatch("GetFailEmails"))
{
GetFailureEmails(EC, NodeToDo, History, JobInfo);
}
}
EC.UpdateECBuildTime(NodeToDo, BuildDuration);
}
LogConsole("{0}", Ex);
if (History != null)
{
LogConsole("Changes since last green *********************************");
LogConsole("");
LogConsole("");
LogConsole("");
PrintDetailedChanges(History, P4Env.Changelist);
LogConsole("End changes since last green");
}
string FailInfo = "";
FailInfo += "********************************* Main log file";
FailInfo += Environment.NewLine + Environment.NewLine;
FailInfo += LogUtils.GetLogTail();
FailInfo += Environment.NewLine + Environment.NewLine + Environment.NewLine;
string OtherLog = "See logfile for details: '";
if (FailInfo.Contains(OtherLog))
{
string LogFile = FailInfo.Substring(FailInfo.IndexOf(OtherLog) + OtherLog.Length);
if (LogFile.Contains("'"))
{
LogFile = CombinePaths(CmdEnv.LogFolder, LogFile.Substring(0, LogFile.IndexOf("'")));
if (FileExists_NoExceptions(LogFile))
{
FailInfo += "********************************* Sub log file " + LogFile;
FailInfo += Environment.NewLine + Environment.NewLine;
FailInfo += LogUtils.GetLogTail(LogFile);
FailInfo += Environment.NewLine + Environment.NewLine + Environment.NewLine;
}
}
}
string Filename = CombinePaths(CmdEnv.LogFolder, "LogTailsAndChanges.log");
WriteAllText(Filename, FailInfo);
throw(Ex);
}
if (SaveSuccessRecords)
{
NodeHistory History = null;
using(TelemetryStopwatch UpdateNodeHistoryStopwatch = new TelemetryStopwatch("UpdateNodeHistory"))
{
History = FindNodeHistory(NodeToDo, JobInfo);
}
using(TelemetryStopwatch SaveNodeStatusStopwatch = new TelemetryStopwatch("SaveNodeStatus"))
{
// We save our status to a new temp storage location specifically named with a suffix so we can find it later.
EC.SaveStatus(new TempStorageNodeInfo(JobInfo, NodeToDo.Name + SucceededTempStorageSuffix), bSaveSharedTempStorage, GameNameIfAny);
}
using(TelemetryStopwatch UpdateECPropsStopwatch = new TelemetryStopwatch("UpdateECProps"))
{
EC.UpdateECProps(NodeToDo);
}
if (IsBuildMachine)
{
using(TelemetryStopwatch GetFailEmailsStopwatch = new TelemetryStopwatch("GetFailEmails"))
{
GetFailureEmails(EC, NodeToDo, History, JobInfo);
}
}
EC.UpdateECBuildTime(NodeToDo, BuildDuration);
}
}
foreach (string Product in NodeToDo.Node.BuildProducts)
{
if (BuildProductToNodeMap.ContainsKey(Product))
{
throw new AutomationException("Overlapping build product: {0} and {1} both produce {2}", BuildProductToNodeMap[Product].ToString(), NodeToDo.Name, Product);
}
BuildProductToNodeMap.Add(Product, NodeToDo);
}
}
PrintRunTime();
}
/// <summary>
/// Retrieve an output of the given node. Fetches and decompresses the files from shared storage if necessary, or validates the local files.
/// </summary>
/// <param name="NodeName">The node to retrieve build products for</param>
/// <param name="OutputName">The name of the node's output. May be null.</param>
/// <returns>Manifest of the files retrieved</returns>
public TempStorageManifest Retreive(string NodeName, string OutputName)
{
using (var TelemetryStopwatch = new TelemetryStopwatch("RetrieveFromTempStorage"))
{
// Get the path to the local manifest
FileReference LocalManifestFile = GetManifestFile(LocalDir, NodeName, OutputName);
bool bLocal = LocalManifestFile.Exists();
// Read the manifest, either from local storage or shared storage
TempStorageManifest Manifest;
if (bLocal)
{
CommandUtils.Log("Reading shared manifest from {0}", LocalManifestFile.FullName);
Manifest = TempStorageManifest.Load(LocalManifestFile);
}
else
{
// Check we have shared storage
if (SharedDir == null)
{
throw new AutomationException("Missing local manifest for node - {0}", LocalManifestFile.FullName);
}
// Get the shared directory for this node
FileReference SharedManifestFile = GetManifestFile(SharedDir, NodeName, OutputName);
// Make sure the manifest exists
if (!SharedManifestFile.Exists())
{
throw new AutomationException("Missing local or shared manifest for node - {0}", SharedManifestFile.FullName);
}
// Read the shared manifest
CommandUtils.Log("Copying shared manifest from {0} to {1}", SharedManifestFile.FullName, LocalManifestFile.FullName);
Manifest = TempStorageManifest.Load(SharedManifestFile);
// Unzip all the build products
DirectoryReference SharedNodeDir = GetDirectoryForNode(SharedDir, NodeName);
FileInfo[] ZipFiles = Manifest.ZipFiles.Select(x => new FileInfo(FileReference.Combine(SharedNodeDir, x.Name).FullName)).ToArray();
ParallelUnzipFiles(ZipFiles, RootDir);
// Fix any Unix permissions/chmod issues, and update the timestamps to match the manifest. Zip files only use local time, and there's no guarantee it matches the local clock.
foreach (TempStorageFile ManifestFile in Manifest.Files)
{
FileReference File = ManifestFile.ToFileReference(RootDir);
if (Utils.IsRunningOnMono)
{
CommandUtils.FixUnixFilePermissions(File.FullName);
}
System.IO.File.SetLastWriteTimeUtc(File.FullName, new DateTime(ManifestFile.LastWriteTimeUtcTicks, DateTimeKind.Utc));
}
// Save the manifest locally
LocalManifestFile.Directory.CreateDirectory();
Manifest.Save(LocalManifestFile);
}
// Check all the local files are as expected
bool bAllMatch = true;
foreach (TempStorageFile File in Manifest.Files)
{
bAllMatch &= File.Compare(RootDir);
}
if (!bAllMatch)
{
throw new AutomationException("Files have been modified");
}
// Update the stats and return
TelemetryStopwatch.Finish(string.Format("RetrieveFromTempStorage.{0}.{1}.{2}.{3}.{4}.{5}.{6}", Manifest.Files.Length, Manifest.Files.Sum(x => x.Length), bLocal? 0 : Manifest.ZipFiles.Sum(x => x.Length), bLocal? "Local" : "Remote", 0, 0, OutputName));
return(Manifest);
}
}
private void WriteJobSteps(List<BuildNode> OrderedToDo, bool bSkipTriggers)
{
BuildNode LastSticky = null;
bool HitNonSticky = false;
bool bHaveECNodes = false;
// sticky nodes are ones that we run on the main agent. We run then first and they must not be intermixed with parallel jobs
foreach (BuildNode NodeToDo in OrderedToDo)
{
if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC
{
bHaveECNodes = true;
if (NodeToDo.IsSticky)
{
LastSticky = NodeToDo;
if (HitNonSticky && !bSkipTriggers)
{
throw new AutomationException("Sticky and non-sticky jobs did not sort right.");
}
}
else
{
HitNonSticky = true;
}
}
}
List<JobStep> Steps = new List<JobStep>();
using (TelemetryStopwatch PerlOutputStopwatch = new TelemetryStopwatch("PerlOutput"))
{
string ParentPath = Command.ParseParamValue("ParentPath");
bool bHasNoop = false;
if (LastSticky == null && bHaveECNodes)
{
// if we don't have any sticky nodes and we have other nodes, we run a fake noop just to release the resource
JobStep NoopStep = new JobStep(ParentPath, "Noop", "GUBP_UAT_Node", false, null, null, JobStepReleaseMode.Release);
NoopStep.ActualParameters.Add("NodeName", "Noop");
NoopStep.ActualParameters.Add("Sticky", "1");
Steps.Add(NoopStep);
bHasNoop = true;
}
Dictionary<string, List<BuildNode>> AgentGroupChains = new Dictionary<string, List<BuildNode>>();
List<BuildNode> StickyChain = new List<BuildNode>();
foreach (BuildNode NodeToDo in OrderedToDo)
{
if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC
{
string MyAgentGroup = NodeToDo.AgentSharingGroup;
if (MyAgentGroup != "")
{
if (!AgentGroupChains.ContainsKey(MyAgentGroup))
{
AgentGroupChains.Add(MyAgentGroup, new List<BuildNode> { NodeToDo });
}
else
{
AgentGroupChains[MyAgentGroup].Add(NodeToDo);
}
}
}
if (NodeToDo.IsSticky)
{
if (!StickyChain.Contains(NodeToDo))
{
StickyChain.Add(NodeToDo);
}
}
}
foreach (BuildNode NodeToDo in OrderedToDo)
{
if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC
{
bool Sticky = NodeToDo.IsSticky;
if (NodeToDo.IsSticky)
{
if (NodeToDo.AgentSharingGroup != "")
{
throw new AutomationException("Node {0} is both agent sharing and sitcky.", NodeToDo.Name);
}
if (NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64)
{
throw new AutomationException("Node {0} is sticky, but {1} hosted. Sticky nodes must be PC hosted.", NodeToDo.Name, NodeToDo.AgentPlatform);
}
if (NodeToDo.AgentRequirements != "")
{
throw new AutomationException("Node {0} is sticky but has agent requirements.", NodeToDo.Name);
}
}
string ProcedureInfix = "";
if (NodeToDo.AgentPlatform != UnrealTargetPlatform.Unknown && NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64)
{
ProcedureInfix = "_" + NodeToDo.AgentPlatform.ToString();
}
bool DoParallel = !Sticky || NodeToDo.IsParallelAgentShareEditor;
List<BuildNode> UncompletedEcDeps = new List<BuildNode>();
foreach (BuildNode Dep in FindDirectOrderDependencies(NodeToDo))
{
if (!Dep.IsComplete && OrderedToDo.Contains(Dep)) // if something is already finished, we don't put it into EC
{
if (OrderedToDo.IndexOf(Dep) > OrderedToDo.IndexOf(NodeToDo))
{
throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", NodeToDo.Name, Dep.Name);
}
UncompletedEcDeps.Add(Dep);
}
}
string PreCondition = GetPreConditionForNode(OrderedToDo, ParentPath, bHasNoop, AgentGroupChains, StickyChain, NodeToDo, UncompletedEcDeps);
string RunCondition = GetRunConditionForNode(UncompletedEcDeps, ParentPath);
// Create the job steps for this node
TriggerNode TriggerNodeToDo = NodeToDo as TriggerNode;
if (TriggerNodeToDo == null)
{
// Create the jobs to setup the agent sharing group if necessary
string NodeParentPath = ParentPath;
if (NodeToDo.AgentSharingGroup != "")
{
NodeParentPath = String.Format("{0}/jobSteps[{1}]", NodeParentPath, NodeToDo.AgentSharingGroup);
List<BuildNode> MyChain = AgentGroupChains[NodeToDo.AgentSharingGroup];
if (MyChain.IndexOf(NodeToDo) <= 0)
{
// Create the parent job step for this group
JobStep ParentStep = new JobStep(ParentPath, NodeToDo.AgentSharingGroup, null, true, PreCondition, null, JobStepReleaseMode.Keep);
Steps.Add(ParentStep);
// Get the resource pool
JobStep GetPoolStep = new JobStep(NodeParentPath, String.Format("{0}_GetPool", NodeToDo.AgentSharingGroup), String.Format("GUBP{0}_AgentShare_GetPool", ProcedureInfix), true, PreCondition, null, JobStepReleaseMode.Keep);
GetPoolStep.ActualParameters.Add("AgentSharingGroup", NodeToDo.AgentSharingGroup);
GetPoolStep.ActualParameters.Add("NodeName", NodeToDo.Name);
Steps.Add(GetPoolStep);
// Get the agent for this sharing group
JobStep GetAgentStep = new JobStep(NodeParentPath, String.Format("{0}_GetAgent", NodeToDo.AgentSharingGroup), String.Format("GUBP{0}_AgentShare_GetAgent", ProcedureInfix), true, GetPoolStep.GetCompletedCondition(), null, JobStepReleaseMode.Keep);
GetAgentStep.Exclusive = JobStepExclusiveMode.Call;
GetAgentStep.ResourceName = String.Format("$[/myJob/jobSteps[{0}]/ResourcePool]", NodeToDo.AgentSharingGroup);
GetAgentStep.ActualParameters.Add("AgentSharingGroup", NodeToDo.AgentSharingGroup);
GetAgentStep.ActualParameters.Add("NodeName", NodeToDo.Name);
Steps.Add(GetAgentStep);
// Set the precondition from this point onwards to be whether the group was set up, since it can't succeed unless the original precondition succeeded
PreCondition = GetAgentStep.GetCompletedCondition();
}
}
// Get the procedure name
string Procedure;
if (NodeToDo.IsParallelAgentShareEditor)
{
if (NodeToDo.AgentSharingGroup == "")
{
Procedure = "GUBP_UAT_Node_Parallel_AgentShare_Editor";
}
else
{
Procedure = "GUBP_UAT_Node_Parallel_AgentShare3_Editor";
}
}
else
{
if (NodeToDo.IsSticky)
{
Procedure = "GUBP" + ProcedureInfix + "_UAT_Node";
}
else if (NodeToDo.AgentSharingGroup == "")
{
Procedure = String.Format("GUBP{0}_UAT_Node_Parallel", ProcedureInfix);
}
else
{
Procedure = String.Format("GUBP{0}_UAT_Node_Parallel_AgentShare3", ProcedureInfix);
}
}
if (NodeToDo.IsSticky && NodeToDo == LastSticky)
{
Procedure += "_Release";
}
// Build the job step for this node
JobStep MainStep = new JobStep(NodeParentPath, NodeToDo.Name, Procedure, DoParallel, PreCondition, RunCondition, (NodeToDo.IsSticky && NodeToDo == LastSticky) ? JobStepReleaseMode.Release : JobStepReleaseMode.Keep);
MainStep.ActualParameters.Add("NodeName", NodeToDo.Name);
MainStep.ActualParameters.Add("Sticky", NodeToDo.IsSticky ? "1" : "0");
if (NodeToDo.AgentSharingGroup != "")
{
MainStep.ActualParameters.Add("AgentSharingGroup", NodeToDo.AgentSharingGroup);
}
Steps.Add(MainStep);
}
else
{
// Get the procedure name
string Procedure;
if (TriggerNodeToDo.IsTriggered)
{
Procedure = String.Format("GUBP{0}_UAT_Node{1}", ProcedureInfix, (NodeToDo == LastSticky) ? "_Release" : "");
}
else if (TriggerNodeToDo.RequiresRecursiveWorkflow)
{
Procedure = "GUBP_UAT_Trigger"; //here we run a recursive workflow to wait for the trigger
}
else
{
Procedure = "GUBP_Hardcoded_Trigger"; //here we advance the state in the hardcoded workflow so folks can approve
}
// Create the job step
JobStep TriggerStep = new JobStep(ParentPath, NodeToDo.Name, Procedure, DoParallel, PreCondition, RunCondition, (NodeToDo.IsSticky && NodeToDo == LastSticky) ? JobStepReleaseMode.Release : JobStepReleaseMode.Keep);
TriggerStep.ActualParameters.Add("NodeName", NodeToDo.Name);
TriggerStep.ActualParameters.Add("Sticky", NodeToDo.IsSticky ? "1" : "0");
if (!TriggerNodeToDo.IsTriggered)
{
TriggerStep.ActualParameters.Add("TriggerState", TriggerNodeToDo.StateName);
TriggerStep.ActualParameters.Add("ActionText", TriggerNodeToDo.ActionText);
TriggerStep.ActualParameters.Add("DescText", TriggerNodeToDo.DescriptionText);
if (NodeToDo.RecipientsForFailureEmails.Length > 0)
{
TriggerStep.ActualParameters.Add("EmailsForTrigger", String.Join(" ", NodeToDo.RecipientsForFailureEmails));
}
}
Steps.Add(TriggerStep);
}
}
}
WriteECPerl(Steps);
}
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
/// <returns>True if the task succeeded</returns>
public override bool Execute(JobContext Job, HashSet<FileReference> BuildProducts, Dictionary<string, HashSet<FileReference>> TagNameToFileSet)
{
// Figure out the project that this target belongs to
FileReference ProjectFile = null;
if(Parameters.Project != null)
{
ProjectFile = new FileReference(Parameters.Project);
if(!ProjectFile.Exists())
{
CommandUtils.LogError("Missing project file - {0}", ProjectFile.FullName);
return false;
}
}
// Execute the cooker
using(TelemetryStopwatch CookStopwatch = new TelemetryStopwatch("Cook.{0}.{1}", (ProjectFile == null)? "UE4" : ProjectFile.GetFileNameWithoutExtension(), Parameters.Platform))
{
string[] Maps = (Parameters.Maps == null)? null : Parameters.Maps.Split(new char[]{ '+' });
string Arguments = (Parameters.Versioned ? "" : "-Unversioned ") + "-LogCmds=\"LogSavePackage Warning\" " + Parameters.Arguments;
CommandUtils.CookCommandlet(ProjectFile, "UE4Editor-Cmd.exe", Maps, null, null, null, Parameters.Platform, Arguments);
}
// Find all the cooked files
List<FileReference> CookedFiles = new List<FileReference>();
foreach(string Platform in Parameters.Platform.Split('+'))
{
DirectoryReference PlatformCookedDirectory = DirectoryReference.Combine(ProjectFile.Directory, "Saved", "Cooked", Platform);
if(!PlatformCookedDirectory.Exists())
{
CommandUtils.LogError("Cook output directory not found ({0})", PlatformCookedDirectory.FullName);
return false;
}
List<FileReference> PlatformCookedFiles = PlatformCookedDirectory.EnumerateFileReferences("*", System.IO.SearchOption.AllDirectories).ToList();
if(PlatformCookedFiles.Count == 0)
{
CommandUtils.LogError("Cooking did not produce any files in {0}", PlatformCookedDirectory.FullName);
return false;
}
CookedFiles.AddRange(PlatformCookedFiles);
}
// Apply the optional tag to the build products
foreach(string TagName in FindTagNamesFromList(Parameters.Tag))
{
FindOrAddTagSet(TagNameToFileSet, TagName).UnionWith(CookedFiles);
}
// Add them to the set of build products
BuildProducts.UnionWith(CookedFiles);
return true;
}
/// <summary>
/// Retrieve an output of the given node. Fetches and decompresses the files from shared storage if necessary, or validates the local files.
/// </summary>
/// <param name="NodeName">The node which created the storage block</param>
/// <param name="OutputName">Name of the block to retrieve. May be null or empty.</param>
/// <returns>Manifest of the files retrieved</returns>
public TempStorageManifest Retreive(string NodeName, string OutputName)
{
using(var TelemetryStopwatch = new TelemetryStopwatch("RetrieveFromTempStorage"))
{
// Get the path to the local manifest
FileReference LocalManifestFile = GetManifestLocation(LocalDir, NodeName, OutputName);
bool bLocal = LocalManifestFile.Exists();
// Read the manifest, either from local storage or shared storage
TempStorageManifest Manifest;
if(bLocal)
{
CommandUtils.Log("Reading shared manifest from {0}", LocalManifestFile.FullName);
Manifest = TempStorageManifest.Load(LocalManifestFile);
}
else
{
// Check we have shared storage
if(SharedDir == null)
{
throw new AutomationException("Missing local manifest for node - {0}", LocalManifestFile.FullName);
}
// Get the shared directory for this node
FileReference SharedManifestFile = GetManifestLocation(SharedDir, NodeName, OutputName);
// Make sure the manifest exists
if(!SharedManifestFile.Exists())
{
throw new AutomationException("Missing local or shared manifest for node - {0}", SharedManifestFile.FullName);
}
// Read the shared manifest
CommandUtils.Log("Copying shared manifest from {0} to {1}", SharedManifestFile.FullName, LocalManifestFile.FullName);
Manifest = TempStorageManifest.Load(SharedManifestFile);
// Unzip all the build products
DirectoryReference SharedNodeDir = GetDirectoryForNode(SharedDir, NodeName);
FileInfo[] ZipFiles = Manifest.ZipFiles.Select(x => new FileInfo(FileReference.Combine(SharedNodeDir, x.Name).FullName)).ToArray();
ParallelUnzipFiles(ZipFiles, RootDir);
// Fix any Unix permissions/chmod issues, and update the timestamps to match the manifest. Zip files only use local time, and there's no guarantee it matches the local clock.
foreach(TempStorageFile ManifestFile in Manifest.Files)
{
FileReference File = ManifestFile.ToFileReference(RootDir);
if (Utils.IsRunningOnMono)
{
CommandUtils.FixUnixFilePermissions(File.FullName);
}
System.IO.File.SetLastWriteTimeUtc(File.FullName, new DateTime(ManifestFile.LastWriteTimeUtcTicks, DateTimeKind.Utc));
}
// Save the manifest locally
LocalManifestFile.Directory.CreateDirectory();
Manifest.Save(LocalManifestFile);
}
// Check all the local files are as expected
bool bAllMatch = true;
foreach(TempStorageFile File in Manifest.Files)
{
bAllMatch &= File.Compare(RootDir);
}
if(!bAllMatch)
{
throw new AutomationException("Files have been modified");
}
// Update the stats and return
TelemetryStopwatch.Finish(string.Format("RetrieveFromTempStorage.{0}.{1}.{2}.{3}.{4}.{5}.{6}", Manifest.Files.Length, Manifest.Files.Sum(x => x.Length), bLocal? 0 : Manifest.ZipFiles.Sum(x => x.Length), bLocal? "Local" : "Remote", 0, 0, OutputName));
return Manifest;
}
}
public BranchInfo(List<UnrealTargetPlatform> InHostPlatforms)
{
BaseEngineProject = new BranchUProject();
var AllProjects = UnrealBuildTool.UProjectInfo.FilterGameProjects(false, null);
using(TelemetryStopwatch SortProjectsStopwatch = new TelemetryStopwatch("SortProjects"))
{
foreach (var InfoEntry in AllProjects)
{
var UProject = new BranchUProject(InfoEntry);
if (UProject.Properties.bIsCodeBasedProject)
{
CodeProjects.Add(UProject);
}
else
{
NonCodeProjects.Add(UProject);
// the base project uses BlankProject if it really needs a .uproject file
if (String.IsNullOrEmpty(BaseEngineProject.FilePath) && UProject.GameName == "BlankProject")
{
BaseEngineProject.FilePath = UProject.FilePath;
}
}
}
}
/* if (String.IsNullOrEmpty(BaseEngineProject.FilePath))
{
throw new AutomationException("All branches must have the blank project /Samples/Sandbox/BlankProject");
}*/
using(TelemetryStopwatch ProjectDumpStopwatch = new TelemetryStopwatch("Project Dump"))
{
CommandUtils.LogVerbose(" Base Engine:");
BaseEngineProject.Dump(InHostPlatforms);
CommandUtils.LogVerbose(" {0} Code projects:", CodeProjects.Count);
foreach (var Proj in CodeProjects)
{
Proj.Dump(InHostPlatforms);
}
CommandUtils.LogVerbose(" {0} Non-Code projects:", CodeProjects.Count);
foreach (var Proj in NonCodeProjects)
{
Proj.Dump(InHostPlatforms);
}
}
}
private void DoCommanderSetup(ElectricCommander EC, IEnumerable<BuildNode> AllNodes, IEnumerable<AggregateNode> AllAggregates, List<BuildNode> OrdereredToDo, int TimeIndex, int TimeQuantum, bool bSkipTriggers, bool bFake, bool bFakeEC, string CLString, TriggerNode ExplicitTrigger, List<TriggerNode> UnfinishedTriggers, string FakeFail, bool bPreflightBuild)
{
List<BuildNode> SortedNodes = TopologicalSort(new HashSet<BuildNode>(AllNodes), null, SubSort: false, DoNotConsiderCompletion: true);
Log("******* {0} GUBP Nodes", SortedNodes.Count);
List<BuildNode> FilteredOrdereredToDo = new List<BuildNode>();
using(TelemetryStopwatch StartFilterTimer = new TelemetryStopwatch("FilterNodes"))
{
// remove nodes that have unfinished triggers
foreach (BuildNode NodeToDo in OrdereredToDo)
{
if (NodeToDo.ControllingTriggers.Length == 0 || !UnfinishedTriggers.Contains(NodeToDo.ControllingTriggers.Last()))
{
// if we are triggering, then remove nodes that are not controlled by the trigger or are dependencies of this trigger
if (ExplicitTrigger != null && ExplicitTrigger != NodeToDo && !ExplicitTrigger.DependsOn(NodeToDo) && !NodeToDo.DependsOn(ExplicitTrigger))
{
continue; // this wasn't on the chain related to the trigger we are triggering, so it is not relevant
}
// in preflight builds, we are either skipping triggers (and running things downstream) or we just stop at triggers and don't make them available for triggering.
if (bPreflightBuild && !bSkipTriggers && (NodeToDo is TriggerNode))
{
continue;
}
FilteredOrdereredToDo.Add(NodeToDo);
}
}
}
using(TelemetryStopwatch PrintNodesTimer = new TelemetryStopwatch("SetupCommanderPrint"))
{
Log("*********** EC Nodes, in order.");
PrintNodes(this, FilteredOrdereredToDo, AllAggregates, UnfinishedTriggers, TimeQuantum);
}
EC.DoCommanderSetup(AllNodes, AllAggregates, FilteredOrdereredToDo, SortedNodes, TimeIndex, TimeQuantum, bSkipTriggers, bFake, bFakeEC, CLString, ExplicitTrigger, UnfinishedTriggers, FakeFail);
}
/// <summary>
/// Inverse of <see cref="StoreToTempStorage"/>.
/// Copies a block of files from a temp storage location given by a temp storage node and game to local storage rooted at the given root dir.
/// If the temp manifest for this block is found locally, the copy is skipped, as we assume this is the same machine that created the temp storage and the files are still there.
/// </summary>
/// <param name="TempStorageNodeInfo">Node info descibing the block of temp storage (essentially used to identify a subdirectory insides the game's temp storage folder).</param>
/// <param name="WasLocal">upon return, this parameter is set to true if the temp manifest was found locally and the copy was avoided.</param>
/// <param name="GameName">game name to determine the temp storage folder for. Empty is equivalent to "UE4".</param>
/// <param name="RootDir">Folder that all the retrieved files should be rooted from. If null or empty, CmdEnv.LocalRoot is used.</param>
/// <returns>List of fully qualified paths to all the files that were retrieved. This is returned even if we skip the copy (set WasLocal = true) .</returns>
public static List<string> RetrieveFromTempStorage(TempStorageNodeInfo TempStorageNodeInfo, out bool WasLocal, string GameName, string RootDir)
{
using (var TelemetryStopwatch = new TelemetryStopwatch("RetrieveFromTempStorage"))
{
// use LocalRoot if one is not specified
if (String.IsNullOrEmpty(RootDir))
{
RootDir = CommandUtils.CmdEnv.LocalRoot;
}
// First see if the local manifest is there.
// If it is, then we must be on the same node as the one that originally created the temp storage.
// In that case, we just verify all the files exist as described in the manifest and use that.
// If there was any tampering, abort immediately because we never accept tampering, and it signifies a build problem.
var LocalManifest = LocalTempStorageManifestFilename(TempStorageNodeInfo);
if (CommandUtils.FileExists_NoExceptions(LocalManifest))
{
CommandUtils.LogVerbose("Found local manifest {0}", LocalManifest);
var Local = TempStorageManifest.Load(LocalManifest);
var Files = Local.GetFiles(RootDir);
var LocalTest = TempStorageManifest.Create(Files, RootDir);
if (!Local.Compare(LocalTest))
{
throw new AutomationException("Local files in manifest {0} were tampered with.", LocalManifest);
}
WasLocal = true;
TelemetryStopwatch.Finish(string.Format("RetrieveFromTempStorage.{0}.{1}.{2}.Local.{3}.{4}.{5}", Files.Count, Local.GetTotalSize(), 0L, 0L, 0L, TempStorageNodeInfo.NodeStorageName));
return Files;
}
WasLocal = false;
// We couldn't find the node storage locally, so get it from the shared location.
var BlockPath = SharedTempStorageDirectory(TempStorageNodeInfo, GameName);
CommandUtils.LogVerbose("Attempting to retrieve from {0}", BlockPath);
if (!CommandUtils.DirectoryExists_NoExceptions(BlockPath))
{
throw new AutomationException("Storage Block Does Not Exists! {0}", BlockPath);
}
var SharedManifest = SharedTempStorageManifestFilename(TempStorageNodeInfo, GameName);
InternalUtils.Robust_FileExists(SharedManifest, "Storage Block Manifest Does Not Exists! {0}");
var Shared = TempStorageManifest.Load(SharedManifest);
var SharedFiles = Shared.GetFiles(BlockPath, !bZipTempStorage);
// We know the source files exist and are under RootDir because we created the manifest, which verifies it.
// Now create the list of target files
var DestFiles = SharedFiles.Select(Filename => CommandUtils.MakeRerootedFilePath(Filename, BlockPath, RootDir)).ToList();
long ZipFileSize = 0L;
long CopyTimeMS = 0L;
long UnzipTimeMS = 0L;
if (bZipTempStorage)
{
var LocalZipFilename = Path.ChangeExtension(LocalManifest, "zip");
var SharedZipFilename = Path.ChangeExtension(SharedManifest, "zip");
if (!CommandUtils.FileExists(SharedZipFilename))
{
throw new AutomationException("Storage block zip file does not exist! {0}", SharedZipFilename);
}
var CopyTimer = DateTimeStopwatch.Start();
InternalUtils.Robust_CopyFile(SharedZipFilename, LocalZipFilename);
CopyTimeMS = (long)CopyTimer.ElapsedTime.TotalMilliseconds;
ZipFileSize = new FileInfo(LocalZipFilename).Length;
var UnzipTimer = DateTimeStopwatch.Start();
using (Ionic.Zip.ZipFile Zip = new Ionic.Zip.ZipFile(LocalZipFilename))
{
Zip.ExtractAll(RootDir, Ionic.Zip.ExtractExistingFileAction.OverwriteSilently);
}
CommandUtils.DeleteFile(LocalZipFilename);
UnzipTimeMS = (long)UnzipTimer.ElapsedTime.TotalMilliseconds;
}
else
{
var CopyTimer = DateTimeStopwatch.Start();
// If the local file already exists, it will be overwritten.
foreach (var DestFile in DestFiles)
{
if (CommandUtils.FileExists_NoExceptions(true, DestFile))
{
CommandUtils.LogVerbose("Dest file {0} already exists, deleting and overwriting", DestFile);
CommandUtils.DeleteFile(DestFile);
}
}
// Do the threaded copy to the local file system.
CommandUtils.ThreadedCopyFiles(SharedFiles, DestFiles, ThreadsToCopyWith());
CopyTimeMS = (long)CopyTimer.ElapsedTime.TotalMilliseconds;
}
// Handle unix permissions/chmod issues.
if (UnrealBuildTool.Utils.IsRunningOnMono)
{
foreach (string DestFile in DestFiles)
{
CommandUtils.FixUnixFilePermissions(DestFile);
}
}
var NewLocal = SaveLocalTempStorageManifest(RootDir, TempStorageNodeInfo, DestFiles);
// Now compare the created local files to ensure their attributes match the one we copied from the network.
if (!NewLocal.Compare(Shared))
{
// we will rename this so it can't be used, but leave it around for inspection
CommandUtils.RenameFile_NoExceptions(LocalManifest, LocalManifest + ".broken");
throw new AutomationException("Shared and Local manifest mismatch.");
}
TelemetryStopwatch.Finish(string.Format("RetrieveFromTempStorage.{0}.{1}.{2}.Remote.{3}.{4}.{5}", DestFiles.Count, Shared.GetTotalSize(), ZipFileSize, CopyTimeMS, UnzipTimeMS, TempStorageNodeInfo.NodeStorageName));
return DestFiles;
}
}
/// <summary>
/// Cleans all temp storage data from the given directory that is older than a certain threshold (defined directly in the code).
/// </summary>
/// <param name="TopDirectory">Fully qualified path of the folder that is the root of a bunch of temp storage folders. Should be a string returned by <see cref="ResolveSharedTempStorageDirectory"/> </param>
private static void CleanSharedTempStorage(string TopDirectory)
{
CleanSharedTempStorageLegacy(TopDirectory);
Action<string> TryToDeletePossiblyEmptyFolder = (string FolderName) =>
{
try
{
Directory.Delete(FolderName);
}
catch (IOException)
{
// only catch "directory is not empty type exceptions, if possible. Best we can do is check for IOException.
}
catch (Exception Ex)
{
CommandUtils.LogWarning("Unexpected failure trying to delete possibly empty temp folder {0}: {1}", FolderName, Ex);
}
};
CommandUtils.LogConsole("Cleaning temp storage for {0}...", TopDirectory);
int FoldersCleaned = 0;
int FoldersManuallyCleaned = 0;
int FoldersFailedCleaned = 0;
using (var TelemetryStopwatch = new TelemetryStopwatch("CleanSharedTempStorage"))
{
const double MaximumDaysToKeepTempStorage = 3;
var Now = DateTime.UtcNow;
// This will search legacy folders as well, but those should go away within a few days.
foreach (var OldManifestInfo in
// First subdirectory is the branch name
from BranchDir in Directory.EnumerateDirectories(TopDirectory)
// Second subdirectory is the changelist
from CLDir in Directory.EnumerateDirectories(BranchDir)
// third subdirectory is the node storage name
from NodeNameDir in Directory.EnumerateDirectories(CLDir)
// now look for manifest files.
let ManifestFile = Directory.EnumerateFiles(NodeNameDir, "*.TempManifest").SingleOrDefault()
where ManifestFile != null
// only choose ones that are old enough.
where (Now - new FileInfo(ManifestFile).LastWriteTimeUtc).TotalDays > MaximumDaysToKeepTempStorage
select new { BranchDir, CLDir, NodeNameDir })
{
try
{
CommandUtils.LogConsole("Deleting folder with old temp storage {0}...", OldManifestInfo.NodeNameDir);
Directory.Delete(OldManifestInfo.NodeNameDir, true);
FoldersCleaned++;
}
catch (Exception Ex)
{
CommandUtils.LogWarning("Failed to delete old manifest folder '{0}', will try one file at a time: {1}", OldManifestInfo.NodeNameDir, Ex);
if (CommandUtils.DeleteDirectory_NoExceptions(true, OldManifestInfo.NodeNameDir))
{
FoldersManuallyCleaned++;
}
else
{
FoldersFailedCleaned++;
}
}
// Once we are done, try to delete the CLDir and BranchDir (will fail if not empty, so no worries).
TryToDeletePossiblyEmptyFolder(OldManifestInfo.CLDir);
TryToDeletePossiblyEmptyFolder(OldManifestInfo.BranchDir);
}
TelemetryStopwatch.Finish(string.Format("CleanSharedTempStorage.{0}.{1}.{2}", FoldersCleaned, FoldersManuallyCleaned, FoldersFailedCleaned));
}
}
public override void DoBuild(GUBP bp)
{
BuildProducts = new List<string>();
var UE4Build = new UE4Build(bp);
UE4Build.BuildAgenda Agenda = GetAgenda(bp);
if (Agenda != null)
{
bool ReallyDeleteBuildProducts = DeleteBuildProducts() && !BranchConfig.bForceIncrementalCompile;
Agenda.DoRetries = false; // these would delete build products
bool UseParallelExecutor = bDependentOnCompileTools && (HostPlatform == UnrealTargetPlatform.Win64);
UE4Build.Build(Agenda, InDeleteBuildProducts: ReallyDeleteBuildProducts, InUpdateVersionFiles: false, InForceNoXGE: true, InForceUnity: true, InUseParallelExecutor: UseParallelExecutor);
using(TelemetryStopwatch PostBuildStopwatch = new TelemetryStopwatch("PostBuild"))
{
PostBuild(bp, UE4Build);
}
UE4Build.CheckBuildProducts(UE4Build.BuildProductFiles);
foreach (var Product in UE4Build.BuildProductFiles)
{
AddBuildProduct(Product);
}
RemoveOveralppingBuildProducts();
PostBuildProducts(bp);
}
if (Agenda == null || (BuildProducts.Count == 0 && BranchConfig.bForceIncrementalCompile))
{
SaveRecordOfSuccessAndAddToBuildProducts("Nothing to actually compile");
}
}
XGEItem XGEPrepareBuildWithUBT(string TargetName, UnrealBuildTool.UnrealTargetPlatform Platform, string Config, FileReference UprojectPath, bool ForceMonolithic = false, bool ForceNonUnity = false, bool ForceDebugInfo = false, string InAddArgs = "", bool ForceUnity = false, Dictionary<string, string> EnvVars = null)
{
string AddArgs = "";
if (UprojectPath != null)
{
AddArgs += " " + CommandUtils.MakePathSafeToUseWithCommandLine(UprojectPath.FullName);
}
AddArgs += " " + InAddArgs;
if (ForceMonolithic)
{
AddArgs += " -monolithic";
}
if (ForceNonUnity)
{
AddArgs += " -disableunity";
}
if (ForceUnity)
{
AddArgs += " -forceunity";
}
if (ForceDebugInfo)
{
AddArgs += " -forcedebuginfo";
}
PrepareUBT();
string UBTManifest = GetUBTManifest(UprojectPath, AddArgs);
DeleteFile(UBTManifest);
XGEItem Result = new XGEItem();
ClearExportedXGEXML();
using(TelemetryStopwatch GenerateManifestStopwatch = new TelemetryStopwatch("GenerateXGEManifest.{0}.{1}.{2}", TargetName, Platform.ToString(), Config))
{
RunUBT(CmdEnv, UBTExecutable: UBTExecutable, Project: UprojectPath, Target: TargetName, Platform: Platform.ToString(), Config: Config, AdditionalArgs: "-generatemanifest -nobuilduht -xgeexport" + AddArgs, EnvVars: EnvVars);
}
PrepareManifest(UBTManifest, true);
Result.Platform = Platform;
Result.Config = Config;
Result.TargetName = TargetName;
Result.UProjectPath = UprojectPath;
Result.Manifest = ReadManifest(UBTManifest);
Result.OutputCaption = String.Format("{0}-{1}-{2}", TargetName, Platform.ToString(), Config.ToString());
DeleteFile(UBTManifest);
Result.CommandLine = UBTExecutable + " " + UBTCommandline(Project: UprojectPath, Target: TargetName, Platform: Platform.ToString(), Config: Config, AdditionalArgs: "-noxge -nobuilduht" + AddArgs);
Result.XgeXmlFiles = new List<string>();
foreach (var XGEFile in FindXGEFiles())
{
if (!FileExists_NoExceptions(XGEFile))
{
throw new AutomationException("BUILD FAILED: Couldn't find file: {0}", XGEFile);
}
int FileNum = 0;
string OutFile;
while (true)
{
OutFile = CombinePaths(CmdEnv.LogFolder, String.Format("UBTExport.{0}.xge.xml", FileNum));
FileInfo ItemInfo = new FileInfo(OutFile);
if (!ItemInfo.Exists)
{
break;
}
FileNum++;
}
CopyFile(XGEFile, OutFile);
Result.XgeXmlFiles.Add(OutFile);
}
ClearExportedXGEXML();
return Result;
}
void CleanWithUBT(string TargetName, UnrealBuildTool.UnrealTargetPlatform Platform, string Config, FileReference UprojectPath, bool ForceMonolithic = false, bool ForceNonUnity = false, bool ForceDebugInfo = false, string InAddArgs = "", bool ForceUnity = false, Dictionary<string, string> EnvVars = null)
{
string AddArgs = "";
if (UprojectPath != null)
{
AddArgs += " " + CommandUtils.MakePathSafeToUseWithCommandLine(UprojectPath.FullName);
}
AddArgs += " " + InAddArgs;
if (ForceMonolithic)
{
AddArgs += " -monolithic";
}
if (ForceNonUnity)
{
AddArgs += " -disableunity";
}
if (ForceUnity)
{
AddArgs += " -forceunity";
}
if (ForceDebugInfo)
{
AddArgs += " -forcedebuginfo";
}
if (!TargetName.Equals("UnrealHeaderTool", StringComparison.InvariantCultureIgnoreCase))
{
AddArgs += " -nobuilduht";
}
PrepareUBT();
using(TelemetryStopwatch CleanStopwatch = new TelemetryStopwatch("CleanWithUBT.{0}.{1}.{2}", TargetName, Platform.ToString(), Config))
{
RunUBT(CmdEnv, UBTExecutable: UBTExecutable, Project: UprojectPath, Target: TargetName, Platform: Platform.ToString(), Config: Config, AdditionalArgs: "-clean" + AddArgs, EnvVars: EnvVars);
}
}
/// <summary>
/// Main method.
/// </summary>
/// <param name="Arguments">Command line</param>
public static ExitCode Process(string[] Arguments)
{
// Initial check for local or build machine runs BEFORE we parse the command line (We need this value set
// in case something throws the exception while parsing the command line)
IsBuildMachine = !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("uebp_LOCAL_ROOT"));
// Scan the command line for commands to execute.
var CommandsToExecute = new List<CommandInfo>();
string OutScriptsForProjectFileName;
var AdditionalScriptsFolders = new List<string>();
ParseCommandLine(Arguments, CommandsToExecute, out OutScriptsForProjectFileName, AdditionalScriptsFolders);
// Get the path to the telemetry file, if present
string TelemetryFile = CommandUtils.ParseParamValue(Arguments, "-Telemetry");
// Check for build machine override (force local)
IsBuildMachine = GlobalCommandLine.ForceLocal ? false : IsBuildMachine;
Log.TraceVerbose("IsBuildMachine={0}", IsBuildMachine);
Environment.SetEnvironmentVariable("IsBuildMachine", IsBuildMachine ? "1" : "0");
// should we kill processes on exit
ShouldKillProcesses = !GlobalCommandLine.NoKill;
Log.TraceVerbose("ShouldKillProcesses={0}", ShouldKillProcesses);
if (CommandsToExecute.Count == 0 && GlobalCommandLine.Help)
{
DisplayHelp();
return ExitCode.Success;
}
// Disable AutoSDKs if specified on the command line
if (GlobalCommandLine.NoAutoSDK)
{
UEBuildPlatformSDK.bAllowAutoSDKSwitching = false;
}
// Setup environment
Log.TraceInformation("Setting up command environment.");
CommandUtils.InitCommandEnvironment();
// Change CWD to UE4 root.
Environment.CurrentDirectory = CommandUtils.CmdEnv.LocalRoot;
// Fill in the project info
UnrealBuildTool.UProjectInfo.FillProjectInfo();
// Clean rules folders up
ProjectUtils.CleanupFolders();
// Compile scripts.
ScriptCompiler Compiler = new ScriptCompiler();
using(TelemetryStopwatch ScriptCompileStopwatch = new TelemetryStopwatch("ScriptCompile"))
{
Compiler.FindAndCompileAllScripts(OutScriptsForProjectFileName, AdditionalScriptsFolders);
}
if (GlobalCommandLine.CompileOnly)
{
Log.TraceInformation("Compilation successful, exiting (CompileOnly)");
return ExitCode.Success;
}
if (GlobalCommandLine.List)
{
ListAvailableCommands(Compiler.Commands);
return ExitCode.Success;
}
if (GlobalCommandLine.Help)
{
DisplayHelp(CommandsToExecute, Compiler.Commands);
return ExitCode.Success;
}
// Enable or disable P4 support
CommandUtils.InitP4Support(CommandsToExecute, Compiler.Commands);
if (CommandUtils.P4Enabled)
{
Log.TraceInformation("Setting up Perforce environment.");
CommandUtils.InitP4Environment();
CommandUtils.InitDefaultP4Connection();
}
// Find and execute commands.
ExitCode Result = Execute(CommandsToExecute, Compiler.Commands);
if (TelemetryFile != null)
{
Directory.CreateDirectory(Path.GetDirectoryName(TelemetryFile));
CommandUtils.Telemetry.Write(TelemetryFile);
}
return Result;
}
void ExecuteNodes(ElectricCommander EC, List<BuildNode> OrdereredToDo, bool bFake, bool bFakeEC, bool bSaveSharedTempStorage, string CLString, string StoreName, string FakeFail)
{
Dictionary<string, BuildNode> BuildProductToNodeMap = new Dictionary<string, BuildNode>();
foreach (BuildNode NodeToDo in OrdereredToDo)
{
if (NodeToDo.Node.BuildProducts != null || NodeToDo.Node.AllDependencyBuildProducts != null)
{
throw new AutomationException("topological sort error");
}
NodeToDo.Node.AllDependencyBuildProducts = new List<string>();
NodeToDo.Node.AllDependencies = new List<string>();
foreach (BuildNode Dep in NodeToDo.Dependencies)
{
NodeToDo.Node.AddAllDependent(Dep.Name);
if (Dep.Node.AllDependencies == null)
{
throw new AutomationException("Node {0} was not processed yet? Processing {1}", Dep, NodeToDo.Name);
}
foreach (string DepDep in Dep.Node.AllDependencies)
{
NodeToDo.Node.AddAllDependent(DepDep);
}
if (Dep.Node.BuildProducts == null)
{
throw new AutomationException("Node {0} was not processed yet? Processing {1}", Dep, NodeToDo.Name);
}
foreach (string Prod in Dep.Node.BuildProducts)
{
NodeToDo.Node.AddDependentBuildProduct(Prod);
}
if (Dep.Node.AllDependencyBuildProducts == null)
{
throw new AutomationException("Node {0} was not processed yet2? Processing {1}", Dep.Name, NodeToDo.Name);
}
foreach (string Prod in Dep.Node.AllDependencyBuildProducts)
{
NodeToDo.Node.AddDependentBuildProduct(Prod);
}
}
string NodeStoreName = StoreName + "-" + NodeToDo.Name;
string GameNameIfAny = NodeToDo.Node.GameNameIfAnyForTempStorage();
string StorageRootIfAny = NodeToDo.Node.RootIfAnyForTempStorage();
if (bFake)
{
StorageRootIfAny = ""; // we don't rebase fake runs since those are entirely "records of success", which are always in the logs folder
}
// this is kinda complicated
bool SaveSuccessRecords = (IsBuildMachine || bFakeEC) && // no real reason to make these locally except for fakeEC tests
(!(NodeToDo is TriggerNode) || NodeToDo.IsSticky); // trigger nodes are run twice, one to start the new workflow and once when it is actually triggered, we will save reconds for the latter
Log("***** Running GUBP Node {0} -> {1} : {2}", NodeToDo.Name, GameNameIfAny, NodeStoreName);
if (NodeToDo.IsComplete)
{
if (NodeToDo.Name == VersionFilesNode.StaticGetFullName() && !IsBuildMachine)
{
Log("***** NOT ****** Retrieving GUBP Node {0} from {1}; it is the version files.", NodeToDo.Name, NodeStoreName);
NodeToDo.Node.BuildProducts = new List<string>();
}
else
{
Log("***** Retrieving GUBP Node {0} from {1}", NodeToDo.Name, NodeStoreName);
bool WasLocal;
try
{
NodeToDo.Node.BuildProducts = TempStorage.RetrieveFromTempStorage(CmdEnv, NodeStoreName, out WasLocal, GameNameIfAny, StorageRootIfAny);
}
catch
{
if(GameNameIfAny != "")
{
NodeToDo.Node.BuildProducts = TempStorage.RetrieveFromTempStorage(CmdEnv, NodeStoreName, out WasLocal, "", StorageRootIfAny);
}
else
{
throw new AutomationException("Build Products cannot be found for node {0}", NodeToDo.Name);
}
}
if (!WasLocal)
{
NodeToDo.Node.PostLoadFromSharedTempStorage(this);
}
}
}
else
{
if (SaveSuccessRecords)
{
EC.SaveStatus(NodeToDo, StartedTempStorageSuffix, NodeStoreName, bSaveSharedTempStorage, GameNameIfAny);
}
double BuildDuration = 0.0;
try
{
if (!String.IsNullOrEmpty(FakeFail) && FakeFail.Equals(NodeToDo.Name, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Failing node {0} by request.", NodeToDo.Name);
}
if (bFake)
{
Log("***** FAKE!! Building GUBP Node {0} for {1}", NodeToDo.Name, NodeStoreName);
NodeToDo.Node.DoFakeBuild(this);
}
else
{
Log("***** Building GUBP Node {0} for {1}", NodeToDo.Name, NodeStoreName);
DateTime StartTime = DateTime.UtcNow;
using(TelemetryStopwatch DoBuildStopwatch = new TelemetryStopwatch("DoBuild.{0}", NodeToDo.Name))
{
NodeToDo.Node.DoBuild(this);
}
BuildDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds / 1000;
}
using(TelemetryStopwatch StoreBuildProductsStopwatch = new TelemetryStopwatch("StoreBuildProducts"))
{
double StoreDuration = 0.0;
DateTime StartTime = DateTime.UtcNow;
TempStorage.StoreToTempStorage(CmdEnv, NodeStoreName, NodeToDo.Node.BuildProducts, !bSaveSharedTempStorage, GameNameIfAny, StorageRootIfAny);
StoreDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds / 1000;
Log("Took {0} seconds to store build products", StoreDuration);
if (IsBuildMachine)
{
EC.RunECTool(String.Format("setProperty \"/myJobStep/StoreDuration\" \"{0}\"", StoreDuration.ToString()));
}
}
if (ParseParam("StompCheck"))
{
foreach (string Dep in NodeToDo.Node.AllDependencies)
{
try
{
bool WasLocal;
using(TelemetryStopwatch RetrieveBuildProductsStopwatch = new TelemetryStopwatch("RetrieveBuildProducts"))
{
TempStorage.RetrieveFromTempStorage(CmdEnv, NodeStoreName, out WasLocal, GameNameIfAny, StorageRootIfAny);
}
if (!WasLocal)
{
throw new AutomationException("Retrieve was not local?");
}
}
catch(Exception Ex)
{
throw new AutomationException("Node {0} stomped Node {1} Ex: {2}", NodeToDo.Name, Dep, LogUtils.FormatException(Ex));
}
}
}
}
catch (Exception Ex)
{
NodeHistory History = null;
if (SaveSuccessRecords)
{
using(TelemetryStopwatch UpdateNodeHistoryStopwatch = new TelemetryStopwatch("UpdateNodeHistory"))
{
History = FindNodeHistory(NodeToDo, CLString, StoreName);
}
using(TelemetryStopwatch SaveNodeStatusStopwatch = new TelemetryStopwatch("SaveNodeStatus"))
{
EC.SaveStatus(NodeToDo, FailedTempStorageSuffix, NodeStoreName, bSaveSharedTempStorage, GameNameIfAny, ParseParamValue("MyJobStepId"));
}
using(TelemetryStopwatch UpdateECPropsStopwatch = new TelemetryStopwatch("UpdateECProps"))
{
EC.UpdateECProps(NodeToDo);
}
if (IsBuildMachine)
{
using(TelemetryStopwatch GetFailEmailsStopwatch = new TelemetryStopwatch("GetFailEmails"))
{
GetFailureEmails(EC, NodeToDo, History, CLString, StoreName);
}
}
EC.UpdateECBuildTime(NodeToDo, BuildDuration);
}
Log("{0}", ExceptionToString(Ex));
if (History != null)
{
Log("Changes since last green *********************************");
Log("");
Log("");
Log("");
PrintDetailedChanges(History, P4Env.Changelist);
Log("End changes since last green");
}
string FailInfo = "";
FailInfo += "********************************* Main log file";
FailInfo += Environment.NewLine + Environment.NewLine;
FailInfo += LogUtils.GetLogTail();
FailInfo += Environment.NewLine + Environment.NewLine + Environment.NewLine;
string OtherLog = "See logfile for details: '";
if (FailInfo.Contains(OtherLog))
{
string LogFile = FailInfo.Substring(FailInfo.IndexOf(OtherLog) + OtherLog.Length);
if (LogFile.Contains("'"))
{
LogFile = CombinePaths(CmdEnv.LogFolder, LogFile.Substring(0, LogFile.IndexOf("'")));
if (FileExists_NoExceptions(LogFile))
{
FailInfo += "********************************* Sub log file " + LogFile;
FailInfo += Environment.NewLine + Environment.NewLine;
FailInfo += LogUtils.GetLogTail(LogFile);
FailInfo += Environment.NewLine + Environment.NewLine + Environment.NewLine;
}
}
}
string Filename = CombinePaths(CmdEnv.LogFolder, "LogTailsAndChanges.log");
WriteAllText(Filename, FailInfo);
throw(Ex);
}
if (SaveSuccessRecords)
{
NodeHistory History = null;
using(TelemetryStopwatch UpdateNodeHistoryStopwatch = new TelemetryStopwatch("UpdateNodeHistory"))
{
History = FindNodeHistory(NodeToDo, CLString, StoreName);
}
using(TelemetryStopwatch SaveNodeStatusStopwatch = new TelemetryStopwatch("SaveNodeStatus"))
{
EC.SaveStatus(NodeToDo, SucceededTempStorageSuffix, NodeStoreName, bSaveSharedTempStorage, GameNameIfAny);
}
using(TelemetryStopwatch UpdateECPropsStopwatch = new TelemetryStopwatch("UpdateECProps"))
{
EC.UpdateECProps(NodeToDo);
}
if (IsBuildMachine)
{
using(TelemetryStopwatch GetFailEmailsStopwatch = new TelemetryStopwatch("GetFailEmails"))
{
GetFailureEmails(EC, NodeToDo, History, CLString, StoreName);
}
}
EC.UpdateECBuildTime(NodeToDo, BuildDuration);
}
}
foreach (string Product in NodeToDo.Node.BuildProducts)
{
if (BuildProductToNodeMap.ContainsKey(Product))
{
throw new AutomationException("Overlapping build product: {0} and {1} both produce {2}", BuildProductToNodeMap[Product], NodeToDo.Name, Product);
}
BuildProductToNodeMap.Add(Product, NodeToDo);
}
}
PrintRunTime();
}
/// <summary>
/// Legayc code to clean all temp storage data from the given directory that is older than a certain threshold (defined directly in the code).
/// </summary>
/// <param name="TopDirectory">Fully qualified path of the folder that is the root of a bunch of temp storage folders. Should be a string returned by <see cref="ResolveSharedTempStorageDirectory"/> </param>
private static void CleanSharedTempStorageLegacy(string TopDirectory)
{
// This is a hack to clean out the old temp storage in the old folder name.
TopDirectory = TopDirectory.Replace(Path.DirectorySeparatorChar + TempStorageSubdirectoryName + Path.DirectorySeparatorChar, Path.DirectorySeparatorChar + "GUBP" + Path.DirectorySeparatorChar);
using (var TelemetryStopwatch = new TelemetryStopwatch("CleanSharedTempStorageLegacy"))
{
const int MaximumDaysToKeepLegacyTempStorage = 1;
var StartTimeDir = DateTime.UtcNow;
DirectoryInfo DirInfo = new DirectoryInfo(TopDirectory);
var TopLevelDirs = DirInfo.GetDirectories();
{
var BuildDuration = (DateTime.UtcNow - StartTimeDir).TotalMilliseconds;
CommandUtils.Log("Took {0}s to enumerate {1} directories.", BuildDuration / 1000, TopLevelDirs.Length);
}
foreach (var TopLevelDir in TopLevelDirs)
{
if (CommandUtils.DirectoryExists_NoExceptions(TopLevelDir.FullName))
{
bool bOld = false;
foreach (var ThisFile in CommandUtils.FindFiles_NoExceptions(true, "*.TempManifest", false, TopLevelDir.FullName))
{
FileInfo Info = new FileInfo(ThisFile);
if ((DateTime.UtcNow - Info.LastWriteTimeUtc).TotalDays > MaximumDaysToKeepLegacyTempStorage)
{
bOld = true;
}
}
if (bOld)
{
CommandUtils.Log("Deleting temp storage directory {0}, because it is more than {1} days old.", TopLevelDir.FullName, MaximumDaysToKeepLegacyTempStorage);
var StartTime = DateTime.UtcNow;
try
{
if (Directory.Exists(TopLevelDir.FullName))
{
// try the direct approach first
Directory.Delete(TopLevelDir.FullName, true);
}
}
catch
{
}
CommandUtils.DeleteDirectory_NoExceptions(true, TopLevelDir.FullName);
var BuildDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds;
CommandUtils.Log("Took {0}s to delete {1}.", BuildDuration / 1000, TopLevelDir.FullName);
}
}
}
}
}
