/// <summary> /// Runs UBT with the specified commandline. Automatically creates a logfile. When /// no LogName is specified, the executable name is used as logfile base name. /// </summary> /// <param name="Env">Environment to use.</param> /// <param name="CommandLine">Commandline to pass on to UBT.</param> /// <param name="LogName">Optional logfile name.</param> public static void RunUBT(CommandEnvironment Env, string UBTExecutable, string CommandLine, string LogName = null, Dictionary<string, string> EnvVars = null) { if (!FileExists(UBTExecutable)) { throw new AutomationException("Unable to find UBT executable: " + UBTExecutable); } if (GlobalCommandLine.Rocket) { CommandLine += " -rocket"; } if (!IsBuildMachine && UnrealBuildTool.BuildHostPlatform.Current.Platform == UnrealBuildTool.UnrealTargetPlatform.Mac) { CommandLine += " -nocreatestub"; } CommandLine += " -NoHotReloadFromIDE"; if (bJunkDeleted || GlobalCommandLine.IgnoreJunk) { // UBT has already deleted junk files, make sure it doesn't do it again CommandLine += " -ignorejunk"; } else { // UBT will delete junk on first run bJunkDeleted = true; } CommandUtils.RunAndLog(Env, UBTExecutable, CommandLine, LogName, EnvVars: EnvVars); }
protected virtual void InitEnvironment(P4Connection Connection, CommandEnvironment CmdEnv) { // // P4 Environment // P4Port = CommandUtils.GetEnvVar(EnvVarNames.P4Port); ClientRoot = CommandUtils.GetEnvVar(EnvVarNames.ClientRoot); User = CommandUtils.GetEnvVar(EnvVarNames.User); ChangelistStringInternal = CommandUtils.GetEnvVar(EnvVarNames.Changelist, null); Client = CommandUtils.GetEnvVar(EnvVarNames.Client); BuildRootP4 = CommandUtils.GetEnvVar(EnvVarNames.BuildRootP4); if (BuildRootP4.EndsWith("/", StringComparison.InvariantCultureIgnoreCase) || BuildRootP4.EndsWith("\\", StringComparison.InvariantCultureIgnoreCase)) { // We expect the build root to not end with a path separator BuildRootP4 = BuildRootP4.Substring(0, BuildRootP4.Length - 1); CommandUtils.SetEnvVar(EnvVarNames.BuildRootP4, BuildRootP4); } BuildRootEscaped = CommandUtils.GetEnvVar(EnvVarNames.BuildRootEscaped); LabelToSync = CommandUtils.GetEnvVar(EnvVarNames.LabelToSync); string CodeChangelistString = CommandUtils.GetEnvVar(EnvVarNames.CodeChangelist); if(!String.IsNullOrEmpty(CodeChangelistString)) { CodeChangelist = Int32.Parse(CodeChangelistString); } if (((CommandUtils.P4Enabled || CommandUtils.IsBuildMachine) && (ClientRoot == String.Empty || User == String.Empty || (String.IsNullOrEmpty(ChangelistStringInternal) && CommandUtils.IsBuildMachine) || Client == String.Empty || BuildRootP4 == String.Empty))) { Log.TraceInformation("P4Enabled={0}", CommandUtils.P4Enabled); Log.TraceInformation("ClientRoot={0}",ClientRoot ); Log.TraceInformation("User={0}", User); Log.TraceInformation("ChangelistString={0}", ChangelistStringInternal); Log.TraceInformation("Client={0}", Client); Log.TraceInformation("BuildRootP4={0}", BuildRootP4); throw new AutomationException("BUILD FAILED Perforce Environment is not set up correctly. Please check your environment variables."); } LabelPrefix = BuildRootP4 + "/"; if (CommandUtils.P4Enabled) { if (CommandUtils.IsBuildMachine || ChangelistStringInternal != null) { // We may not always need the changelist number if we're not a build machine. // In local runs, changelist initialization can be really slow! VerifyChangelistStringAndSetChangelistNumber(); } } LogSettings(); }
/// <summary> /// Builds a Visual Studio solution with MsDevEnv. Automatically creates a logfile. When /// no LogName is specified, the executable name is used as logfile base name. /// </summary> /// <param name="Env">BuildEnvironment to use.</param> /// <param name="SolutionFile">Path to the solution file</param> /// <param name="BuildConfig">Configuration to build.</param> /// <param name="LogName">Optional logfile name.</param> public static void BuildSolution(CommandEnvironment Env, string SolutionFile, string BuildConfig = "Development", string LogName = null) { if (!FileExists(SolutionFile)) { throw new AutomationException(String.Format("Unabled to build Solution {0}. Solution file not found.", SolutionFile)); } if (String.IsNullOrEmpty(Env.MsDevExe)) { throw new AutomationException(String.Format("Unabled to build Solution {0}. devenv.com not found.", SolutionFile)); } string CmdLine = String.Format("\"{0}\" /build \"{1}\"", SolutionFile, BuildConfig); RunAndLog(Env, Env.MsDevExe, CmdLine, LogName); }
/// <summary> /// Runs UBT with the specified commandline. Automatically creates a logfile. When /// no LogName is specified, the executable name is used as logfile base name. /// </summary> /// <param name="Env">Environment to use.</param> /// <param name="CommandLine">Commandline to pass on to UBT.</param> /// <param name="LogName">Optional logfile name.</param> public static void RunUBT(CommandEnvironment Env, string UBTExecutable, string CommandLine, string LogName = null) { if (!FileExists(UBTExecutable)) { throw new AutomationException("Unable to find UBT executable: " + UBTExecutable); } if (GlobalCommandLine.Rocket) { CommandLine += " -rocket"; } CommandUtils.RunAndLog(Env, UBTExecutable, CommandLine, LogName); }
/// <summary> /// Builds a CSharp project with msbuild.exe. Automatically creates a logfile. When /// no LogName is specified, the executable name is used as logfile base name. /// </summary> /// <param name="Env">BuildEnvironment to use.</param> /// <param name="ProjectFile">Path to the project file. Must be a .csproj file.</param> /// <param name="BuildConfig">Configuration to build.</param> /// <param name="LogName">Optional logfile name.</param> public static void BuildCSharpProject(CommandEnvironment Env, string ProjectFile, string BuildConfig = "Development", string LogName = null) { if (!ProjectFile.EndsWith(".csproj")) { throw new AutomationException(String.Format("Unabled to build Project {0}. Not a valid .csproj file.", ProjectFile)); } if (!FileExists(ProjectFile)) { throw new AutomationException(String.Format("Unabled to build Project {0}. Project file not found.", ProjectFile)); } string CmdLine = String.Format(@"/verbosity:normal /target:Rebuild /property:Configuration={0} /property:Platform=AnyCPU", BuildConfig); MsBuild(Env, ProjectFile, CmdLine, LogName); }
/// <summary> /// Builds a Visual Studio solution with MsBuild (using msbuild.exe rather than devenv.com can help circumvent issues with expired Visual Studio licenses). /// Automatically creates a logfile. When no LogName is specified, the executable name is used as logfile base name. /// </summary> /// <param name="Env">BuildEnvironment to use.</param> /// <param name="SolutionFile">Path to the solution file</param> /// <param name="BuildConfig">Configuration to build.</param> /// <param name="BuildPlatform">Platform to build.</param> /// <param name="LogName">Optional logfile name.</param> public static void BuildSolution(CommandEnvironment Env, string SolutionFile, string BuildConfig, string BuildPlatform, string LogName = null) { if (!FileExists(SolutionFile)) { throw new AutomationException(String.Format("Unabled to build Solution {0}. Solution file not found.", SolutionFile)); } if (String.IsNullOrEmpty(Env.MsBuildExe)) { throw new AutomationException("Unable to find msbuild.exe at: \"{0}\"", Env.MsBuildExe); } string CmdLine = String.Format("\"{0}\" /m /t:Build /p:Configuration=\"{1}\" /p:Platform=\"{2}\" /verbosity:minimal /nologo", SolutionFile, BuildConfig, BuildPlatform); using (TelemetryStopwatch CompileStopwatch = new TelemetryStopwatch("Compile.{0}.{1}.{2}", Path.GetFileName(SolutionFile), "WinC#", BuildConfig)) { RunAndLog(Env, Env.MsBuildExe, CmdLine, LogName); } }
/// <summary> /// Builds a Visual Studio solution with MsDevEnv. Automatically creates a logfile. When /// no LogName is specified, the executable name is used as logfile base name. /// </summary> /// <param name="Env">BuildEnvironment to use.</param> /// <param name="SolutionFile">Path to the solution file</param> /// <param name="BuildConfig">Configuration to build.</param> /// <param name="LogName">Optional logfile name.</param> public static void BuildSolution(CommandEnvironment Env, string SolutionFile, string BuildConfig = "Development", string LogName = null) { if (!FileExists(SolutionFile)) { throw new AutomationException(String.Format("Unabled to build Solution {0}. Solution file not found.", SolutionFile)); } if (String.IsNullOrEmpty(Env.MsDevExe)) { throw new AutomationException(String.Format("Unabled to build Solution {0}. devenv.com not found.", SolutionFile)); } string CmdLine = String.Format("\"{0}\" /build \"{1}\"", SolutionFile, BuildConfig); using(TelemetryStopwatch CompileStopwatch = new TelemetryStopwatch("Compile.{0}.{1}.{2}", Path.GetFileName(SolutionFile), "WinC#", BuildConfig)) { RunAndLog(Env, Env.MsDevExe, CmdLine, LogName); } }
/// <summary> /// Builds a Visual Studio solution with MsDevEnv. Automatically creates a logfile. When /// no LogName is specified, the executable name is used as logfile base name. /// </summary> /// <param name="Env">BuildEnvironment to use.</param> /// <param name="SolutionFile">Path to the solution file</param> /// <param name="BuildConfig">Configuration to build.</param> /// <param name="LogName">Optional logfile name.</param> public static void BuildSolution(CommandEnvironment Env, string SolutionFile, string BuildConfig = "Development", string LogName = null) { if (!FileExists(SolutionFile)) { throw new AutomationException(String.Format("Unabled to build Solution {0}. Solution file not found.", SolutionFile)); } if (String.IsNullOrEmpty(Env.MsDevExe)) { throw new AutomationException(String.Format("Unabled to build Solution {0}. devenv.com not found.", SolutionFile)); } string CmdLine = String.Format("\"{0}\" /build \"{1}\"", SolutionFile, BuildConfig); var StartBuildSolution = DateTime.Now.ToString(); RunAndLog(Env, Env.MsDevExe, CmdLine, LogName); var FinishBuildSolution = DateTime.Now.ToString(); PrintCSVFile(String.Format("UAT,Compile,{0},{1}", StartBuildSolution, FinishBuildSolution)); }
/// <summary> /// Runs msbuild.exe with the specified arguments. Automatically creates a logfile. When /// no LogName is specified, the executable name is used as logfile base name. /// </summary> /// <param name="Env">BuildEnvironment to use.</param> /// <param name="Project">Path to the project to build.</param> /// <param name="Arguments">Arguments to pass to msbuild.exe.</param> /// <param name="LogName">Optional logfile name.</param> public static void MsBuild(CommandEnvironment Env, string Project, string Arguments, string LogName) { if (String.IsNullOrEmpty(Env.MsBuildExe)) { throw new AutomationException("Unable to find msbuild.exe at: \"{0}\"", Env.MsBuildExe); } if (!FileExists(Project)) { throw new AutomationException("Project {0} does not exist!", Project); } var RunArguments = MakePathSafeToUseWithCommandLine(Project); if (!String.IsNullOrEmpty(Arguments)) { RunArguments += " " + Arguments; } RunAndLog(Env, Env.MsBuildExe, RunArguments, LogName); }
/// <summary> /// Runs UBT with the specified commandline. Automatically creates a logfile. When /// no LogName is specified, the executable name is used as logfile base name. /// </summary> /// <param name="Env">Environment to use.</param> /// <param name="CommandLine">Commandline to pass on to UBT.</param> /// <param name="LogName">Optional logfile name.</param> public static void RunUBT(CommandEnvironment Env, string UBTExecutable, string CommandLine, string LogName = null) { if (!FileExists(UBTExecutable)) { throw new AutomationException("Unable to find UBT executable: " + UBTExecutable); } if (GlobalCommandLine.Rocket) { CommandLine += " -rocket"; } if (!IsBuildMachine && UnrealBuildTool.BuildHostPlatform.Current.Platform == UnrealBuildTool.UnrealTargetPlatform.Mac) { CommandLine += " -nocreatestub"; } CommandUtils.RunAndLog(Env, UBTExecutable, CommandLine, LogName); }
public static TempStorageManifest SaveSharedTempStorageManifest(CommandEnvironment Env, string StorageBlockName, string GameFolder, List<string> Files) { return SaveTempStorageManifest(SharedTempStorageDirectory(StorageBlockName, GameFolder), SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder), Files); }
public static void StoreToTempStorage(CommandEnvironment Env, string StorageBlockName, List <string> Files, bool bLocalOnly = false, string GameFolder = "", string BaseFolder = "") { if (String.IsNullOrEmpty(BaseFolder)) { BaseFolder = Env.LocalRoot; } BaseFolder = CombinePaths(BaseFolder, "/"); if (!BaseFolder.EndsWith("/") && !BaseFolder.EndsWith("\\")) { throw new AutomationException("base folder {0} should end with a separator", BaseFolder); } var Local = SaveLocalTempStorageManifest(Env, BaseFolder, StorageBlockName, Files); if (!bLocalOnly) { var StartTime = DateTime.UtcNow; var BlockPath = SharedTempStorageDirectory(StorageBlockName, GameFolder); Log("Storing to {0}", BlockPath); if (DirectoryExists_NoExceptions(BlockPath)) { throw new AutomationException("Storage Block Already Exists! {0}", BlockPath); } CreateDirectory(true, BlockPath); if (!DirectoryExists_NoExceptions(BlockPath)) { throw new AutomationException("Storage Block Could Not Be Created! {0}", BlockPath); } var DestFiles = new List <string>(); if (ThreadsToCopyWith() < 2) { foreach (string InFilename in Files) { var Filename = CombinePaths(InFilename); Robust_FileExists_NoExceptions(false, Filename, "Could not add {0} to manifest because it does not exist"); if (!Filename.StartsWith(BaseFolder, StringComparison.InvariantCultureIgnoreCase)) { throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BaseFolder); } var RelativeFile = Filename.Substring(BaseFolder.Length); var DestFile = CombinePaths(BlockPath, RelativeFile); if (FileExists_NoExceptions(true, DestFile)) { throw new AutomationException("Dest file {0} already exists.", DestFile); } CopyFile(Filename, DestFile, true); Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}"); DestFiles.Add(DestFile); } } else { var SrcFiles = new List <string>(); foreach (string InFilename in Files) { var Filename = CombinePaths(InFilename); Robust_FileExists_NoExceptions(false, Filename, "Could not add {0} to manifest because it does not exist"); if (!Filename.StartsWith(BaseFolder, StringComparison.InvariantCultureIgnoreCase)) { throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BaseFolder); } var RelativeFile = Filename.Substring(BaseFolder.Length); var DestFile = CombinePaths(BlockPath, RelativeFile); if (FileExists_NoExceptions(true, DestFile)) { throw new AutomationException("Dest file {0} already exists.", DestFile); } SrcFiles.Add(Filename); DestFiles.Add(DestFile); } ThreadedCopyFiles(SrcFiles.ToArray(), DestFiles.ToArray(), ThreadsToCopyWith()); foreach (string DestFile in DestFiles) { Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}"); } } var Shared = SaveSharedTempStorageManifest(Env, StorageBlockName, GameFolder, DestFiles); if (!Local.Compare(Shared)) { // we will rename this so it can't be used, but leave it around for inspection RenameFile_NoExceptions(SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder), SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder) + ".broken"); throw new AutomationException("Shared and Local manifest mismatch."); } float BuildDuration = (float)((DateTime.UtcNow - StartTime).TotalSeconds); if (BuildDuration > 60.0f && Shared.GetTotalSize() > 0) { var MBSec = (((float)(Shared.GetTotalSize())) / (1024.0f * 1024.0f)) / BuildDuration; Log("Wrote to shared temp storage at {0} MB/s {1}B {2}s", MBSec, Shared.GetTotalSize(), BuildDuration); } } }
/// <summary> /// Runs UAT recursively /// </summary> /// <param name="Env">Environment to use.</param> /// <param name="CommandLine">Commandline to pass on to the executable</param> public static string RunUAT(CommandEnvironment Env, string CommandLine) { // this doesn't do much, but it does need to make sure log folders are reasonable and don't collide string BaseLogSubdir = "Recur"; if (!String.IsNullOrEmpty(CommandLine)) { int Space = CommandLine.IndexOf(" "); if (Space > 0) { BaseLogSubdir = BaseLogSubdir + "_" + CommandLine.Substring(0, Space); } else { BaseLogSubdir = BaseLogSubdir + "_" + CommandLine; } } int Index = 0; BaseLogSubdir = BaseLogSubdir.Trim(); string DirOnlyName = BaseLogSubdir; string LogSubdir = CombinePaths(CmdEnv.LogFolder, DirOnlyName, ""); while (true) { var ExistingFiles = FindFiles(DirOnlyName + "*", false, CmdEnv.LogFolder); if (ExistingFiles.Length == 0) { break; } Index++; if (Index == 1000) { throw new AutomationException("Couldn't seem to create a log subdir {0}", LogSubdir); } DirOnlyName = String.Format("{0}_{1}_", BaseLogSubdir, Index); LogSubdir = CombinePaths(CmdEnv.LogFolder, DirOnlyName, ""); } string LogFile = CombinePaths(CmdEnv.LogFolder, DirOnlyName + ".log"); Log("Recursive UAT Run, in log folder {0}, main log file {1}", LogSubdir, LogFile); CreateDirectory(LogSubdir); string App = CmdEnv.UATExe; Log("Running {0} {1}", App, CommandLine); var OSEnv = new Dictionary<string, string>(); OSEnv.Add(AutomationTool.EnvVarNames.LogFolder, LogSubdir); OSEnv.Add("uebp_UATMutexNoWait", "1"); if (!IsBuildMachine) { OSEnv.Add(AutomationTool.EnvVarNames.LocalRoot, ""); // if we don't clear this out, it will think it is a build machine; it will rederive everything } ProcessResult Result = Run(App, CommandLine, null, ERunOptions.Default, OSEnv); if (Result.Output.Length > 0) { WriteToFile(LogFile, Result.Output); } else { WriteToFile(LogFile, "[None!, no output produced]"); } Log("Flattening log folder {0}", LogSubdir); var Files = FindFiles("*", true, LogSubdir); string MyLogFolder = CombinePaths(CmdEnv.LogFolder, ""); foreach (var ThisFile in Files) { if (!ThisFile.StartsWith(MyLogFolder, StringComparison.InvariantCultureIgnoreCase)) { throw new AutomationException("Can't rebase {0} because it doesn't start with {1}", ThisFile, MyLogFolder); } string NewFilename = ThisFile.Substring(MyLogFolder.Length).Replace("/", "_").Replace("\\", "_"); NewFilename = CombinePaths(CmdEnv.LogFolder, NewFilename); if (FileExists_NoExceptions(NewFilename)) { throw new AutomationException("Destination log file already exists? {0}", NewFilename); } CopyFile(ThisFile, NewFilename); if (!FileExists_NoExceptions(NewFilename)) { throw new AutomationException("Destination log file could not be copied {0}", NewFilename); } DeleteFile_NoExceptions(ThisFile); } DeleteDirectory_NoExceptions(LogSubdir); if (Result != 0) { throw new AutomationException(String.Format("Recursive UAT Command failed (Result:{3}): {0} {1}. See logfile for details: '{2}' ", App, CommandLine, Path.GetFileName(LogFile), Result.ExitCode)); } return LogFile; }
public static TempStorageManifest SaveLocalTempStorageManifest(CommandEnvironment Env, string BaseFolder, string StorageBlockName, List <string> Files) { return(SaveTempStorageManifest(BaseFolder, LocalTempStorageManifestFilename(Env, StorageBlockName), Files)); }
public static string LocalTempStorageManifestDirectory(CommandEnvironment Env) { return(CombinePaths(Env.LocalRoot, "Engine", "Saved", "GUBP")); }
/// <summary> /// Runs external program and writes the output to a logfile. /// </summary> /// <param name="Env">Environment to use.</param> /// <param name="App">Executable to run</param> /// <param name="CommandLine">Commandline to pass on to the executable</param> /// <param name="LogName">Name of the logfile ( if null, executable name is used )</param> /// <returns>Whether the program executed successfully or not.</returns> public static string RunAndLog(CommandEnvironment Env, string App, string CommandLine, out int SuccessCode, string LogName = null) { return(RunAndLog(App, CommandLine, out SuccessCode, GetRunAndLogLogName(Env, App, LogName))); }
public static bool TempStorageExists(CommandEnvironment Env, string StorageBlockName, string GameFolder = "", bool bLocalOnly = false, bool bQuiet = false) { return LocalTempStorageExists(Env, StorageBlockName, bQuiet) || (!bLocalOnly && SharedTempStorageExists(Env, StorageBlockName, GameFolder, bQuiet)); }
/// <summary> /// Constructor. Derives the Perforce environment settings. /// </summary> internal P4Environment(CommandEnvironment CmdEnv) { // Get the Perforce port setting ServerAndPort = CommandUtils.GetEnvVar(EnvVarNames.P4Port); if (String.IsNullOrEmpty(ServerAndPort)) { ServerAndPort = DetectP4Port(); CommandUtils.SetEnvVar(EnvVarNames.P4Port, ServerAndPort); } // Get the Perforce user setting User = CommandUtils.GetEnvVar(EnvVarNames.User); if (String.IsNullOrEmpty(User)) { P4Connection DefaultConnection = new P4Connection(User: null, Client: null, ServerAndPort: ServerAndPort); User = DetectUserName(DefaultConnection); CommandUtils.SetEnvVar(EnvVarNames.User, User); } // Get the Perforce client setting Client = CommandUtils.GetEnvVar(EnvVarNames.Client); if (String.IsNullOrEmpty(Client)) { P4Connection DefaultConnection = new P4Connection(User: User, Client: null, ServerAndPort: ServerAndPort); P4ClientInfo ThisClient = DetectClient(DefaultConnection, User, Environment.MachineName.ToLower(), CmdEnv.UATExe); Log.TraceInformation("Using user {0} clientspec {1} {2}", User, ThisClient.Name, ThisClient.RootPath); string BranchPath; string ClientRootPath; P4Connection ClientConnection = new P4Connection(User: User, Client: ThisClient.Name, ServerAndPort: ServerAndPort); DetectRootPaths(ClientConnection, CmdEnv.LocalRoot, ThisClient, out BranchPath, out ClientRootPath); Client = ThisClient.Name; CommandUtils.SetEnvVar(EnvVarNames.Client, Client); Branch = BranchPath; CommandUtils.SetEnvVar(EnvVarNames.BuildRootP4, Branch); ClientRoot = ClientRootPath; CommandUtils.SetEnvVar(EnvVarNames.ClientRoot, ClientRootPath); } else { Branch = CommandUtils.GetEnvVar(EnvVarNames.BuildRootP4); ClientRoot = CommandUtils.GetEnvVar(EnvVarNames.ClientRoot); if (String.IsNullOrEmpty(Branch) || String.IsNullOrEmpty(ClientRoot)) { throw new AutomationException("{0} and {1} must also be set with {2}", EnvVarNames.ClientRoot, EnvVarNames.BuildRootP4, EnvVarNames.Client); } } // We expect the build root to not end with a path separator if (Branch.EndsWith("/")) { Branch = Branch.TrimEnd('/'); CommandUtils.SetEnvVar(EnvVarNames.BuildRootP4, Branch); } // Set the current changelist string ChangelistString = CommandUtils.GetEnvVar(EnvVarNames.Changelist, null); if (String.IsNullOrEmpty(ChangelistString) && CommandUtils.P4CLRequired) { P4Connection Connection = new P4Connection(User, Client, ServerAndPort); ChangelistString = DetectCurrentCL(Connection, ClientRoot); CommandUtils.SetEnvVar(EnvVarNames.Changelist, ChangelistString); } if (!String.IsNullOrEmpty(ChangelistString)) { Changelist = int.Parse(ChangelistString); } // Set the current code changelist string CodeChangelistString = CommandUtils.GetEnvVar(EnvVarNames.CodeChangelist); if (String.IsNullOrEmpty(CodeChangelistString) && CommandUtils.P4CLRequired) { P4Connection Connection = new P4Connection(User, Client, ServerAndPort); CodeChangelistString = DetectCurrentCodeCL(Connection, ClientRoot); CommandUtils.SetEnvVar(EnvVarNames.CodeChangelist, CodeChangelistString); } if (!String.IsNullOrEmpty(CodeChangelistString)) { CodeChangelist = int.Parse(CodeChangelistString); } // Set the standard environment variables based on the values we've found CommandUtils.SetEnvVar("P4PORT", ServerAndPort); CommandUtils.SetEnvVar("P4USER", User); CommandUtils.SetEnvVar("P4CLIENT", Client); // Write a summary of the settings to the output window if (!CommandUtils.CmdEnv.IsChildInstance) { Log.TraceInformation("Detected Perforce Settings:"); Log.TraceInformation(" Server: {0}", ServerAndPort); Log.TraceInformation(" User: {0}", User); Log.TraceInformation(" Client: {0}", Client); Log.TraceInformation(" Branch: {0}", Branch); if (ChangelistInternal != -1) { Log.TraceInformation(" Last Change: {0}", Changelist); } if (CodeChangelistInternal != -1) { Log.TraceInformation(" Last Code Change: {0}", CodeChangelist); } } // Write all the environment variables to the log Log.TraceLog("Perforce Environment Variables:"); Log.TraceLog(" {0}={1}", EnvVarNames.P4Port, InternalUtils.GetEnvironmentVariable(EnvVarNames.P4Port, "", true)); Log.TraceLog(" {0}={1}", EnvVarNames.User, InternalUtils.GetEnvironmentVariable(EnvVarNames.User, "", true)); Log.TraceLog(" {0}={1}", EnvVarNames.Client, InternalUtils.GetEnvironmentVariable(EnvVarNames.Client, "", true)); Log.TraceLog(" {0}={1}", EnvVarNames.BuildRootP4, InternalUtils.GetEnvironmentVariable(EnvVarNames.BuildRootP4, "", true)); Log.TraceLog(" {0}={1}", EnvVarNames.BuildRootEscaped, InternalUtils.GetEnvironmentVariable(EnvVarNames.BuildRootEscaped, "", true)); Log.TraceLog(" {0}={1}", EnvVarNames.ClientRoot, InternalUtils.GetEnvironmentVariable(EnvVarNames.ClientRoot, "", true)); Log.TraceLog(" {0}={1}", EnvVarNames.Changelist, InternalUtils.GetEnvironmentVariable(EnvVarNames.Changelist, "", true)); Log.TraceLog(" {0}={1}", EnvVarNames.CodeChangelist, InternalUtils.GetEnvironmentVariable(EnvVarNames.CodeChangelist, "", true)); Log.TraceLog(" {0}={1}", "P4PORT", InternalUtils.GetEnvironmentVariable("P4PORT", "", true)); Log.TraceLog(" {0}={1}", "P4USER", InternalUtils.GetEnvironmentVariable("P4USER", "", true)); Log.TraceLog(" {0}={1}", "P4CLIENT", InternalUtils.GetEnvironmentVariable("P4CLIENT", "", true)); }
/// <summary> /// Runs external program and writes the output to a logfile. /// </summary> /// <param name="Env">Environment to use.</param> /// <param name="App">Executable to run</param> /// <param name="CommandLine">Commandline to pass on to the executable</param> /// <param name="LogName">Name of the logfile ( if null, executable name is used )</param> /// <param name="Input">Optional Input for the program (will be provided as stdin)</param> /// <param name="Options">Defines the options how to run. See ERunOptions.</param> public static void RunAndLog(CommandEnvironment Env, string App, string CommandLine, string LogName = null, int MaxSuccessCode = 0, string Input = null, ERunOptions Options = ERunOptions.Default) { RunAndLog(App, CommandLine, GetRunAndLogLogName(Env, App, LogName), MaxSuccessCode, Input, Options); }
/// <summary> /// Runs external program and writes the output to a logfile. /// </summary> /// <param name="Env">Environment to use.</param> /// <param name="App">Executable to run</param> /// <param name="CommandLine">Commandline to pass on to the executable</param> /// <param name="LogName">Name of the logfile ( if null, executable name is used )</param> /// <returns>Whether the program executed successfully or not.</returns> public static string RunAndLog(CommandEnvironment Env, string App, string CommandLine, out int SuccessCode, string LogName = null, Dictionary <string, string> EnvVars = null) { return(RunAndLog(App, CommandLine, out SuccessCode, GetRunAndLogLogName(Env, App, LogName), EnvVars)); }
public static List <string> FindTempStorageManifests(CommandEnvironment Env, string StorageBlockName, bool LocalOnly = false, bool SharedOnly = false, string GameFolder = "") { var Files = new List <string>(); var LocalFiles = LocalTempStorageManifestFilename(Env, StorageBlockName); var LocalParent = Path.GetDirectoryName(LocalFiles); var WildCard = Path.GetFileName(LocalFiles); int IndexOfStar = WildCard.IndexOf("*"); if (IndexOfStar < 0 || WildCard.LastIndexOf("*") != IndexOfStar) { throw new AutomationException("Wildcard {0} either has no star or it has more than one.", WildCard); } string PreStarWildcard = WildCard.Substring(0, IndexOfStar); string PostStarWildcard = Path.GetFileNameWithoutExtension(WildCard.Substring(IndexOfStar + 1)); if (!SharedOnly && DirectoryExists_NoExceptions(LocalParent)) { foreach (var ThisFile in CommandUtils.FindFiles_NoExceptions(WildCard, true, LocalParent)) { Log(" Found local file {0}", ThisFile); int IndexOfWildcard = ThisFile.IndexOf(PreStarWildcard); if (IndexOfWildcard < 0) { throw new AutomationException("File {0} didn't contain {1}.", ThisFile, PreStarWildcard); } int LastIndexOfWildcardTail = ThisFile.LastIndexOf(PostStarWildcard); if (LastIndexOfWildcardTail < 0 || LastIndexOfWildcardTail < IndexOfWildcard + PreStarWildcard.Length) { throw new AutomationException("File {0} didn't contain {1} or it was before the prefix", ThisFile, PostStarWildcard); } string StarReplacement = ThisFile.Substring(IndexOfWildcard + PreStarWildcard.Length, LastIndexOfWildcardTail - IndexOfWildcard - PreStarWildcard.Length); if (StarReplacement.Length < 1) { throw new AutomationException("Dir {0} didn't have any string to fit the star in the wildcard {1}", ThisFile, WildCard); } if (!Files.Contains(StarReplacement)) { Files.Add(StarReplacement); } } } if (!LocalOnly) { var SharedFiles = SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder); var SharedParent = Path.GetDirectoryName(Path.GetDirectoryName(SharedFiles)); if (DirectoryExists_NoExceptions(SharedParent)) { string[] Dirs = null; try { Dirs = Directory.GetDirectories(SharedParent, Path.GetFileNameWithoutExtension(SharedFiles), SearchOption.TopDirectoryOnly); } catch (Exception Ex) { Log("Unable to Find Directories in {0} with wildcard {1}", SharedParent, Path.GetFileNameWithoutExtension(SharedFiles)); Log(" Exception was {0}", LogUtils.FormatException(Ex)); } if (Dirs != null) { foreach (var ThisSubDir in Dirs) { int IndexOfWildcard = ThisSubDir.IndexOf(PreStarWildcard); if (IndexOfWildcard < 0) { throw new AutomationException("Dir {0} didn't contain {1}.", ThisSubDir, PreStarWildcard); } int LastIndexOfWildcardTail = ThisSubDir.LastIndexOf(PostStarWildcard); if (LastIndexOfWildcardTail < 0 || LastIndexOfWildcardTail < IndexOfWildcard + PreStarWildcard.Length) { throw new AutomationException("Dir {0} didn't contain {1} or it was before the prefix", ThisSubDir, PostStarWildcard); } string StarReplacement = ThisSubDir.Substring(IndexOfWildcard + PreStarWildcard.Length, LastIndexOfWildcardTail - IndexOfWildcard - PreStarWildcard.Length); if (StarReplacement.Length < 1) { throw new AutomationException("Dir {0} didn't have any string to fit the star in the wildcard {1}", ThisSubDir, WildCard); } // these are a bunch of false positives if (StarReplacement.Contains("-")) { continue; } if (!Files.Contains(StarReplacement)) { Files.Add(StarReplacement); } } } } } var OutFiles = new List <string>(); foreach (var StarReplacement in Files) { var NewBlock = StorageBlockName.Replace("*", StarReplacement); if (TempStorageExists(Env, NewBlock, GameFolder, LocalOnly, true)) { OutFiles.Add(StarReplacement); } } return(OutFiles); }
public static List <string> RetrieveFromTempStorage(CommandEnvironment Env, string StorageBlockName, out bool WasLocal, string GameFolder = "", string BaseFolder = "") { if (String.IsNullOrEmpty(BaseFolder)) { BaseFolder = Env.LocalRoot; } BaseFolder = CombinePaths(BaseFolder, "/"); if (!BaseFolder.EndsWith("/") && !BaseFolder.EndsWith("\\")) { throw new AutomationException("base folder {0} should end with a separator", BaseFolder); } var Files = new List <string>(); var LocalManifest = LocalTempStorageManifestFilename(Env, StorageBlockName); if (FileExists_NoExceptions(LocalManifest)) { Log("Found local manifest {0}", LocalManifest); var Local = new TempStorageManifest(); Local.Load(LocalManifest); Files = Local.GetFiles(BaseFolder); var LocalTest = new TempStorageManifest(); LocalTest.Create(Files, BaseFolder); if (!Local.Compare(LocalTest)) { throw new AutomationException("Local files in manifest {0} were tampered with.", LocalManifest); } WasLocal = true; return(Files); } WasLocal = false; var StartTime = DateTime.UtcNow; var BlockPath = CombinePaths(SharedTempStorageDirectory(StorageBlockName, GameFolder), "/"); if (!BlockPath.EndsWith("/") && !BlockPath.EndsWith("\\")) { throw new AutomationException("base folder {0} should end with a separator", BlockPath); } Log("Attempting to retrieve from {0}", BlockPath); if (!DirectoryExists_NoExceptions(BlockPath)) { throw new AutomationException("Storage Block Does Not Exists! {0}", BlockPath); } var SharedManifest = SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder); Robust_FileExists_NoExceptions(SharedManifest, "Storage Block Manifest Does Not Exists! {0}"); var Shared = new TempStorageManifest(); Shared.Load(SharedManifest); var SharedFiles = Shared.GetFiles(BlockPath); var DestFiles = new List <string>(); if (ThreadsToCopyWith() < 2) { foreach (string InFilename in SharedFiles) { var Filename = CombinePaths(InFilename); Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist"); if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase)) { throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath); } var RelativeFile = Filename.Substring(BlockPath.Length); var DestFile = CombinePaths(BaseFolder, RelativeFile); if (FileExists_NoExceptions(true, DestFile)) { Log("Dest file {0} already exists, deleting and overwriting", DestFile); DeleteFile(DestFile); } CopyFile(Filename, DestFile, true); Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}"); if (UnrealBuildTool.Utils.IsRunningOnMono) { FixUnixFilePermissions(DestFile); } FileInfo Info = new FileInfo(DestFile); DestFiles.Add(Info.FullName); } } else { var SrcFiles = new List <string>(); foreach (string InFilename in SharedFiles) { var Filename = CombinePaths(InFilename); //Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist"); if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase)) { throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath); } var RelativeFile = Filename.Substring(BlockPath.Length); var DestFile = CombinePaths(BaseFolder, RelativeFile); if (FileExists_NoExceptions(true, DestFile)) { Log("Dest file {0} already exists, deleting and overwriting", DestFile); DeleteFile(DestFile); } SrcFiles.Add(Filename); DestFiles.Add(DestFile); } ThreadedCopyFiles(SrcFiles.ToArray(), DestFiles.ToArray(), ThreadsToCopyWith()); var NewDestFiles = new List <string>(); foreach (string DestFile in DestFiles) { Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}"); if (UnrealBuildTool.Utils.IsRunningOnMono) { FixUnixFilePermissions(DestFile); } FileInfo Info = new FileInfo(DestFile); NewDestFiles.Add(Info.FullName); } DestFiles = NewDestFiles; } var NewLocal = SaveLocalTempStorageManifest(Env, BaseFolder, StorageBlockName, DestFiles); if (!NewLocal.Compare(Shared)) { // we will rename this so it can't be used, but leave it around for inspection RenameFile_NoExceptions(LocalManifest, LocalManifest + ".broken"); throw new AutomationException("Shared and Local manifest mismatch."); } float BuildDuration = (float)((DateTime.UtcNow - StartTime).TotalSeconds); if (BuildDuration > 60.0f && Shared.GetTotalSize() > 0) { var MBSec = (((float)(Shared.GetTotalSize())) / (1024.0f * 1024.0f)) / BuildDuration; Log("Read from shared temp storage at {0} MB/s {1}B {2}s", MBSec, Shared.GetTotalSize(), BuildDuration); } return(DestFiles); }
public static void DeleteSharedTempStorageManifests(CommandEnvironment Env, string StorageBlockName, string GameFolder = "") { DeleteDirectory(true, SharedTempStorageDirectory(StorageBlockName, GameFolder)); }
/// <summary> /// Runs UAT recursively /// </summary> /// <param name="Env">Environment to use.</param> /// <param name="CommandLine">Commandline to pass on to the executable</param> public static string RunUAT(CommandEnvironment Env, string CommandLine) { // this doesn't do much, but it does need to make sure log folders are reasonable and don't collide string BaseLogSubdir = "Recur"; if (!String.IsNullOrEmpty(CommandLine)) { int Space = CommandLine.IndexOf(" "); if (Space > 0) { BaseLogSubdir = BaseLogSubdir + "_" + CommandLine.Substring(0, Space); } else { BaseLogSubdir = BaseLogSubdir + "_" + CommandLine; } } int Index = 0; BaseLogSubdir = BaseLogSubdir.Trim(); string DirOnlyName = BaseLogSubdir; string LogSubdir = CombinePaths(CmdEnv.LogFolder, DirOnlyName, ""); while (true) { var ExistingFiles = FindFiles(DirOnlyName + "*", false, CmdEnv.LogFolder); if (ExistingFiles.Length == 0) { break; } Index++; if (Index == 1000) { throw new AutomationException("Couldn't seem to create a log subdir {0}", LogSubdir); } DirOnlyName = String.Format("{0}_{1}_", BaseLogSubdir, Index); LogSubdir = CombinePaths(CmdEnv.LogFolder, DirOnlyName, ""); } string LogFile = CombinePaths(CmdEnv.LogFolder, DirOnlyName + ".log"); Log("Recursive UAT Run, in log folder {0}, main log file {1}", LogSubdir, LogFile); CreateDirectory(LogSubdir); CommandLine = CommandLine + " -NoCompile"; string App = CmdEnv.UATExe; Log("Running {0} {1}", App, CommandLine); var OSEnv = new Dictionary <string, string>(); OSEnv.Add(AutomationTool.EnvVarNames.LogFolder, LogSubdir); OSEnv.Add("uebp_UATMutexNoWait", "1"); if (!IsBuildMachine) { OSEnv.Add(AutomationTool.EnvVarNames.LocalRoot, ""); // if we don't clear this out, it will think it is a build machine; it will rederive everything } ProcessResult Result = Run(App, CommandLine, null, ERunOptions.Default, OSEnv); if (Result.Output.Length > 0) { WriteToFile(LogFile, Result.Output); } else { WriteToFile(LogFile, "[None!, no output produced]"); } Log("Flattening log folder {0}", LogSubdir); var Files = FindFiles("*", true, LogSubdir); string MyLogFolder = CombinePaths(CmdEnv.LogFolder, ""); foreach (var ThisFile in Files) { if (!ThisFile.StartsWith(MyLogFolder, StringComparison.InvariantCultureIgnoreCase)) { throw new AutomationException("Can't rebase {0} because it doesn't start with {1}", ThisFile, MyLogFolder); } string NewFilename = ThisFile.Substring(MyLogFolder.Length).Replace("/", "_").Replace("\\", "_"); NewFilename = CombinePaths(CmdEnv.LogFolder, NewFilename); if (FileExists_NoExceptions(NewFilename)) { throw new AutomationException("Destination log file already exists? {0}", NewFilename); } CopyFile(ThisFile, NewFilename); if (!FileExists_NoExceptions(NewFilename)) { throw new AutomationException("Destination log file could not be copied {0}", NewFilename); } DeleteFile_NoExceptions(ThisFile); } DeleteDirectory_NoExceptions(LogSubdir); if (Result != 0) { throw new AutomationException(String.Format("Recursive UAT Command failed (Result:{3}): {0} {1}. See logfile for details: '{2}' ", App, CommandLine, Path.GetFileName(LogFile), Result.ExitCode)); } return(LogFile); }
public static void DeleteLocalTempStorage(CommandEnvironment Env, string StorageBlockName, bool bQuiet = false) { var LocalManifest = LocalTempStorageManifestFilename(Env, StorageBlockName); DeleteFile(bQuiet, LocalManifest); }
public static string SharedTempStorageManifestFilename(CommandEnvironment Env, string StorageBlockName, string GameFolder) { return(CombinePaths(SharedTempStorageDirectory(StorageBlockName, GameFolder), StorageBlockName + ".TempManifest")); }
public static void RetrieveFromPermanentStorage(CommandEnvironment Env, string NetworkRoot, string Build, string Platform) { var LocalManifest = LocalTempStorageManifestFilename(Env, Build); if (!FileExists_NoExceptions(LocalManifest)) { string BaseFolder = CombinePaths(Env.LocalRoot, "Rocket", "TempInst", Platform); string Root = RootSharedTempStorageDirectory(); string SrcFolder = CombinePaths(Root, NetworkRoot, Build, Platform); var SourceFiles = FindFiles_NoExceptions("*", true, SrcFolder); foreach (var File in SourceFiles) { var DestFile = File.Replace(SrcFolder, ""); DestFile = CombinePaths(BaseFolder, DestFile); CopyFile_NoExceptions(File, DestFile); } if(!DirectoryExists(Path.GetDirectoryName(LocalManifest))) { CreateDirectory(Path.GetDirectoryName(LocalManifest)); } XmlDocument Local = new XmlDocument(); XmlElement RootElement = Local.CreateElement("permstorage"); Local.AppendChild(RootElement); Local.Save(LocalManifest); } }
internal P4Environment(P4Connection Connection, CommandEnvironment CmdEnv) { InitEnvironment(Connection, CmdEnv); }
public static string LocalTempStorageManifestFilename(CommandEnvironment Env, string StorageBlockName) { return(CombinePaths(LocalTempStorageManifestDirectory(Env), StorageBlockName + ".TempManifest")); }
public static void DeleteLocalTempStorageManifests(CommandEnvironment Env) { DeleteDirectory(true, LocalTempStorageManifestDirectory(Env)); }
/// <summary> /// Runs external program and writes the output to a logfile. /// </summary> /// <param name="Env">Environment to use.</param> /// <param name="App">Executable to run</param> /// <param name="CommandLine">Commandline to pass on to the executable</param> /// <param name="LogName">Name of the logfile ( if null, executable name is used )</param> /// <returns>Whether the program executed successfully or not.</returns> public static string RunAndLog(CommandEnvironment Env, string App, string CommandLine, out int SuccessCode, string LogName = null, Dictionary<string, string> EnvVars = null) { return RunAndLog(App, CommandLine, out SuccessCode, GetRunAndLogLogName(Env, App, LogName), EnvVars); }
public static TempStorageManifest SaveLocalTempStorageManifest(CommandEnvironment Env, string BaseFolder, string StorageBlockName, List<string> Files) { return SaveTempStorageManifest(BaseFolder, LocalTempStorageManifestFilename(Env, StorageBlockName), Files); }
/// <summary> /// Initializes the environment. Tries to autodetect all source control settings. /// </summary> /// <param name="CompilationEnv">Compilation environment</param> protected override void InitEnvironment(P4Connection Connection, CommandEnvironment CmdEnv) { var HostName = Environment.MachineName.ToLower(); var P4PortEnv = DetectP4Port(); var UserName = CommandUtils.GetEnvVar(EnvVarNames.User); if (String.IsNullOrEmpty(UserName)) { UserName = DetectUserName(Connection); } var CommandLineClient = CommandUtils.GetEnvVar(EnvVarNames.Client); P4ClientInfo ThisClient = null; if (String.IsNullOrEmpty(CommandLineClient) == false) { ThisClient = Connection.GetClientInfo(CommandLineClient); if (ThisClient == null) { throw new AutomationException("Unable to find client {0}", CommandLineClient); } if (String.Compare(ThisClient.Owner, UserName, true) != 0) { throw new AutomationException("Client specified with {0}={1} has a different owner then the detected user name (has: {2}, expected: {3})", EnvVarNames.Client, CommandLineClient, ThisClient.Owner, UserName); } } else { ThisClient = DetectClient(Connection, UserName, HostName, CmdEnv.UATExe); } Log.TraceInformation("Using user {0} clientspec {1} {2}", UserName, ThisClient.Name, ThisClient.RootPath); Environment.SetEnvironmentVariable("P4CLIENT", ThisClient.Name); string BuildRootPath; string ClientRootPath; DetectRootPaths(Connection, CmdEnv.LocalRoot, ThisClient, out BuildRootPath, out ClientRootPath); CommandUtils.ConditionallySetEnvVar(EnvVarNames.P4Port, P4PortEnv); CommandUtils.ConditionallySetEnvVar(EnvVarNames.User, UserName); CommandUtils.ConditionallySetEnvVar(EnvVarNames.Client, ThisClient.Name); CommandUtils.ConditionallySetEnvVar(EnvVarNames.BuildRootP4, BuildRootPath); CommandUtils.ConditionallySetEnvVar(EnvVarNames.ClientRoot, ClientRootPath); var CLString = CommandUtils.GetEnvVar(EnvVarNames.Changelist, null); if (String.IsNullOrEmpty(CLString) && CommandUtils.P4CLRequired) { CLString = DetectCurrentCL(Connection, ClientRootPath); } if (!String.IsNullOrEmpty(CLString)) { CommandUtils.ConditionallySetEnvVar(EnvVarNames.Changelist, CLString); } var CodeCLString = CommandUtils.GetEnvVar(EnvVarNames.CodeChangelist, null); if (String.IsNullOrEmpty(CodeCLString) && CommandUtils.P4CLRequired) { CodeCLString = DetectCurrentCodeCL(Connection, ClientRootPath); } if (!String.IsNullOrEmpty(CodeCLString)) { CommandUtils.ConditionallySetEnvVar(EnvVarNames.CodeChangelist, CodeCLString); } CommandUtils.ConditionallySetEnvVar(EnvVarNames.LabelToSync, ""); CommandUtils.ConditionallySetEnvVar("P4USER", UserName); CommandUtils.ConditionallySetEnvVar("P4CLIENT", ThisClient.Name); var P4Password = Environment.GetEnvironmentVariable(EnvVarNames.P4Password); if (!String.IsNullOrEmpty(P4Password)) { CommandUtils.ConditionallySetEnvVar("P4PASSWD", P4Password); } SetBuildRootEscaped(); base.InitEnvironment(Connection, CmdEnv); }
public static bool TempStorageExists(CommandEnvironment Env, string StorageBlockName, string GameFolder = "", bool bLocalOnly = false, bool bQuiet = false) { return(LocalTempStorageExists(Env, StorageBlockName, bQuiet) || (!bLocalOnly && SharedTempStorageExists(Env, StorageBlockName, GameFolder, bQuiet))); }
public static string LocalTempStorageManifestDirectory(CommandEnvironment Env) { return CombinePaths(Env.LocalRoot, "Engine", "Saved", "GUBP"); }
public static TempStorageManifest SaveSharedTempStorageManifest(CommandEnvironment Env, string StorageBlockName, string GameFolder, List <string> Files) { return(SaveTempStorageManifest(SharedTempStorageDirectory(StorageBlockName, GameFolder), SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder), Files)); }
internal LocalP4Environment(P4Connection Connection, CommandEnvironment CmdEnv) : base(Connection, CmdEnv) { }
/// <summary> /// Builds a target using UBT. Automatically creates a logfile. When /// no LogName is specified, the executable name is used as logfile base name. /// </summary> /// <param name="Env">BuildEnvironment to use.</param> /// <param name="Project">Unreal project to build (optional)</param> /// <param name="Target">Target to build.</param> /// <param name="Platform">Platform to build for.</param> /// <param name="Config">Configuration to build.</param> /// <param name="AdditionalArgs">Additional arguments to pass on to UBT.</param> /// <param name="LogName">Optional logifle name.</param> public static void RunUBT(CommandEnvironment Env, string UBTExecutable, string Project, string Target, string Platform, string Config, string AdditionalArgs = "", string LogName = null, Dictionary <string, string> EnvVars = null) { RunUBT(Env, UBTExecutable, UBTCommandline(Project, Target, Platform, Config, AdditionalArgs), LogName, EnvVars); }
public static List<string> FindTempStorageManifests(CommandEnvironment Env, string StorageBlockName, bool LocalOnly = false, bool SharedOnly = false, string GameFolder = "") { var Files = new List<string>(); var LocalFiles = LocalTempStorageManifestFilename(Env, StorageBlockName); var LocalParent = Path.GetDirectoryName(LocalFiles); var WildCard = Path.GetFileName(LocalFiles); int IndexOfStar = WildCard.IndexOf("*"); if (IndexOfStar < 0 || WildCard.LastIndexOf("*") != IndexOfStar) { throw new AutomationException("Wildcard {0} either has no star or it has more than one.", WildCard); } string PreStarWildcard = WildCard.Substring(0, IndexOfStar); string PostStarWildcard = Path.GetFileNameWithoutExtension(WildCard.Substring(IndexOfStar + 1)); if (!SharedOnly && DirectoryExists_NoExceptions(LocalParent)) { foreach (var ThisFile in CommandUtils.FindFiles_NoExceptions(WildCard, true, LocalParent)) { Log(" Found local file {0}", ThisFile); int IndexOfWildcard = ThisFile.IndexOf(PreStarWildcard); if (IndexOfWildcard < 0) { throw new AutomationException("File {0} didn't contain {1}.", ThisFile, PreStarWildcard); } int LastIndexOfWildcardTail = ThisFile.LastIndexOf(PostStarWildcard); if (LastIndexOfWildcardTail < 0 || LastIndexOfWildcardTail < IndexOfWildcard + PreStarWildcard.Length) { throw new AutomationException("File {0} didn't contain {1} or it was before the prefix", ThisFile, PostStarWildcard); } string StarReplacement = ThisFile.Substring(IndexOfWildcard + PreStarWildcard.Length, LastIndexOfWildcardTail - IndexOfWildcard - PreStarWildcard.Length); if (StarReplacement.Length < 1) { throw new AutomationException("Dir {0} didn't have any string to fit the star in the wildcard {1}", ThisFile, WildCard); } if (!Files.Contains(StarReplacement)) { Files.Add(StarReplacement); } } } if (!LocalOnly) { var SharedFiles = SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder); var SharedParent = Path.GetDirectoryName(Path.GetDirectoryName(SharedFiles)); if (DirectoryExists_NoExceptions(SharedParent)) { string[] Dirs = null; try { Dirs = Directory.GetDirectories(SharedParent, Path.GetFileNameWithoutExtension(SharedFiles), SearchOption.TopDirectoryOnly); } catch (Exception Ex) { Log("Unable to Find Directories in {0} with wildcard {1}", SharedParent, Path.GetFileNameWithoutExtension(SharedFiles)); Log(" Exception was {0}", LogUtils.FormatException(Ex)); } if (Dirs != null) { foreach (var ThisSubDir in Dirs) { int IndexOfWildcard = ThisSubDir.IndexOf(PreStarWildcard); if (IndexOfWildcard < 0) { throw new AutomationException("Dir {0} didn't contain {1}.", ThisSubDir, PreStarWildcard); } int LastIndexOfWildcardTail = ThisSubDir.LastIndexOf(PostStarWildcard); if (LastIndexOfWildcardTail < 0 || LastIndexOfWildcardTail < IndexOfWildcard + PreStarWildcard.Length) { throw new AutomationException("Dir {0} didn't contain {1} or it was before the prefix", ThisSubDir, PostStarWildcard); } string StarReplacement = ThisSubDir.Substring(IndexOfWildcard + PreStarWildcard.Length, LastIndexOfWildcardTail - IndexOfWildcard - PreStarWildcard.Length); if (StarReplacement.Length < 1) { throw new AutomationException("Dir {0} didn't have any string to fit the star in the wildcard {1}", ThisSubDir, WildCard); } // these are a bunch of false positives if (StarReplacement.Contains("-")) { continue; } if (!Files.Contains(StarReplacement)) { Files.Add(StarReplacement); } } } } } var OutFiles = new List<string>(); foreach (var StarReplacement in Files) { var NewBlock = StorageBlockName.Replace("*", StarReplacement); if (TempStorageExists(Env, NewBlock, GameFolder, LocalOnly, true)) { OutFiles.Add(StarReplacement); } } return OutFiles; }
/// <summary> /// Runs UAT recursively /// </summary> /// <param name="Env">Environment to use.</param> /// <param name="CommandLine">Commandline to pass on to the executable</param> /// <param name="Identifier">Log prefix for output</param> public static string RunUAT(CommandEnvironment Env, string CommandLine, string Identifier = null) { // We want to redirect the output from recursive UAT calls into our normal log folder, but prefix everything with a unique identifier. To do so, we set the EnvVarNames.LogFolder environment // variable to a subfolder of it, then copy its contents into the main folder with a prefix after it's finished. Start by finding a base name we can use to identify the output of this run. string BaseLogSubdir = "Recur"; if (!String.IsNullOrEmpty(CommandLine)) { int Space = CommandLine.IndexOf(" "); if (Space > 0) { BaseLogSubdir = BaseLogSubdir + "_" + CommandLine.Substring(0, Space); } else if (CommandLine.Contains("-profile")) { string PathToProfile = CommandLine.Substring(CommandLine.IndexOf('=') + 1); BaseLogSubdir = BaseLogSubdir + "_" + (Path.GetFileNameWithoutExtension(PathToProfile)); } else { BaseLogSubdir = BaseLogSubdir + "_" + CommandLine; } } BaseLogSubdir = BaseLogSubdir.Trim(); // Check if there are already log files which start with this prefix, and try to uniquify it if until there aren't. int Index = 0; string DirOnlyName = BaseLogSubdir; string LogSubdir = CombinePaths(CmdEnv.LogFolder, DirOnlyName, ""); while (true) { var ExistingFiles = FindFiles(DirOnlyName + "*", false, CmdEnv.LogFolder); if (ExistingFiles.Length == 0) { break; } Index++; if (Index == 1000) { throw new AutomationException("Couldn't seem to create a log subdir {0}", LogSubdir); } DirOnlyName = String.Format("{0}_{1}_", BaseLogSubdir, Index); LogSubdir = CombinePaths(CmdEnv.LogFolder, DirOnlyName, ""); } // Get the stdout log file for this run, and create the subdirectory for all the other log output string LogFile = CombinePaths(CmdEnv.LogFolder, DirOnlyName + ".log"); LogVerbose("Recursive UAT Run, in log folder {0}, main log file {1}", LogSubdir, LogFile); CreateDirectory(LogSubdir); // Run UAT with the log folder redirected through the environment string App = CmdEnv.UATExe; Log("Running {0} {1}", App, CommandLine); var OSEnv = new Dictionary <string, string>(); OSEnv.Add(EnvVarNames.LogFolder, LogSubdir); OSEnv.Add(EnvVarNames.DisableStartupMutex, "1"); OSEnv.Add(EnvVarNames.IsChildInstance, "1"); if (!IsBuildMachine) { OSEnv.Add(AutomationTool.EnvVarNames.LocalRoot, ""); // if we don't clear this out, it will think it is a build machine; it will rederive everything } IProcessResult Result = Run(App, CommandLine, null, ERunOptions.Default, OSEnv, Identifier: Identifier); if (Result.Output.Length > 0) { WriteToFile(LogFile, Result.Output); } else { WriteToFile(LogFile, "[None!, no output produced]"); } // Copy everything into the main log folder, using the prefix we decided on earlier. LogVerbose("Flattening log folder {0}", LogSubdir); var Files = FindFiles("*", true, LogSubdir); string MyLogFolder = CombinePaths(CmdEnv.LogFolder, ""); foreach (var ThisFile in Files) { if (!ThisFile.StartsWith(MyLogFolder, StringComparison.InvariantCultureIgnoreCase)) { throw new AutomationException("Can't rebase {0} because it doesn't start with {1}", ThisFile, MyLogFolder); } string NewFilename = ThisFile.Substring(MyLogFolder.Length).Replace("/", "_").Replace("\\", "_"); NewFilename = CombinePaths(CmdEnv.LogFolder, NewFilename); if (FileExists_NoExceptions(NewFilename)) { throw new AutomationException("Destination log file already exists? {0}", NewFilename); } CopyFile(ThisFile, NewFilename); if (!FileExists_NoExceptions(NewFilename)) { throw new AutomationException("Destination log file could not be copied {0}", NewFilename); } DeleteFile_NoExceptions(ThisFile); } DeleteDirectory_NoExceptions(LogSubdir); if (Result.ExitCode != 0) { throw new CommandFailedException(String.Format("Recursive UAT Command failed (Result:{3}): {0} {1}. See logfile for details: '{2}' ", App, CommandLine, Path.GetFileName(LogFile), Result.ExitCode)); } return(LogFile); }
public static string LocalTempStorageManifestFilename(CommandEnvironment Env, string StorageBlockName) { return CombinePaths(LocalTempStorageManifestDirectory(Env), StorageBlockName + ".TempManifest"); }
protected virtual void InitEnvironment(P4Connection Connection, CommandEnvironment CmdEnv) { // // P4 Environment // P4Port = CommandUtils.GetEnvVar(EnvVarNames.P4Port); ClientRoot = CommandUtils.GetEnvVar(EnvVarNames.ClientRoot); User = CommandUtils.GetEnvVar(EnvVarNames.User); ChangelistStringInternal = CommandUtils.GetEnvVar(EnvVarNames.Changelist, null); Client = CommandUtils.GetEnvVar(EnvVarNames.Client); BuildRootP4 = CommandUtils.GetEnvVar(EnvVarNames.BuildRootP4); if (BuildRootP4.EndsWith("/", StringComparison.InvariantCultureIgnoreCase) || BuildRootP4.EndsWith("\\", StringComparison.InvariantCultureIgnoreCase)) { // We expect the build root to not end with a path separator BuildRootP4 = BuildRootP4.Substring(0, BuildRootP4.Length - 1); CommandUtils.SetEnvVar(EnvVarNames.BuildRootP4, BuildRootP4); } BuildRootEscaped = CommandUtils.GetEnvVar(EnvVarNames.BuildRootEscaped); LabelToSync = CommandUtils.GetEnvVar(EnvVarNames.LabelToSync); if (((CommandUtils.P4Enabled || CommandUtils.IsBuildMachine) && (ClientRoot == String.Empty || User == String.Empty || (String.IsNullOrEmpty(ChangelistStringInternal) && CommandUtils.IsBuildMachine) || Client == String.Empty || BuildRootP4 == String.Empty))) { Log.TraceInformation("P4Enabled={0}", CommandUtils.P4Enabled); Log.TraceInformation("ClientRoot={0}", ClientRoot); Log.TraceInformation("User={0}", User); Log.TraceInformation("ChangelistString={0}", ChangelistStringInternal); Log.TraceInformation("Client={0}", Client); Log.TraceInformation("BuildRootP4={0}", BuildRootP4); throw new AutomationException("BUILD FAILED Perforce Environment is not set up correctly. Please check your environment variables."); } LabelPrefix = BuildRootP4 + "/"; if (CommandUtils.P4Enabled) { if (CommandUtils.IsBuildMachine || ChangelistStringInternal != null) { // We may not always need the changelist number if we're not a build machine. // In local runs, changelist initialization can be really slow! VerifyChangelistStringAndSetChangelistNumber(); } // Setup branch name string DepotSuffix = "//depot/"; if (BuildRootP4.StartsWith(DepotSuffix)) { BranchName = BuildRootP4.Substring(DepotSuffix.Length); } else { throw new AutomationException("Needs update to work with a stream"); } if (String.IsNullOrWhiteSpace(BranchName)) { throw new AutomationException("BUILD FAILED no branch name."); } } LogSettings(); }
public static string SharedTempStorageManifestFilename(CommandEnvironment Env, string StorageBlockName, string GameFolder) { return CombinePaths(SharedTempStorageDirectory(StorageBlockName, GameFolder), StorageBlockName + ".TempManifest"); }
/// <summary> /// Initializes the environment. Tries to autodetect all source control settings. /// </summary> /// <param name="CompilationEnv">Compilation environment</param> protected override void InitEnvironment(P4Connection Connection, CommandEnvironment CmdEnv) { var HostName = Environment.MachineName.ToLower(); var P4PortEnv = DetectP4Port(); var UserName = CommandUtils.GetEnvVar(EnvVarNames.User); if (String.IsNullOrEmpty(UserName)) { UserName = DetectUserName(Connection); } var CommandLineClient = CommandUtils.GetEnvVar(EnvVarNames.Client); P4ClientInfo ThisClient = null; if (String.IsNullOrEmpty(CommandLineClient) == false) { ThisClient = Connection.GetClientInfo(CommandLineClient); if (ThisClient == null) { throw new AutomationException("Unable to find client {0}", CommandLineClient); } if (String.Compare(ThisClient.Owner, UserName, true) != 0) { throw new AutomationException("Client specified with {0}={1} has a different owner then the detected user name (has: {2}, expected: {3})", EnvVarNames.Client, CommandLineClient, ThisClient.Owner, UserName); } } else { ThisClient = DetectClient(Connection, UserName, HostName, CmdEnv.UATExe); } Log.TraceInformation("Using user {0} clientspec {1} {2}", UserName, ThisClient.Name, ThisClient.RootPath); Environment.SetEnvironmentVariable("P4CLIENT", ThisClient.Name); string BuildRootPath; string ClientRootPath; DetectRootPaths(Connection, CmdEnv.LocalRoot, ThisClient, out BuildRootPath, out ClientRootPath); CommandUtils.ConditionallySetEnvVar(EnvVarNames.P4Port, P4PortEnv); CommandUtils.ConditionallySetEnvVar(EnvVarNames.User, UserName); CommandUtils.ConditionallySetEnvVar(EnvVarNames.Client, ThisClient.Name); CommandUtils.ConditionallySetEnvVar(EnvVarNames.BuildRootP4, BuildRootPath); CommandUtils.ConditionallySetEnvVar(EnvVarNames.ClientRoot, ClientRootPath); var CLString = CommandUtils.GetEnvVar(EnvVarNames.Changelist, null); if (String.IsNullOrEmpty(CLString) && CommandUtils.P4CLRequired) { CLString = DetectCurrentCL(Connection, ClientRootPath); } if (!String.IsNullOrEmpty(CLString)) { CommandUtils.ConditionallySetEnvVar(EnvVarNames.Changelist, CLString); } var CodeCLString = CommandUtils.GetEnvVar(EnvVarNames.CodeChangelist, null); if (String.IsNullOrEmpty(CodeCLString) && CommandUtils.P4CLRequired) { CodeCLString = DetectCurrentCodeCL(Connection, ClientRootPath); } if (!String.IsNullOrEmpty(CodeCLString)) { CommandUtils.ConditionallySetEnvVar(EnvVarNames.CodeChangelist, CodeCLString); } CommandUtils.ConditionallySetEnvVar("P4USER", UserName); CommandUtils.ConditionallySetEnvVar("P4CLIENT", ThisClient.Name); var P4Password = Environment.GetEnvironmentVariable(EnvVarNames.P4Password); if (!String.IsNullOrEmpty(P4Password)) { CommandUtils.ConditionallySetEnvVar("P4PASSWD", P4Password); } SetBuildRootEscaped(); base.InitEnvironment(Connection, CmdEnv); }
public static bool LocalTempStorageExists(CommandEnvironment Env, string StorageBlockName, bool bQuiet = false) { var LocalManifest = LocalTempStorageManifestFilename(Env, StorageBlockName); if (FileExists_NoExceptions(bQuiet, LocalManifest)) { return true; } return false; }
public static bool SharedTempStorageExists(CommandEnvironment Env, string StorageBlockName, string GameFolder = "", bool bQuiet = false) { var SharedManifest = SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder); if (FileExists_NoExceptions(bQuiet, SharedManifest)) { return true; } return false; }
/// <summary> /// Builds a target using UBT. Automatically creates a logfile. When /// no LogName is specified, the executable name is used as logfile base name. /// </summary> /// <param name="Env">BuildEnvironment to use.</param> /// <param name="Project">Unreal project to build (optional)</param> /// <param name="Target">Target to build.</param> /// <param name="Platform">Platform to build for.</param> /// <param name="Config">Configuration to build.</param> /// <param name="AdditionalArgs">Additional arguments to pass on to UBT.</param> /// <param name="LogName">Optional logifle name.</param> public static void RunUBT(CommandEnvironment Env, string UBTExecutable, FileReference Project, string Target, UnrealTargetPlatform Platform, UnrealTargetConfiguration Config, string AdditionalArgs = "") { RunUBT(Env, UBTExecutable, UBTCommandline(Project, Target, Platform, Config, AdditionalArgs)); }
public static void StoreToTempStorage(CommandEnvironment Env, string StorageBlockName, List<string> Files, bool bLocalOnly = false, string GameFolder = "", string BaseFolder = "") { if (String.IsNullOrEmpty(BaseFolder)) { BaseFolder = Env.LocalRoot; } BaseFolder = CombinePaths(BaseFolder, "/"); if (!BaseFolder.EndsWith("/") && !BaseFolder.EndsWith("\\")) { throw new AutomationException("base folder {0} should end with a separator", BaseFolder); } var Local = SaveLocalTempStorageManifest(Env, BaseFolder, StorageBlockName, Files); if (!bLocalOnly) { var StartTime = DateTime.UtcNow; var BlockPath = SharedTempStorageDirectory(StorageBlockName, GameFolder); Log("Storing to {0}", BlockPath); if (DirectoryExists_NoExceptions(BlockPath)) { throw new AutomationException("Storage Block Already Exists! {0}", BlockPath); } CreateDirectory(true, BlockPath); if (!DirectoryExists_NoExceptions(BlockPath)) { throw new AutomationException("Storage Block Could Not Be Created! {0}", BlockPath); } var DestFiles = new List<string>(); if (ThreadsToCopyWith() < 2) { foreach (string InFilename in Files) { var Filename = CombinePaths(InFilename); Robust_FileExists_NoExceptions(false, Filename, "Could not add {0} to manifest because it does not exist"); if (!Filename.StartsWith(BaseFolder, StringComparison.InvariantCultureIgnoreCase)) { throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BaseFolder); } var RelativeFile = Filename.Substring(BaseFolder.Length); var DestFile = CombinePaths(BlockPath, RelativeFile); if (FileExists_NoExceptions(true, DestFile)) { throw new AutomationException("Dest file {0} already exists.", DestFile); } CopyFile(Filename, DestFile, true); Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}"); DestFiles.Add(DestFile); } } else { var SrcFiles = new List<string>(); foreach (string InFilename in Files) { var Filename = CombinePaths(InFilename); Robust_FileExists_NoExceptions(false, Filename, "Could not add {0} to manifest because it does not exist"); if (!Filename.StartsWith(BaseFolder, StringComparison.InvariantCultureIgnoreCase)) { throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BaseFolder); } var RelativeFile = Filename.Substring(BaseFolder.Length); var DestFile = CombinePaths(BlockPath, RelativeFile); if (FileExists_NoExceptions(true, DestFile)) { throw new AutomationException("Dest file {0} already exists.", DestFile); } SrcFiles.Add(Filename); DestFiles.Add(DestFile); } ThreadedCopyFiles(SrcFiles.ToArray(), DestFiles.ToArray(), ThreadsToCopyWith()); foreach (string DestFile in DestFiles) { Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}"); } } var Shared = SaveSharedTempStorageManifest(Env, StorageBlockName, GameFolder, DestFiles); if (!Local.Compare(Shared)) { // we will rename this so it can't be used, but leave it around for inspection RenameFile_NoExceptions(SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder), SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder) + ".broken"); throw new AutomationException("Shared and Local manifest mismatch."); } float BuildDuration = (float)((DateTime.UtcNow - StartTime).TotalSeconds); if (BuildDuration > 60.0f && Shared.GetTotalSize() > 0) { var MBSec = (((float)(Shared.GetTotalSize())) / (1024.0f * 1024.0f)) / BuildDuration; Log("Wrote to shared temp storage at {0} MB/s {1}B {2}s", MBSec, Shared.GetTotalSize(), BuildDuration); } } }
/// <summary> /// Runs external program and writes the output to a logfile. /// </summary> /// <param name="Env">Environment to use.</param> /// <param name="App">Executable to run</param> /// <param name="CommandLine">Commandline to pass on to the executable</param> /// <param name="LogName">Name of the logfile ( if null, executable name is used )</param> /// <param name="Input">Optional Input for the program (will be provided as stdin)</param> /// <param name="Options">Defines the options how to run. See ERunOptions.</param> /// <param name="FilterCallback">Callback to filter log spew before output.</param> public static void RunAndLog(CommandEnvironment Env, string App, string CommandLine, string LogName = null, int MaxSuccessCode = 0, string Input = null, ERunOptions Options = ERunOptions.Default, Dictionary <string, string> EnvVars = null, ProcessResult.SpewFilterCallbackType SpewFilterCallback = null) { RunAndLog(App, CommandLine, GetRunAndLogOnlyName(Env, App, LogName), MaxSuccessCode, Input, Options, EnvVars, SpewFilterCallback); }
public static List<string> RetrieveFromTempStorage(CommandEnvironment Env, string StorageBlockName, out bool WasLocal, string GameFolder = "", string BaseFolder = "") { if (String.IsNullOrEmpty(BaseFolder)) { BaseFolder = Env.LocalRoot; } BaseFolder = CombinePaths(BaseFolder, "/"); if (!BaseFolder.EndsWith("/") && !BaseFolder.EndsWith("\\")) { throw new AutomationException("base folder {0} should end with a separator", BaseFolder); } var Files = new List<string>(); var LocalManifest = LocalTempStorageManifestFilename(Env, StorageBlockName); if (FileExists_NoExceptions(LocalManifest)) { Log("Found local manifest {0}", LocalManifest); var Local = new TempStorageManifest(); Local.Load(LocalManifest); Files = Local.GetFiles(BaseFolder); var LocalTest = new TempStorageManifest(); LocalTest.Create(Files, BaseFolder); if (!Local.Compare(LocalTest)) { throw new AutomationException("Local files in manifest {0} were tampered with.", LocalManifest); } WasLocal = true; return Files; } WasLocal = false; var StartTime = DateTime.UtcNow; var BlockPath = CombinePaths(SharedTempStorageDirectory(StorageBlockName, GameFolder), "/"); if (!BlockPath.EndsWith("/") && !BlockPath.EndsWith("\\")) { throw new AutomationException("base folder {0} should end with a separator", BlockPath); } Log("Attempting to retrieve from {0}", BlockPath); if (!DirectoryExists_NoExceptions(BlockPath)) { throw new AutomationException("Storage Block Does Not Exists! {0}", BlockPath); } var SharedManifest = SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder); Robust_FileExists_NoExceptions(SharedManifest, "Storage Block Manifest Does Not Exists! {0}"); var Shared = new TempStorageManifest(); Shared.Load(SharedManifest); var SharedFiles = Shared.GetFiles(BlockPath); var DestFiles = new List<string>(); if (ThreadsToCopyWith() < 2) { foreach (string InFilename in SharedFiles) { var Filename = CombinePaths(InFilename); Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist"); if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase)) { throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath); } var RelativeFile = Filename.Substring(BlockPath.Length); var DestFile = CombinePaths(BaseFolder, RelativeFile); if (FileExists_NoExceptions(true, DestFile)) { Log("Dest file {0} already exists, deleting and overwriting", DestFile); DeleteFile(DestFile); } CopyFile(Filename, DestFile, true); Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}"); if (UnrealBuildTool.Utils.IsRunningOnMono) { FixUnixFilePermissions(DestFile); } FileInfo Info = new FileInfo(DestFile); DestFiles.Add(Info.FullName); } } else { var SrcFiles = new List<string>(); foreach (string InFilename in SharedFiles) { var Filename = CombinePaths(InFilename); //Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist"); if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase)) { throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath); } var RelativeFile = Filename.Substring(BlockPath.Length); var DestFile = CombinePaths(BaseFolder, RelativeFile); if (FileExists_NoExceptions(true, DestFile)) { Log("Dest file {0} already exists, deleting and overwriting", DestFile); DeleteFile(DestFile); } SrcFiles.Add(Filename); DestFiles.Add(DestFile); } ThreadedCopyFiles(SrcFiles.ToArray(), DestFiles.ToArray(), ThreadsToCopyWith()); var NewDestFiles = new List<string>(); foreach (string DestFile in DestFiles) { Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}"); if (UnrealBuildTool.Utils.IsRunningOnMono) { FixUnixFilePermissions(DestFile); } FileInfo Info = new FileInfo(DestFile); NewDestFiles.Add(Info.FullName); } DestFiles = NewDestFiles; } var NewLocal = SaveLocalTempStorageManifest(Env, BaseFolder, StorageBlockName, DestFiles); if (!NewLocal.Compare(Shared)) { // we will rename this so it can't be used, but leave it around for inspection RenameFile_NoExceptions(LocalManifest, LocalManifest + ".broken"); throw new AutomationException("Shared and Local manifest mismatch."); } float BuildDuration = (float)((DateTime.UtcNow - StartTime).TotalSeconds); if (BuildDuration > 60.0f && Shared.GetTotalSize() > 0) { var MBSec = (((float)(Shared.GetTotalSize())) / (1024.0f * 1024.0f)) / BuildDuration; Log("Read from shared temp storage at {0} MB/s {1}B {2}s", MBSec, Shared.GetTotalSize(), BuildDuration); } return DestFiles; }
/// <summary> /// Runs external program and writes the output to a logfile. /// </summary> /// <param name="Env">Environment to use.</param> /// <param name="App">Executable to run</param> /// <param name="CommandLine">Commandline to pass on to the executable</param> /// <param name="LogName">Name of the logfile ( if null, executable name is used )</param> /// <param name="FilterCallback">Callback to filter log spew before output.</param> /// <returns>Whether the program executed successfully or not.</returns> public static string RunAndLog(CommandEnvironment Env, string App, string CommandLine, out int SuccessCode, string LogName = null, Dictionary <string, string> EnvVars = null, ProcessResult.SpewFilterCallbackType SpewFilterCallback = null) { return(RunAndLog(App, CommandLine, out SuccessCode, GetRunAndLogOnlyName(Env, App, LogName), EnvVars, SpewFilterCallback)); }