/// <summary>
/// Runs UBT with the specified commandline. Automatically creates a logfile. When
/// no LogName is specified, the executable name is used as logfile base name.
/// </summary>
/// <param name="Env">Environment to use.</param>
/// <param name="CommandLine">Commandline to pass on to UBT.</param>
/// <param name="LogName">Optional logfile name.</param>
public static void RunUBT(CommandEnvironment Env, string UBTExecutable, string CommandLine, string LogName = null, Dictionary<string, string> EnvVars = null)
{
if (!FileExists(UBTExecutable))
{
throw new AutomationException("Unable to find UBT executable: " + UBTExecutable);
}
if (GlobalCommandLine.Rocket)
{
CommandLine += " -rocket";
}
if (!IsBuildMachine && UnrealBuildTool.BuildHostPlatform.Current.Platform == UnrealBuildTool.UnrealTargetPlatform.Mac)
{
CommandLine += " -nocreatestub";
}
CommandLine += " -NoHotReloadFromIDE";
if (bJunkDeleted || GlobalCommandLine.IgnoreJunk)
{
// UBT has already deleted junk files, make sure it doesn't do it again
CommandLine += " -ignorejunk";
}
else
{
// UBT will delete junk on first run
bJunkDeleted = true;
}
CommandUtils.RunAndLog(Env, UBTExecutable, CommandLine, LogName, EnvVars: EnvVars);
}
protected virtual void InitEnvironment(P4Connection Connection, CommandEnvironment CmdEnv)
{
//
// P4 Environment
//
P4Port = CommandUtils.GetEnvVar(EnvVarNames.P4Port);
ClientRoot = CommandUtils.GetEnvVar(EnvVarNames.ClientRoot);
User = CommandUtils.GetEnvVar(EnvVarNames.User);
ChangelistStringInternal = CommandUtils.GetEnvVar(EnvVarNames.Changelist, null);
Client = CommandUtils.GetEnvVar(EnvVarNames.Client);
BuildRootP4 = CommandUtils.GetEnvVar(EnvVarNames.BuildRootP4);
if (BuildRootP4.EndsWith("/", StringComparison.InvariantCultureIgnoreCase) || BuildRootP4.EndsWith("\\", StringComparison.InvariantCultureIgnoreCase))
{
// We expect the build root to not end with a path separator
BuildRootP4 = BuildRootP4.Substring(0, BuildRootP4.Length - 1);
CommandUtils.SetEnvVar(EnvVarNames.BuildRootP4, BuildRootP4);
}
BuildRootEscaped = CommandUtils.GetEnvVar(EnvVarNames.BuildRootEscaped);
LabelToSync = CommandUtils.GetEnvVar(EnvVarNames.LabelToSync);
string CodeChangelistString = CommandUtils.GetEnvVar(EnvVarNames.CodeChangelist);
if(!String.IsNullOrEmpty(CodeChangelistString))
{
CodeChangelist = Int32.Parse(CodeChangelistString);
}
if (((CommandUtils.P4Enabled || CommandUtils.IsBuildMachine) && (ClientRoot == String.Empty || User == String.Empty ||
(String.IsNullOrEmpty(ChangelistStringInternal) && CommandUtils.IsBuildMachine) || Client == String.Empty || BuildRootP4 == String.Empty)))
{
Log.TraceInformation("P4Enabled={0}", CommandUtils.P4Enabled);
Log.TraceInformation("ClientRoot={0}",ClientRoot );
Log.TraceInformation("User={0}", User);
Log.TraceInformation("ChangelistString={0}", ChangelistStringInternal);
Log.TraceInformation("Client={0}", Client);
Log.TraceInformation("BuildRootP4={0}", BuildRootP4);
throw new AutomationException("BUILD FAILED Perforce Environment is not set up correctly. Please check your environment variables.");
}
LabelPrefix = BuildRootP4 + "/";
if (CommandUtils.P4Enabled)
{
if (CommandUtils.IsBuildMachine || ChangelistStringInternal != null)
{
// We may not always need the changelist number if we're not a build machine.
// In local runs, changelist initialization can be really slow!
VerifyChangelistStringAndSetChangelistNumber();
}
}
LogSettings();
}
/// <summary>
/// Builds a Visual Studio solution with MsDevEnv. Automatically creates a logfile. When
/// no LogName is specified, the executable name is used as logfile base name.
/// </summary>
/// <param name="Env">BuildEnvironment to use.</param>
/// <param name="SolutionFile">Path to the solution file</param>
/// <param name="BuildConfig">Configuration to build.</param>
/// <param name="LogName">Optional logfile name.</param>
public static void BuildSolution(CommandEnvironment Env, string SolutionFile, string BuildConfig = "Development", string LogName = null)
{
if (!FileExists(SolutionFile))
{
throw new AutomationException(String.Format("Unabled to build Solution {0}. Solution file not found.", SolutionFile));
}
if (String.IsNullOrEmpty(Env.MsDevExe))
{
throw new AutomationException(String.Format("Unabled to build Solution {0}. devenv.com not found.", SolutionFile));
}
string CmdLine = String.Format("\"{0}\" /build \"{1}\"", SolutionFile, BuildConfig);
RunAndLog(Env, Env.MsDevExe, CmdLine, LogName);
}
/// <summary>
/// Runs UBT with the specified commandline. Automatically creates a logfile. When
/// no LogName is specified, the executable name is used as logfile base name.
/// </summary>
/// <param name="Env">Environment to use.</param>
/// <param name="CommandLine">Commandline to pass on to UBT.</param>
/// <param name="LogName">Optional logfile name.</param>
public static void RunUBT(CommandEnvironment Env, string UBTExecutable, string CommandLine, string LogName = null)
{
if (!FileExists(UBTExecutable))
{
throw new AutomationException("Unable to find UBT executable: " + UBTExecutable);
}
if (GlobalCommandLine.Rocket)
{
CommandLine += " -rocket";
}
CommandUtils.RunAndLog(Env, UBTExecutable, CommandLine, LogName);
}
/// <summary>
/// Builds a CSharp project with msbuild.exe. Automatically creates a logfile. When
/// no LogName is specified, the executable name is used as logfile base name.
/// </summary>
/// <param name="Env">BuildEnvironment to use.</param>
/// <param name="ProjectFile">Path to the project file. Must be a .csproj file.</param>
/// <param name="BuildConfig">Configuration to build.</param>
/// <param name="LogName">Optional logfile name.</param>
public static void BuildCSharpProject(CommandEnvironment Env, string ProjectFile, string BuildConfig = "Development", string LogName = null)
{
if (!ProjectFile.EndsWith(".csproj"))
{
throw new AutomationException(String.Format("Unabled to build Project {0}. Not a valid .csproj file.", ProjectFile));
}
if (!FileExists(ProjectFile))
{
throw new AutomationException(String.Format("Unabled to build Project {0}. Project file not found.", ProjectFile));
}
string CmdLine = String.Format(@"/verbosity:normal /target:Rebuild /property:Configuration={0} /property:Platform=AnyCPU", BuildConfig);
MsBuild(Env, ProjectFile, CmdLine, LogName);
}
/// <summary>
/// Builds a Visual Studio solution with MsBuild (using msbuild.exe rather than devenv.com can help circumvent issues with expired Visual Studio licenses).
/// Automatically creates a logfile. When no LogName is specified, the executable name is used as logfile base name.
/// </summary>
/// <param name="Env">BuildEnvironment to use.</param>
/// <param name="SolutionFile">Path to the solution file</param>
/// <param name="BuildConfig">Configuration to build.</param>
/// <param name="BuildPlatform">Platform to build.</param>
/// <param name="LogName">Optional logfile name.</param>
public static void BuildSolution(CommandEnvironment Env, string SolutionFile, string BuildConfig, string BuildPlatform, string LogName = null)
{
if (!FileExists(SolutionFile))
{
throw new AutomationException(String.Format("Unabled to build Solution {0}. Solution file not found.", SolutionFile));
}
if (String.IsNullOrEmpty(Env.MsBuildExe))
{
throw new AutomationException("Unable to find msbuild.exe at: \"{0}\"", Env.MsBuildExe);
}
string CmdLine = String.Format("\"{0}\" /m /t:Build /p:Configuration=\"{1}\" /p:Platform=\"{2}\" /verbosity:minimal /nologo", SolutionFile, BuildConfig, BuildPlatform);
using (TelemetryStopwatch CompileStopwatch = new TelemetryStopwatch("Compile.{0}.{1}.{2}", Path.GetFileName(SolutionFile), "WinC#", BuildConfig))
{
RunAndLog(Env, Env.MsBuildExe, CmdLine, LogName);
}
}
/// <summary>
/// Builds a Visual Studio solution with MsDevEnv. Automatically creates a logfile. When
/// no LogName is specified, the executable name is used as logfile base name.
/// </summary>
/// <param name="Env">BuildEnvironment to use.</param>
/// <param name="SolutionFile">Path to the solution file</param>
/// <param name="BuildConfig">Configuration to build.</param>
/// <param name="LogName">Optional logfile name.</param>
public static void BuildSolution(CommandEnvironment Env, string SolutionFile, string BuildConfig = "Development", string LogName = null)
{
if (!FileExists(SolutionFile))
{
throw new AutomationException(String.Format("Unabled to build Solution {0}. Solution file not found.", SolutionFile));
}
if (String.IsNullOrEmpty(Env.MsDevExe))
{
throw new AutomationException(String.Format("Unabled to build Solution {0}. devenv.com not found.", SolutionFile));
}
string CmdLine = String.Format("\"{0}\" /build \"{1}\"", SolutionFile, BuildConfig);
using(TelemetryStopwatch CompileStopwatch = new TelemetryStopwatch("Compile.{0}.{1}.{2}", Path.GetFileName(SolutionFile), "WinC#", BuildConfig))
{
RunAndLog(Env, Env.MsDevExe, CmdLine, LogName);
}
}
/// <summary>
/// Builds a Visual Studio solution with MsDevEnv. Automatically creates a logfile. When
/// no LogName is specified, the executable name is used as logfile base name.
/// </summary>
/// <param name="Env">BuildEnvironment to use.</param>
/// <param name="SolutionFile">Path to the solution file</param>
/// <param name="BuildConfig">Configuration to build.</param>
/// <param name="LogName">Optional logfile name.</param>
public static void BuildSolution(CommandEnvironment Env, string SolutionFile, string BuildConfig = "Development", string LogName = null)
{
if (!FileExists(SolutionFile))
{
throw new AutomationException(String.Format("Unabled to build Solution {0}. Solution file not found.", SolutionFile));
}
if (String.IsNullOrEmpty(Env.MsDevExe))
{
throw new AutomationException(String.Format("Unabled to build Solution {0}. devenv.com not found.", SolutionFile));
}
string CmdLine = String.Format("\"{0}\" /build \"{1}\"", SolutionFile, BuildConfig);
var StartBuildSolution = DateTime.Now.ToString();
RunAndLog(Env, Env.MsDevExe, CmdLine, LogName);
var FinishBuildSolution = DateTime.Now.ToString();
PrintCSVFile(String.Format("UAT,Compile,{0},{1}", StartBuildSolution, FinishBuildSolution));
}
/// <summary>
/// Runs msbuild.exe with the specified arguments. Automatically creates a logfile. When
/// no LogName is specified, the executable name is used as logfile base name.
/// </summary>
/// <param name="Env">BuildEnvironment to use.</param>
/// <param name="Project">Path to the project to build.</param>
/// <param name="Arguments">Arguments to pass to msbuild.exe.</param>
/// <param name="LogName">Optional logfile name.</param>
public static void MsBuild(CommandEnvironment Env, string Project, string Arguments, string LogName)
{
if (String.IsNullOrEmpty(Env.MsBuildExe))
{
throw new AutomationException("Unable to find msbuild.exe at: \"{0}\"", Env.MsBuildExe);
}
if (!FileExists(Project))
{
throw new AutomationException("Project {0} does not exist!", Project);
}
var RunArguments = MakePathSafeToUseWithCommandLine(Project);
if (!String.IsNullOrEmpty(Arguments))
{
RunArguments += " " + Arguments;
}
RunAndLog(Env, Env.MsBuildExe, RunArguments, LogName);
}
/// <summary>
/// Runs UBT with the specified commandline. Automatically creates a logfile. When
/// no LogName is specified, the executable name is used as logfile base name.
/// </summary>
/// <param name="Env">Environment to use.</param>
/// <param name="CommandLine">Commandline to pass on to UBT.</param>
/// <param name="LogName">Optional logfile name.</param>
public static void RunUBT(CommandEnvironment Env, string UBTExecutable, string CommandLine, string LogName = null)
{
if (!FileExists(UBTExecutable))
{
throw new AutomationException("Unable to find UBT executable: " + UBTExecutable);
}
if (GlobalCommandLine.Rocket)
{
CommandLine += " -rocket";
}
if (!IsBuildMachine && UnrealBuildTool.BuildHostPlatform.Current.Platform == UnrealBuildTool.UnrealTargetPlatform.Mac)
{
CommandLine += " -nocreatestub";
}
CommandUtils.RunAndLog(Env, UBTExecutable, CommandLine, LogName);
}
public static TempStorageManifest SaveSharedTempStorageManifest(CommandEnvironment Env, string StorageBlockName, string GameFolder, List<string> Files)
{
return SaveTempStorageManifest(SharedTempStorageDirectory(StorageBlockName, GameFolder), SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder), Files);
}
public static void StoreToTempStorage(CommandEnvironment Env, string StorageBlockName, List <string> Files, bool bLocalOnly = false, string GameFolder = "", string BaseFolder = "")
{
if (String.IsNullOrEmpty(BaseFolder))
{
BaseFolder = Env.LocalRoot;
}
BaseFolder = CombinePaths(BaseFolder, "/");
if (!BaseFolder.EndsWith("/") && !BaseFolder.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BaseFolder);
}
var Local = SaveLocalTempStorageManifest(Env, BaseFolder, StorageBlockName, Files);
if (!bLocalOnly)
{
var StartTime = DateTime.UtcNow;
var BlockPath = SharedTempStorageDirectory(StorageBlockName, GameFolder);
Log("Storing to {0}", BlockPath);
if (DirectoryExists_NoExceptions(BlockPath))
{
throw new AutomationException("Storage Block Already Exists! {0}", BlockPath);
}
CreateDirectory(true, BlockPath);
if (!DirectoryExists_NoExceptions(BlockPath))
{
throw new AutomationException("Storage Block Could Not Be Created! {0}", BlockPath);
}
var DestFiles = new List <string>();
if (ThreadsToCopyWith() < 2)
{
foreach (string InFilename in Files)
{
var Filename = CombinePaths(InFilename);
Robust_FileExists_NoExceptions(false, Filename, "Could not add {0} to manifest because it does not exist");
if (!Filename.StartsWith(BaseFolder, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BaseFolder);
}
var RelativeFile = Filename.Substring(BaseFolder.Length);
var DestFile = CombinePaths(BlockPath, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
throw new AutomationException("Dest file {0} already exists.", DestFile);
}
CopyFile(Filename, DestFile, true);
Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}");
DestFiles.Add(DestFile);
}
}
else
{
var SrcFiles = new List <string>();
foreach (string InFilename in Files)
{
var Filename = CombinePaths(InFilename);
Robust_FileExists_NoExceptions(false, Filename, "Could not add {0} to manifest because it does not exist");
if (!Filename.StartsWith(BaseFolder, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BaseFolder);
}
var RelativeFile = Filename.Substring(BaseFolder.Length);
var DestFile = CombinePaths(BlockPath, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
throw new AutomationException("Dest file {0} already exists.", DestFile);
}
SrcFiles.Add(Filename);
DestFiles.Add(DestFile);
}
ThreadedCopyFiles(SrcFiles.ToArray(), DestFiles.ToArray(), ThreadsToCopyWith());
foreach (string DestFile in DestFiles)
{
Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}");
}
}
var Shared = SaveSharedTempStorageManifest(Env, StorageBlockName, GameFolder, DestFiles);
if (!Local.Compare(Shared))
{
// we will rename this so it can't be used, but leave it around for inspection
RenameFile_NoExceptions(SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder), SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder) + ".broken");
throw new AutomationException("Shared and Local manifest mismatch.");
}
float BuildDuration = (float)((DateTime.UtcNow - StartTime).TotalSeconds);
if (BuildDuration > 60.0f && Shared.GetTotalSize() > 0)
{
var MBSec = (((float)(Shared.GetTotalSize())) / (1024.0f * 1024.0f)) / BuildDuration;
Log("Wrote to shared temp storage at {0} MB/s {1}B {2}s", MBSec, Shared.GetTotalSize(), BuildDuration);
}
}
}
/// <summary>
/// Runs UAT recursively
/// </summary>
/// <param name="Env">Environment to use.</param>
/// <param name="CommandLine">Commandline to pass on to the executable</param>
public static string RunUAT(CommandEnvironment Env, string CommandLine)
{
// this doesn't do much, but it does need to make sure log folders are reasonable and don't collide
string BaseLogSubdir = "Recur";
if (!String.IsNullOrEmpty(CommandLine))
{
int Space = CommandLine.IndexOf(" ");
if (Space > 0)
{
BaseLogSubdir = BaseLogSubdir + "_" + CommandLine.Substring(0, Space);
}
else
{
BaseLogSubdir = BaseLogSubdir + "_" + CommandLine;
}
}
int Index = 0;
BaseLogSubdir = BaseLogSubdir.Trim();
string DirOnlyName = BaseLogSubdir;
string LogSubdir = CombinePaths(CmdEnv.LogFolder, DirOnlyName, "");
while (true)
{
var ExistingFiles = FindFiles(DirOnlyName + "*", false, CmdEnv.LogFolder);
if (ExistingFiles.Length == 0)
{
break;
}
Index++;
if (Index == 1000)
{
throw new AutomationException("Couldn't seem to create a log subdir {0}", LogSubdir);
}
DirOnlyName = String.Format("{0}_{1}_", BaseLogSubdir, Index);
LogSubdir = CombinePaths(CmdEnv.LogFolder, DirOnlyName, "");
}
string LogFile = CombinePaths(CmdEnv.LogFolder, DirOnlyName + ".log");
Log("Recursive UAT Run, in log folder {0}, main log file {1}", LogSubdir, LogFile);
CreateDirectory(LogSubdir);
string App = CmdEnv.UATExe;
Log("Running {0} {1}", App, CommandLine);
var OSEnv = new Dictionary<string, string>();
OSEnv.Add(AutomationTool.EnvVarNames.LogFolder, LogSubdir);
OSEnv.Add("uebp_UATMutexNoWait", "1");
if (!IsBuildMachine)
{
OSEnv.Add(AutomationTool.EnvVarNames.LocalRoot, ""); // if we don't clear this out, it will think it is a build machine; it will rederive everything
}
ProcessResult Result = Run(App, CommandLine, null, ERunOptions.Default, OSEnv);
if (Result.Output.Length > 0)
{
WriteToFile(LogFile, Result.Output);
}
else
{
WriteToFile(LogFile, "[None!, no output produced]");
}
Log("Flattening log folder {0}", LogSubdir);
var Files = FindFiles("*", true, LogSubdir);
string MyLogFolder = CombinePaths(CmdEnv.LogFolder, "");
foreach (var ThisFile in Files)
{
if (!ThisFile.StartsWith(MyLogFolder, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Can't rebase {0} because it doesn't start with {1}", ThisFile, MyLogFolder);
}
string NewFilename = ThisFile.Substring(MyLogFolder.Length).Replace("/", "_").Replace("\\", "_");
NewFilename = CombinePaths(CmdEnv.LogFolder, NewFilename);
if (FileExists_NoExceptions(NewFilename))
{
throw new AutomationException("Destination log file already exists? {0}", NewFilename);
}
CopyFile(ThisFile, NewFilename);
if (!FileExists_NoExceptions(NewFilename))
{
throw new AutomationException("Destination log file could not be copied {0}", NewFilename);
}
DeleteFile_NoExceptions(ThisFile);
}
DeleteDirectory_NoExceptions(LogSubdir);
if (Result != 0)
{
throw new AutomationException(String.Format("Recursive UAT Command failed (Result:{3}): {0} {1}. See logfile for details: '{2}' ",
App, CommandLine, Path.GetFileName(LogFile), Result.ExitCode));
}
return LogFile;
}
public static TempStorageManifest SaveLocalTempStorageManifest(CommandEnvironment Env, string BaseFolder, string StorageBlockName, List <string> Files)
{
return(SaveTempStorageManifest(BaseFolder, LocalTempStorageManifestFilename(Env, StorageBlockName), Files));
}
public static string LocalTempStorageManifestDirectory(CommandEnvironment Env)
{
return(CombinePaths(Env.LocalRoot, "Engine", "Saved", "GUBP"));
}
/// <summary>
/// Runs external program and writes the output to a logfile.
/// </summary>
/// <param name="Env">Environment to use.</param>
/// <param name="App">Executable to run</param>
/// <param name="CommandLine">Commandline to pass on to the executable</param>
/// <param name="LogName">Name of the logfile ( if null, executable name is used )</param>
/// <returns>Whether the program executed successfully or not.</returns>
public static string RunAndLog(CommandEnvironment Env, string App, string CommandLine, out int SuccessCode, string LogName = null)
{
return(RunAndLog(App, CommandLine, out SuccessCode, GetRunAndLogLogName(Env, App, LogName)));
}
public static bool TempStorageExists(CommandEnvironment Env, string StorageBlockName, string GameFolder = "", bool bLocalOnly = false, bool bQuiet = false)
{
return LocalTempStorageExists(Env, StorageBlockName, bQuiet) || (!bLocalOnly && SharedTempStorageExists(Env, StorageBlockName, GameFolder, bQuiet));
}
/// <summary>
/// Constructor. Derives the Perforce environment settings.
/// </summary>
internal P4Environment(CommandEnvironment CmdEnv)
{
// Get the Perforce port setting
ServerAndPort = CommandUtils.GetEnvVar(EnvVarNames.P4Port);
if (String.IsNullOrEmpty(ServerAndPort))
{
ServerAndPort = DetectP4Port();
CommandUtils.SetEnvVar(EnvVarNames.P4Port, ServerAndPort);
}
// Get the Perforce user setting
User = CommandUtils.GetEnvVar(EnvVarNames.User);
if (String.IsNullOrEmpty(User))
{
P4Connection DefaultConnection = new P4Connection(User: null, Client: null, ServerAndPort: ServerAndPort);
User = DetectUserName(DefaultConnection);
CommandUtils.SetEnvVar(EnvVarNames.User, User);
}
// Get the Perforce client setting
Client = CommandUtils.GetEnvVar(EnvVarNames.Client);
if (String.IsNullOrEmpty(Client))
{
P4Connection DefaultConnection = new P4Connection(User: User, Client: null, ServerAndPort: ServerAndPort);
P4ClientInfo ThisClient = DetectClient(DefaultConnection, User, Environment.MachineName.ToLower(), CmdEnv.UATExe);
Log.TraceInformation("Using user {0} clientspec {1} {2}", User, ThisClient.Name, ThisClient.RootPath);
string BranchPath;
string ClientRootPath;
P4Connection ClientConnection = new P4Connection(User: User, Client: ThisClient.Name, ServerAndPort: ServerAndPort);
DetectRootPaths(ClientConnection, CmdEnv.LocalRoot, ThisClient, out BranchPath, out ClientRootPath);
Client = ThisClient.Name;
CommandUtils.SetEnvVar(EnvVarNames.Client, Client);
Branch = BranchPath;
CommandUtils.SetEnvVar(EnvVarNames.BuildRootP4, Branch);
ClientRoot = ClientRootPath;
CommandUtils.SetEnvVar(EnvVarNames.ClientRoot, ClientRootPath);
}
else
{
Branch = CommandUtils.GetEnvVar(EnvVarNames.BuildRootP4);
ClientRoot = CommandUtils.GetEnvVar(EnvVarNames.ClientRoot);
if (String.IsNullOrEmpty(Branch) || String.IsNullOrEmpty(ClientRoot))
{
throw new AutomationException("{0} and {1} must also be set with {2}", EnvVarNames.ClientRoot, EnvVarNames.BuildRootP4, EnvVarNames.Client);
}
}
// We expect the build root to not end with a path separator
if (Branch.EndsWith("/"))
{
Branch = Branch.TrimEnd('/');
CommandUtils.SetEnvVar(EnvVarNames.BuildRootP4, Branch);
}
// Set the current changelist
string ChangelistString = CommandUtils.GetEnvVar(EnvVarNames.Changelist, null);
if (String.IsNullOrEmpty(ChangelistString) && CommandUtils.P4CLRequired)
{
P4Connection Connection = new P4Connection(User, Client, ServerAndPort);
ChangelistString = DetectCurrentCL(Connection, ClientRoot);
CommandUtils.SetEnvVar(EnvVarNames.Changelist, ChangelistString);
}
if (!String.IsNullOrEmpty(ChangelistString))
{
Changelist = int.Parse(ChangelistString);
}
// Set the current code changelist
string CodeChangelistString = CommandUtils.GetEnvVar(EnvVarNames.CodeChangelist);
if (String.IsNullOrEmpty(CodeChangelistString) && CommandUtils.P4CLRequired)
{
P4Connection Connection = new P4Connection(User, Client, ServerAndPort);
CodeChangelistString = DetectCurrentCodeCL(Connection, ClientRoot);
CommandUtils.SetEnvVar(EnvVarNames.CodeChangelist, CodeChangelistString);
}
if (!String.IsNullOrEmpty(CodeChangelistString))
{
CodeChangelist = int.Parse(CodeChangelistString);
}
// Set the standard environment variables based on the values we've found
CommandUtils.SetEnvVar("P4PORT", ServerAndPort);
CommandUtils.SetEnvVar("P4USER", User);
CommandUtils.SetEnvVar("P4CLIENT", Client);
// Write a summary of the settings to the output window
if (!CommandUtils.CmdEnv.IsChildInstance)
{
Log.TraceInformation("Detected Perforce Settings:");
Log.TraceInformation(" Server: {0}", ServerAndPort);
Log.TraceInformation(" User: {0}", User);
Log.TraceInformation(" Client: {0}", Client);
Log.TraceInformation(" Branch: {0}", Branch);
if (ChangelistInternal != -1)
{
Log.TraceInformation(" Last Change: {0}", Changelist);
}
if (CodeChangelistInternal != -1)
{
Log.TraceInformation(" Last Code Change: {0}", CodeChangelist);
}
}
// Write all the environment variables to the log
Log.TraceLog("Perforce Environment Variables:");
Log.TraceLog(" {0}={1}", EnvVarNames.P4Port, InternalUtils.GetEnvironmentVariable(EnvVarNames.P4Port, "", true));
Log.TraceLog(" {0}={1}", EnvVarNames.User, InternalUtils.GetEnvironmentVariable(EnvVarNames.User, "", true));
Log.TraceLog(" {0}={1}", EnvVarNames.Client, InternalUtils.GetEnvironmentVariable(EnvVarNames.Client, "", true));
Log.TraceLog(" {0}={1}", EnvVarNames.BuildRootP4, InternalUtils.GetEnvironmentVariable(EnvVarNames.BuildRootP4, "", true));
Log.TraceLog(" {0}={1}", EnvVarNames.BuildRootEscaped, InternalUtils.GetEnvironmentVariable(EnvVarNames.BuildRootEscaped, "", true));
Log.TraceLog(" {0}={1}", EnvVarNames.ClientRoot, InternalUtils.GetEnvironmentVariable(EnvVarNames.ClientRoot, "", true));
Log.TraceLog(" {0}={1}", EnvVarNames.Changelist, InternalUtils.GetEnvironmentVariable(EnvVarNames.Changelist, "", true));
Log.TraceLog(" {0}={1}", EnvVarNames.CodeChangelist, InternalUtils.GetEnvironmentVariable(EnvVarNames.CodeChangelist, "", true));
Log.TraceLog(" {0}={1}", "P4PORT", InternalUtils.GetEnvironmentVariable("P4PORT", "", true));
Log.TraceLog(" {0}={1}", "P4USER", InternalUtils.GetEnvironmentVariable("P4USER", "", true));
Log.TraceLog(" {0}={1}", "P4CLIENT", InternalUtils.GetEnvironmentVariable("P4CLIENT", "", true));
}
/// <summary>
/// Runs external program and writes the output to a logfile.
/// </summary>
/// <param name="Env">Environment to use.</param>
/// <param name="App">Executable to run</param>
/// <param name="CommandLine">Commandline to pass on to the executable</param>
/// <param name="LogName">Name of the logfile ( if null, executable name is used )</param>
/// <param name="Input">Optional Input for the program (will be provided as stdin)</param>
/// <param name="Options">Defines the options how to run. See ERunOptions.</param>
public static void RunAndLog(CommandEnvironment Env, string App, string CommandLine, string LogName = null, int MaxSuccessCode = 0, string Input = null, ERunOptions Options = ERunOptions.Default)
{
RunAndLog(App, CommandLine, GetRunAndLogLogName(Env, App, LogName), MaxSuccessCode, Input, Options);
}
/// <summary>
/// Runs external program and writes the output to a logfile.
/// </summary>
/// <param name="Env">Environment to use.</param>
/// <param name="App">Executable to run</param>
/// <param name="CommandLine">Commandline to pass on to the executable</param>
/// <param name="LogName">Name of the logfile ( if null, executable name is used )</param>
/// <returns>Whether the program executed successfully or not.</returns>
public static string RunAndLog(CommandEnvironment Env, string App, string CommandLine, out int SuccessCode, string LogName = null, Dictionary <string, string> EnvVars = null)
{
return(RunAndLog(App, CommandLine, out SuccessCode, GetRunAndLogLogName(Env, App, LogName), EnvVars));
}
public static List <string> FindTempStorageManifests(CommandEnvironment Env, string StorageBlockName, bool LocalOnly = false, bool SharedOnly = false, string GameFolder = "")
{
var Files = new List <string>();
var LocalFiles = LocalTempStorageManifestFilename(Env, StorageBlockName);
var LocalParent = Path.GetDirectoryName(LocalFiles);
var WildCard = Path.GetFileName(LocalFiles);
int IndexOfStar = WildCard.IndexOf("*");
if (IndexOfStar < 0 || WildCard.LastIndexOf("*") != IndexOfStar)
{
throw new AutomationException("Wildcard {0} either has no star or it has more than one.", WildCard);
}
string PreStarWildcard = WildCard.Substring(0, IndexOfStar);
string PostStarWildcard = Path.GetFileNameWithoutExtension(WildCard.Substring(IndexOfStar + 1));
if (!SharedOnly && DirectoryExists_NoExceptions(LocalParent))
{
foreach (var ThisFile in CommandUtils.FindFiles_NoExceptions(WildCard, true, LocalParent))
{
Log(" Found local file {0}", ThisFile);
int IndexOfWildcard = ThisFile.IndexOf(PreStarWildcard);
if (IndexOfWildcard < 0)
{
throw new AutomationException("File {0} didn't contain {1}.", ThisFile, PreStarWildcard);
}
int LastIndexOfWildcardTail = ThisFile.LastIndexOf(PostStarWildcard);
if (LastIndexOfWildcardTail < 0 || LastIndexOfWildcardTail < IndexOfWildcard + PreStarWildcard.Length)
{
throw new AutomationException("File {0} didn't contain {1} or it was before the prefix", ThisFile, PostStarWildcard);
}
string StarReplacement = ThisFile.Substring(IndexOfWildcard + PreStarWildcard.Length, LastIndexOfWildcardTail - IndexOfWildcard - PreStarWildcard.Length);
if (StarReplacement.Length < 1)
{
throw new AutomationException("Dir {0} didn't have any string to fit the star in the wildcard {1}", ThisFile, WildCard);
}
if (!Files.Contains(StarReplacement))
{
Files.Add(StarReplacement);
}
}
}
if (!LocalOnly)
{
var SharedFiles = SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder);
var SharedParent = Path.GetDirectoryName(Path.GetDirectoryName(SharedFiles));
if (DirectoryExists_NoExceptions(SharedParent))
{
string[] Dirs = null;
try
{
Dirs = Directory.GetDirectories(SharedParent, Path.GetFileNameWithoutExtension(SharedFiles), SearchOption.TopDirectoryOnly);
}
catch (Exception Ex)
{
Log("Unable to Find Directories in {0} with wildcard {1}", SharedParent, Path.GetFileNameWithoutExtension(SharedFiles));
Log(" Exception was {0}", LogUtils.FormatException(Ex));
}
if (Dirs != null)
{
foreach (var ThisSubDir in Dirs)
{
int IndexOfWildcard = ThisSubDir.IndexOf(PreStarWildcard);
if (IndexOfWildcard < 0)
{
throw new AutomationException("Dir {0} didn't contain {1}.", ThisSubDir, PreStarWildcard);
}
int LastIndexOfWildcardTail = ThisSubDir.LastIndexOf(PostStarWildcard);
if (LastIndexOfWildcardTail < 0 || LastIndexOfWildcardTail < IndexOfWildcard + PreStarWildcard.Length)
{
throw new AutomationException("Dir {0} didn't contain {1} or it was before the prefix", ThisSubDir, PostStarWildcard);
}
string StarReplacement = ThisSubDir.Substring(IndexOfWildcard + PreStarWildcard.Length, LastIndexOfWildcardTail - IndexOfWildcard - PreStarWildcard.Length);
if (StarReplacement.Length < 1)
{
throw new AutomationException("Dir {0} didn't have any string to fit the star in the wildcard {1}", ThisSubDir, WildCard);
}
// these are a bunch of false positives
if (StarReplacement.Contains("-"))
{
continue;
}
if (!Files.Contains(StarReplacement))
{
Files.Add(StarReplacement);
}
}
}
}
}
var OutFiles = new List <string>();
foreach (var StarReplacement in Files)
{
var NewBlock = StorageBlockName.Replace("*", StarReplacement);
if (TempStorageExists(Env, NewBlock, GameFolder, LocalOnly, true))
{
OutFiles.Add(StarReplacement);
}
}
return(OutFiles);
}
public static List <string> RetrieveFromTempStorage(CommandEnvironment Env, string StorageBlockName, out bool WasLocal, string GameFolder = "", string BaseFolder = "")
{
if (String.IsNullOrEmpty(BaseFolder))
{
BaseFolder = Env.LocalRoot;
}
BaseFolder = CombinePaths(BaseFolder, "/");
if (!BaseFolder.EndsWith("/") && !BaseFolder.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BaseFolder);
}
var Files = new List <string>();
var LocalManifest = LocalTempStorageManifestFilename(Env, StorageBlockName);
if (FileExists_NoExceptions(LocalManifest))
{
Log("Found local manifest {0}", LocalManifest);
var Local = new TempStorageManifest();
Local.Load(LocalManifest);
Files = Local.GetFiles(BaseFolder);
var LocalTest = new TempStorageManifest();
LocalTest.Create(Files, BaseFolder);
if (!Local.Compare(LocalTest))
{
throw new AutomationException("Local files in manifest {0} were tampered with.", LocalManifest);
}
WasLocal = true;
return(Files);
}
WasLocal = false;
var StartTime = DateTime.UtcNow;
var BlockPath = CombinePaths(SharedTempStorageDirectory(StorageBlockName, GameFolder), "/");
if (!BlockPath.EndsWith("/") && !BlockPath.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BlockPath);
}
Log("Attempting to retrieve from {0}", BlockPath);
if (!DirectoryExists_NoExceptions(BlockPath))
{
throw new AutomationException("Storage Block Does Not Exists! {0}", BlockPath);
}
var SharedManifest = SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder);
Robust_FileExists_NoExceptions(SharedManifest, "Storage Block Manifest Does Not Exists! {0}");
var Shared = new TempStorageManifest();
Shared.Load(SharedManifest);
var SharedFiles = Shared.GetFiles(BlockPath);
var DestFiles = new List <string>();
if (ThreadsToCopyWith() < 2)
{
foreach (string InFilename in SharedFiles)
{
var Filename = CombinePaths(InFilename);
Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist");
if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath);
}
var RelativeFile = Filename.Substring(BlockPath.Length);
var DestFile = CombinePaths(BaseFolder, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
Log("Dest file {0} already exists, deleting and overwriting", DestFile);
DeleteFile(DestFile);
}
CopyFile(Filename, DestFile, true);
Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}");
if (UnrealBuildTool.Utils.IsRunningOnMono)
{
FixUnixFilePermissions(DestFile);
}
FileInfo Info = new FileInfo(DestFile);
DestFiles.Add(Info.FullName);
}
}
else
{
var SrcFiles = new List <string>();
foreach (string InFilename in SharedFiles)
{
var Filename = CombinePaths(InFilename);
//Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist");
if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath);
}
var RelativeFile = Filename.Substring(BlockPath.Length);
var DestFile = CombinePaths(BaseFolder, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
Log("Dest file {0} already exists, deleting and overwriting", DestFile);
DeleteFile(DestFile);
}
SrcFiles.Add(Filename);
DestFiles.Add(DestFile);
}
ThreadedCopyFiles(SrcFiles.ToArray(), DestFiles.ToArray(), ThreadsToCopyWith());
var NewDestFiles = new List <string>();
foreach (string DestFile in DestFiles)
{
Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}");
if (UnrealBuildTool.Utils.IsRunningOnMono)
{
FixUnixFilePermissions(DestFile);
}
FileInfo Info = new FileInfo(DestFile);
NewDestFiles.Add(Info.FullName);
}
DestFiles = NewDestFiles;
}
var NewLocal = SaveLocalTempStorageManifest(Env, BaseFolder, StorageBlockName, DestFiles);
if (!NewLocal.Compare(Shared))
{
// we will rename this so it can't be used, but leave it around for inspection
RenameFile_NoExceptions(LocalManifest, LocalManifest + ".broken");
throw new AutomationException("Shared and Local manifest mismatch.");
}
float BuildDuration = (float)((DateTime.UtcNow - StartTime).TotalSeconds);
if (BuildDuration > 60.0f && Shared.GetTotalSize() > 0)
{
var MBSec = (((float)(Shared.GetTotalSize())) / (1024.0f * 1024.0f)) / BuildDuration;
Log("Read from shared temp storage at {0} MB/s {1}B {2}s", MBSec, Shared.GetTotalSize(), BuildDuration);
}
return(DestFiles);
}
public static void DeleteSharedTempStorageManifests(CommandEnvironment Env, string StorageBlockName, string GameFolder = "")
{
DeleteDirectory(true, SharedTempStorageDirectory(StorageBlockName, GameFolder));
}
/// <summary>
/// Runs UAT recursively
/// </summary>
/// <param name="Env">Environment to use.</param>
/// <param name="CommandLine">Commandline to pass on to the executable</param>
public static string RunUAT(CommandEnvironment Env, string CommandLine)
{
// this doesn't do much, but it does need to make sure log folders are reasonable and don't collide
string BaseLogSubdir = "Recur";
if (!String.IsNullOrEmpty(CommandLine))
{
int Space = CommandLine.IndexOf(" ");
if (Space > 0)
{
BaseLogSubdir = BaseLogSubdir + "_" + CommandLine.Substring(0, Space);
}
else
{
BaseLogSubdir = BaseLogSubdir + "_" + CommandLine;
}
}
int Index = 0;
BaseLogSubdir = BaseLogSubdir.Trim();
string DirOnlyName = BaseLogSubdir;
string LogSubdir = CombinePaths(CmdEnv.LogFolder, DirOnlyName, "");
while (true)
{
var ExistingFiles = FindFiles(DirOnlyName + "*", false, CmdEnv.LogFolder);
if (ExistingFiles.Length == 0)
{
break;
}
Index++;
if (Index == 1000)
{
throw new AutomationException("Couldn't seem to create a log subdir {0}", LogSubdir);
}
DirOnlyName = String.Format("{0}_{1}_", BaseLogSubdir, Index);
LogSubdir = CombinePaths(CmdEnv.LogFolder, DirOnlyName, "");
}
string LogFile = CombinePaths(CmdEnv.LogFolder, DirOnlyName + ".log");
Log("Recursive UAT Run, in log folder {0}, main log file {1}", LogSubdir, LogFile);
CreateDirectory(LogSubdir);
CommandLine = CommandLine + " -NoCompile";
string App = CmdEnv.UATExe;
Log("Running {0} {1}", App, CommandLine);
var OSEnv = new Dictionary <string, string>();
OSEnv.Add(AutomationTool.EnvVarNames.LogFolder, LogSubdir);
OSEnv.Add("uebp_UATMutexNoWait", "1");
if (!IsBuildMachine)
{
OSEnv.Add(AutomationTool.EnvVarNames.LocalRoot, ""); // if we don't clear this out, it will think it is a build machine; it will rederive everything
}
ProcessResult Result = Run(App, CommandLine, null, ERunOptions.Default, OSEnv);
if (Result.Output.Length > 0)
{
WriteToFile(LogFile, Result.Output);
}
else
{
WriteToFile(LogFile, "[None!, no output produced]");
}
Log("Flattening log folder {0}", LogSubdir);
var Files = FindFiles("*", true, LogSubdir);
string MyLogFolder = CombinePaths(CmdEnv.LogFolder, "");
foreach (var ThisFile in Files)
{
if (!ThisFile.StartsWith(MyLogFolder, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Can't rebase {0} because it doesn't start with {1}", ThisFile, MyLogFolder);
}
string NewFilename = ThisFile.Substring(MyLogFolder.Length).Replace("/", "_").Replace("\\", "_");
NewFilename = CombinePaths(CmdEnv.LogFolder, NewFilename);
if (FileExists_NoExceptions(NewFilename))
{
throw new AutomationException("Destination log file already exists? {0}", NewFilename);
}
CopyFile(ThisFile, NewFilename);
if (!FileExists_NoExceptions(NewFilename))
{
throw new AutomationException("Destination log file could not be copied {0}", NewFilename);
}
DeleteFile_NoExceptions(ThisFile);
}
DeleteDirectory_NoExceptions(LogSubdir);
if (Result != 0)
{
throw new AutomationException(String.Format("Recursive UAT Command failed (Result:{3}): {0} {1}. See logfile for details: '{2}' ",
App, CommandLine, Path.GetFileName(LogFile), Result.ExitCode));
}
return(LogFile);
}
public static void DeleteLocalTempStorage(CommandEnvironment Env, string StorageBlockName, bool bQuiet = false)
{
var LocalManifest = LocalTempStorageManifestFilename(Env, StorageBlockName);
DeleteFile(bQuiet, LocalManifest);
}
public static string SharedTempStorageManifestFilename(CommandEnvironment Env, string StorageBlockName, string GameFolder)
{
return(CombinePaths(SharedTempStorageDirectory(StorageBlockName, GameFolder), StorageBlockName + ".TempManifest"));
}
public static void RetrieveFromPermanentStorage(CommandEnvironment Env, string NetworkRoot, string Build, string Platform)
{
var LocalManifest = LocalTempStorageManifestFilename(Env, Build);
if (!FileExists_NoExceptions(LocalManifest))
{
string BaseFolder = CombinePaths(Env.LocalRoot, "Rocket", "TempInst", Platform);
string Root = RootSharedTempStorageDirectory();
string SrcFolder = CombinePaths(Root, NetworkRoot, Build, Platform);
var SourceFiles = FindFiles_NoExceptions("*", true, SrcFolder);
foreach (var File in SourceFiles)
{
var DestFile = File.Replace(SrcFolder, "");
DestFile = CombinePaths(BaseFolder, DestFile);
CopyFile_NoExceptions(File, DestFile);
}
if(!DirectoryExists(Path.GetDirectoryName(LocalManifest)))
{
CreateDirectory(Path.GetDirectoryName(LocalManifest));
}
XmlDocument Local = new XmlDocument();
XmlElement RootElement = Local.CreateElement("permstorage");
Local.AppendChild(RootElement);
Local.Save(LocalManifest);
}
}
internal P4Environment(P4Connection Connection, CommandEnvironment CmdEnv)
{
InitEnvironment(Connection, CmdEnv);
}
public static void DeleteLocalTempStorage(CommandEnvironment Env, string StorageBlockName, bool bQuiet = false)
{
var LocalManifest = LocalTempStorageManifestFilename(Env, StorageBlockName);
DeleteFile(bQuiet, LocalManifest);
}
public static string LocalTempStorageManifestFilename(CommandEnvironment Env, string StorageBlockName)
{
return(CombinePaths(LocalTempStorageManifestDirectory(Env), StorageBlockName + ".TempManifest"));
}
public static void DeleteSharedTempStorageManifests(CommandEnvironment Env, string StorageBlockName, string GameFolder = "")
{
DeleteDirectory(true, SharedTempStorageDirectory(StorageBlockName, GameFolder));
}
public static void DeleteLocalTempStorageManifests(CommandEnvironment Env)
{
DeleteDirectory(true, LocalTempStorageManifestDirectory(Env));
}
/// <summary>
/// Runs external program and writes the output to a logfile.
/// </summary>
/// <param name="Env">Environment to use.</param>
/// <param name="App">Executable to run</param>
/// <param name="CommandLine">Commandline to pass on to the executable</param>
/// <param name="LogName">Name of the logfile ( if null, executable name is used )</param>
/// <returns>Whether the program executed successfully or not.</returns>
public static string RunAndLog(CommandEnvironment Env, string App, string CommandLine, out int SuccessCode, string LogName = null, Dictionary<string, string> EnvVars = null)
{
return RunAndLog(App, CommandLine, out SuccessCode, GetRunAndLogLogName(Env, App, LogName), EnvVars);
}
public static TempStorageManifest SaveLocalTempStorageManifest(CommandEnvironment Env, string BaseFolder, string StorageBlockName, List<string> Files)
{
return SaveTempStorageManifest(BaseFolder, LocalTempStorageManifestFilename(Env, StorageBlockName), Files);
}
/// <summary>
/// Initializes the environment. Tries to autodetect all source control settings.
/// </summary>
/// <param name="CompilationEnv">Compilation environment</param>
protected override void InitEnvironment(P4Connection Connection, CommandEnvironment CmdEnv)
{
var HostName = Environment.MachineName.ToLower();
var P4PortEnv = DetectP4Port();
var UserName = CommandUtils.GetEnvVar(EnvVarNames.User);
if (String.IsNullOrEmpty(UserName))
{
UserName = DetectUserName(Connection);
}
var CommandLineClient = CommandUtils.GetEnvVar(EnvVarNames.Client);
P4ClientInfo ThisClient = null;
if (String.IsNullOrEmpty(CommandLineClient) == false)
{
ThisClient = Connection.GetClientInfo(CommandLineClient);
if (ThisClient == null)
{
throw new AutomationException("Unable to find client {0}", CommandLineClient);
}
if (String.Compare(ThisClient.Owner, UserName, true) != 0)
{
throw new AutomationException("Client specified with {0}={1} has a different owner then the detected user name (has: {2}, expected: {3})",
EnvVarNames.Client, CommandLineClient, ThisClient.Owner, UserName);
}
}
else
{
ThisClient = DetectClient(Connection, UserName, HostName, CmdEnv.UATExe);
}
Log.TraceInformation("Using user {0} clientspec {1} {2}", UserName, ThisClient.Name, ThisClient.RootPath);
Environment.SetEnvironmentVariable("P4CLIENT", ThisClient.Name);
string BuildRootPath;
string ClientRootPath;
DetectRootPaths(Connection, CmdEnv.LocalRoot, ThisClient, out BuildRootPath, out ClientRootPath);
CommandUtils.ConditionallySetEnvVar(EnvVarNames.P4Port, P4PortEnv);
CommandUtils.ConditionallySetEnvVar(EnvVarNames.User, UserName);
CommandUtils.ConditionallySetEnvVar(EnvVarNames.Client, ThisClient.Name);
CommandUtils.ConditionallySetEnvVar(EnvVarNames.BuildRootP4, BuildRootPath);
CommandUtils.ConditionallySetEnvVar(EnvVarNames.ClientRoot, ClientRootPath);
var CLString = CommandUtils.GetEnvVar(EnvVarNames.Changelist, null);
if (String.IsNullOrEmpty(CLString) && CommandUtils.P4CLRequired)
{
CLString = DetectCurrentCL(Connection, ClientRootPath);
}
if (!String.IsNullOrEmpty(CLString))
{
CommandUtils.ConditionallySetEnvVar(EnvVarNames.Changelist, CLString);
}
var CodeCLString = CommandUtils.GetEnvVar(EnvVarNames.CodeChangelist, null);
if (String.IsNullOrEmpty(CodeCLString) && CommandUtils.P4CLRequired)
{
CodeCLString = DetectCurrentCodeCL(Connection, ClientRootPath);
}
if (!String.IsNullOrEmpty(CodeCLString))
{
CommandUtils.ConditionallySetEnvVar(EnvVarNames.CodeChangelist, CodeCLString);
}
CommandUtils.ConditionallySetEnvVar(EnvVarNames.LabelToSync, "");
CommandUtils.ConditionallySetEnvVar("P4USER", UserName);
CommandUtils.ConditionallySetEnvVar("P4CLIENT", ThisClient.Name);
var P4Password = Environment.GetEnvironmentVariable(EnvVarNames.P4Password);
if (!String.IsNullOrEmpty(P4Password))
{
CommandUtils.ConditionallySetEnvVar("P4PASSWD", P4Password);
}
SetBuildRootEscaped();
base.InitEnvironment(Connection, CmdEnv);
}
public static bool TempStorageExists(CommandEnvironment Env, string StorageBlockName, string GameFolder = "", bool bLocalOnly = false, bool bQuiet = false)
{
return(LocalTempStorageExists(Env, StorageBlockName, bQuiet) || (!bLocalOnly && SharedTempStorageExists(Env, StorageBlockName, GameFolder, bQuiet)));
}
public static string LocalTempStorageManifestDirectory(CommandEnvironment Env)
{
return CombinePaths(Env.LocalRoot, "Engine", "Saved", "GUBP");
}
public static TempStorageManifest SaveSharedTempStorageManifest(CommandEnvironment Env, string StorageBlockName, string GameFolder, List <string> Files)
{
return(SaveTempStorageManifest(SharedTempStorageDirectory(StorageBlockName, GameFolder), SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder), Files));
}
internal LocalP4Environment(P4Connection Connection, CommandEnvironment CmdEnv)
: base(Connection, CmdEnv)
{
}
/// <summary>
/// Builds a target using UBT. Automatically creates a logfile. When
/// no LogName is specified, the executable name is used as logfile base name.
/// </summary>
/// <param name="Env">BuildEnvironment to use.</param>
/// <param name="Project">Unreal project to build (optional)</param>
/// <param name="Target">Target to build.</param>
/// <param name="Platform">Platform to build for.</param>
/// <param name="Config">Configuration to build.</param>
/// <param name="AdditionalArgs">Additional arguments to pass on to UBT.</param>
/// <param name="LogName">Optional logifle name.</param>
public static void RunUBT(CommandEnvironment Env, string UBTExecutable, string Project, string Target, string Platform, string Config, string AdditionalArgs = "", string LogName = null, Dictionary <string, string> EnvVars = null)
{
RunUBT(Env, UBTExecutable, UBTCommandline(Project, Target, Platform, Config, AdditionalArgs), LogName, EnvVars);
}
public static List<string> FindTempStorageManifests(CommandEnvironment Env, string StorageBlockName, bool LocalOnly = false, bool SharedOnly = false, string GameFolder = "")
{
var Files = new List<string>();
var LocalFiles = LocalTempStorageManifestFilename(Env, StorageBlockName);
var LocalParent = Path.GetDirectoryName(LocalFiles);
var WildCard = Path.GetFileName(LocalFiles);
int IndexOfStar = WildCard.IndexOf("*");
if (IndexOfStar < 0 || WildCard.LastIndexOf("*") != IndexOfStar)
{
throw new AutomationException("Wildcard {0} either has no star or it has more than one.", WildCard);
}
string PreStarWildcard = WildCard.Substring(0, IndexOfStar);
string PostStarWildcard = Path.GetFileNameWithoutExtension(WildCard.Substring(IndexOfStar + 1));
if (!SharedOnly && DirectoryExists_NoExceptions(LocalParent))
{
foreach (var ThisFile in CommandUtils.FindFiles_NoExceptions(WildCard, true, LocalParent))
{
Log(" Found local file {0}", ThisFile);
int IndexOfWildcard = ThisFile.IndexOf(PreStarWildcard);
if (IndexOfWildcard < 0)
{
throw new AutomationException("File {0} didn't contain {1}.", ThisFile, PreStarWildcard);
}
int LastIndexOfWildcardTail = ThisFile.LastIndexOf(PostStarWildcard);
if (LastIndexOfWildcardTail < 0 || LastIndexOfWildcardTail < IndexOfWildcard + PreStarWildcard.Length)
{
throw new AutomationException("File {0} didn't contain {1} or it was before the prefix", ThisFile, PostStarWildcard);
}
string StarReplacement = ThisFile.Substring(IndexOfWildcard + PreStarWildcard.Length, LastIndexOfWildcardTail - IndexOfWildcard - PreStarWildcard.Length);
if (StarReplacement.Length < 1)
{
throw new AutomationException("Dir {0} didn't have any string to fit the star in the wildcard {1}", ThisFile, WildCard);
}
if (!Files.Contains(StarReplacement))
{
Files.Add(StarReplacement);
}
}
}
if (!LocalOnly)
{
var SharedFiles = SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder);
var SharedParent = Path.GetDirectoryName(Path.GetDirectoryName(SharedFiles));
if (DirectoryExists_NoExceptions(SharedParent))
{
string[] Dirs = null;
try
{
Dirs = Directory.GetDirectories(SharedParent, Path.GetFileNameWithoutExtension(SharedFiles), SearchOption.TopDirectoryOnly);
}
catch (Exception Ex)
{
Log("Unable to Find Directories in {0} with wildcard {1}", SharedParent, Path.GetFileNameWithoutExtension(SharedFiles));
Log(" Exception was {0}", LogUtils.FormatException(Ex));
}
if (Dirs != null)
{
foreach (var ThisSubDir in Dirs)
{
int IndexOfWildcard = ThisSubDir.IndexOf(PreStarWildcard);
if (IndexOfWildcard < 0)
{
throw new AutomationException("Dir {0} didn't contain {1}.", ThisSubDir, PreStarWildcard);
}
int LastIndexOfWildcardTail = ThisSubDir.LastIndexOf(PostStarWildcard);
if (LastIndexOfWildcardTail < 0 || LastIndexOfWildcardTail < IndexOfWildcard + PreStarWildcard.Length)
{
throw new AutomationException("Dir {0} didn't contain {1} or it was before the prefix", ThisSubDir, PostStarWildcard);
}
string StarReplacement = ThisSubDir.Substring(IndexOfWildcard + PreStarWildcard.Length, LastIndexOfWildcardTail - IndexOfWildcard - PreStarWildcard.Length);
if (StarReplacement.Length < 1)
{
throw new AutomationException("Dir {0} didn't have any string to fit the star in the wildcard {1}", ThisSubDir, WildCard);
}
// these are a bunch of false positives
if (StarReplacement.Contains("-"))
{
continue;
}
if (!Files.Contains(StarReplacement))
{
Files.Add(StarReplacement);
}
}
}
}
}
var OutFiles = new List<string>();
foreach (var StarReplacement in Files)
{
var NewBlock = StorageBlockName.Replace("*", StarReplacement);
if (TempStorageExists(Env, NewBlock, GameFolder, LocalOnly, true))
{
OutFiles.Add(StarReplacement);
}
}
return OutFiles;
}
/// <summary>
/// Runs UAT recursively
/// </summary>
/// <param name="Env">Environment to use.</param>
/// <param name="CommandLine">Commandline to pass on to the executable</param>
/// <param name="Identifier">Log prefix for output</param>
public static string RunUAT(CommandEnvironment Env, string CommandLine, string Identifier = null)
{
// We want to redirect the output from recursive UAT calls into our normal log folder, but prefix everything with a unique identifier. To do so, we set the EnvVarNames.LogFolder environment
// variable to a subfolder of it, then copy its contents into the main folder with a prefix after it's finished. Start by finding a base name we can use to identify the output of this run.
string BaseLogSubdir = "Recur";
if (!String.IsNullOrEmpty(CommandLine))
{
int Space = CommandLine.IndexOf(" ");
if (Space > 0)
{
BaseLogSubdir = BaseLogSubdir + "_" + CommandLine.Substring(0, Space);
}
else if (CommandLine.Contains("-profile"))
{
string PathToProfile = CommandLine.Substring(CommandLine.IndexOf('=') + 1);
BaseLogSubdir = BaseLogSubdir + "_" + (Path.GetFileNameWithoutExtension(PathToProfile));
}
else
{
BaseLogSubdir = BaseLogSubdir + "_" + CommandLine;
}
}
BaseLogSubdir = BaseLogSubdir.Trim();
// Check if there are already log files which start with this prefix, and try to uniquify it if until there aren't.
int Index = 0;
string DirOnlyName = BaseLogSubdir;
string LogSubdir = CombinePaths(CmdEnv.LogFolder, DirOnlyName, "");
while (true)
{
var ExistingFiles = FindFiles(DirOnlyName + "*", false, CmdEnv.LogFolder);
if (ExistingFiles.Length == 0)
{
break;
}
Index++;
if (Index == 1000)
{
throw new AutomationException("Couldn't seem to create a log subdir {0}", LogSubdir);
}
DirOnlyName = String.Format("{0}_{1}_", BaseLogSubdir, Index);
LogSubdir = CombinePaths(CmdEnv.LogFolder, DirOnlyName, "");
}
// Get the stdout log file for this run, and create the subdirectory for all the other log output
string LogFile = CombinePaths(CmdEnv.LogFolder, DirOnlyName + ".log");
LogVerbose("Recursive UAT Run, in log folder {0}, main log file {1}", LogSubdir, LogFile);
CreateDirectory(LogSubdir);
// Run UAT with the log folder redirected through the environment
string App = CmdEnv.UATExe;
Log("Running {0} {1}", App, CommandLine);
var OSEnv = new Dictionary <string, string>();
OSEnv.Add(EnvVarNames.LogFolder, LogSubdir);
OSEnv.Add(EnvVarNames.DisableStartupMutex, "1");
OSEnv.Add(EnvVarNames.IsChildInstance, "1");
if (!IsBuildMachine)
{
OSEnv.Add(AutomationTool.EnvVarNames.LocalRoot, ""); // if we don't clear this out, it will think it is a build machine; it will rederive everything
}
IProcessResult Result = Run(App, CommandLine, null, ERunOptions.Default, OSEnv, Identifier: Identifier);
if (Result.Output.Length > 0)
{
WriteToFile(LogFile, Result.Output);
}
else
{
WriteToFile(LogFile, "[None!, no output produced]");
}
// Copy everything into the main log folder, using the prefix we decided on earlier.
LogVerbose("Flattening log folder {0}", LogSubdir);
var Files = FindFiles("*", true, LogSubdir);
string MyLogFolder = CombinePaths(CmdEnv.LogFolder, "");
foreach (var ThisFile in Files)
{
if (!ThisFile.StartsWith(MyLogFolder, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Can't rebase {0} because it doesn't start with {1}", ThisFile, MyLogFolder);
}
string NewFilename = ThisFile.Substring(MyLogFolder.Length).Replace("/", "_").Replace("\\", "_");
NewFilename = CombinePaths(CmdEnv.LogFolder, NewFilename);
if (FileExists_NoExceptions(NewFilename))
{
throw new AutomationException("Destination log file already exists? {0}", NewFilename);
}
CopyFile(ThisFile, NewFilename);
if (!FileExists_NoExceptions(NewFilename))
{
throw new AutomationException("Destination log file could not be copied {0}", NewFilename);
}
DeleteFile_NoExceptions(ThisFile);
}
DeleteDirectory_NoExceptions(LogSubdir);
if (Result.ExitCode != 0)
{
throw new CommandFailedException(String.Format("Recursive UAT Command failed (Result:{3}): {0} {1}. See logfile for details: '{2}' ",
App, CommandLine, Path.GetFileName(LogFile), Result.ExitCode));
}
return(LogFile);
}
public static string LocalTempStorageManifestFilename(CommandEnvironment Env, string StorageBlockName)
{
return CombinePaths(LocalTempStorageManifestDirectory(Env), StorageBlockName + ".TempManifest");
}
protected virtual void InitEnvironment(P4Connection Connection, CommandEnvironment CmdEnv)
{
//
// P4 Environment
//
P4Port = CommandUtils.GetEnvVar(EnvVarNames.P4Port);
ClientRoot = CommandUtils.GetEnvVar(EnvVarNames.ClientRoot);
User = CommandUtils.GetEnvVar(EnvVarNames.User);
ChangelistStringInternal = CommandUtils.GetEnvVar(EnvVarNames.Changelist, null);
Client = CommandUtils.GetEnvVar(EnvVarNames.Client);
BuildRootP4 = CommandUtils.GetEnvVar(EnvVarNames.BuildRootP4);
if (BuildRootP4.EndsWith("/", StringComparison.InvariantCultureIgnoreCase) || BuildRootP4.EndsWith("\\", StringComparison.InvariantCultureIgnoreCase))
{
// We expect the build root to not end with a path separator
BuildRootP4 = BuildRootP4.Substring(0, BuildRootP4.Length - 1);
CommandUtils.SetEnvVar(EnvVarNames.BuildRootP4, BuildRootP4);
}
BuildRootEscaped = CommandUtils.GetEnvVar(EnvVarNames.BuildRootEscaped);
LabelToSync = CommandUtils.GetEnvVar(EnvVarNames.LabelToSync);
if (((CommandUtils.P4Enabled || CommandUtils.IsBuildMachine) && (ClientRoot == String.Empty || User == String.Empty ||
(String.IsNullOrEmpty(ChangelistStringInternal) && CommandUtils.IsBuildMachine) || Client == String.Empty || BuildRootP4 == String.Empty)))
{
Log.TraceInformation("P4Enabled={0}", CommandUtils.P4Enabled);
Log.TraceInformation("ClientRoot={0}", ClientRoot);
Log.TraceInformation("User={0}", User);
Log.TraceInformation("ChangelistString={0}", ChangelistStringInternal);
Log.TraceInformation("Client={0}", Client);
Log.TraceInformation("BuildRootP4={0}", BuildRootP4);
throw new AutomationException("BUILD FAILED Perforce Environment is not set up correctly. Please check your environment variables.");
}
LabelPrefix = BuildRootP4 + "/";
if (CommandUtils.P4Enabled)
{
if (CommandUtils.IsBuildMachine || ChangelistStringInternal != null)
{
// We may not always need the changelist number if we're not a build machine.
// In local runs, changelist initialization can be really slow!
VerifyChangelistStringAndSetChangelistNumber();
}
// Setup branch name
string DepotSuffix = "//depot/";
if (BuildRootP4.StartsWith(DepotSuffix))
{
BranchName = BuildRootP4.Substring(DepotSuffix.Length);
}
else
{
throw new AutomationException("Needs update to work with a stream");
}
if (String.IsNullOrWhiteSpace(BranchName))
{
throw new AutomationException("BUILD FAILED no branch name.");
}
}
LogSettings();
}
public static string SharedTempStorageManifestFilename(CommandEnvironment Env, string StorageBlockName, string GameFolder)
{
return CombinePaths(SharedTempStorageDirectory(StorageBlockName, GameFolder), StorageBlockName + ".TempManifest");
}
internal LocalP4Environment(P4Connection Connection, CommandEnvironment CmdEnv)
: base(Connection, CmdEnv)
{
}
public static void DeleteLocalTempStorageManifests(CommandEnvironment Env)
{
DeleteDirectory(true, LocalTempStorageManifestDirectory(Env));
}
/// <summary>
/// Initializes the environment. Tries to autodetect all source control settings.
/// </summary>
/// <param name="CompilationEnv">Compilation environment</param>
protected override void InitEnvironment(P4Connection Connection, CommandEnvironment CmdEnv)
{
var HostName = Environment.MachineName.ToLower();
var P4PortEnv = DetectP4Port();
var UserName = CommandUtils.GetEnvVar(EnvVarNames.User);
if (String.IsNullOrEmpty(UserName))
{
UserName = DetectUserName(Connection);
}
var CommandLineClient = CommandUtils.GetEnvVar(EnvVarNames.Client);
P4ClientInfo ThisClient = null;
if (String.IsNullOrEmpty(CommandLineClient) == false)
{
ThisClient = Connection.GetClientInfo(CommandLineClient);
if (ThisClient == null)
{
throw new AutomationException("Unable to find client {0}", CommandLineClient);
}
if (String.Compare(ThisClient.Owner, UserName, true) != 0)
{
throw new AutomationException("Client specified with {0}={1} has a different owner then the detected user name (has: {2}, expected: {3})",
EnvVarNames.Client, CommandLineClient, ThisClient.Owner, UserName);
}
}
else
{
ThisClient = DetectClient(Connection, UserName, HostName, CmdEnv.UATExe);
}
Log.TraceInformation("Using user {0} clientspec {1} {2}", UserName, ThisClient.Name, ThisClient.RootPath);
Environment.SetEnvironmentVariable("P4CLIENT", ThisClient.Name);
string BuildRootPath;
string ClientRootPath;
DetectRootPaths(Connection, CmdEnv.LocalRoot, ThisClient, out BuildRootPath, out ClientRootPath);
CommandUtils.ConditionallySetEnvVar(EnvVarNames.P4Port, P4PortEnv);
CommandUtils.ConditionallySetEnvVar(EnvVarNames.User, UserName);
CommandUtils.ConditionallySetEnvVar(EnvVarNames.Client, ThisClient.Name);
CommandUtils.ConditionallySetEnvVar(EnvVarNames.BuildRootP4, BuildRootPath);
CommandUtils.ConditionallySetEnvVar(EnvVarNames.ClientRoot, ClientRootPath);
var CLString = CommandUtils.GetEnvVar(EnvVarNames.Changelist, null);
if (String.IsNullOrEmpty(CLString) && CommandUtils.P4CLRequired)
{
CLString = DetectCurrentCL(Connection, ClientRootPath);
}
if (!String.IsNullOrEmpty(CLString))
{
CommandUtils.ConditionallySetEnvVar(EnvVarNames.Changelist, CLString);
}
var CodeCLString = CommandUtils.GetEnvVar(EnvVarNames.CodeChangelist, null);
if (String.IsNullOrEmpty(CodeCLString) && CommandUtils.P4CLRequired)
{
CodeCLString = DetectCurrentCodeCL(Connection, ClientRootPath);
}
if (!String.IsNullOrEmpty(CodeCLString))
{
CommandUtils.ConditionallySetEnvVar(EnvVarNames.CodeChangelist, CodeCLString);
}
CommandUtils.ConditionallySetEnvVar("P4USER", UserName);
CommandUtils.ConditionallySetEnvVar("P4CLIENT", ThisClient.Name);
var P4Password = Environment.GetEnvironmentVariable(EnvVarNames.P4Password);
if (!String.IsNullOrEmpty(P4Password))
{
CommandUtils.ConditionallySetEnvVar("P4PASSWD", P4Password);
}
SetBuildRootEscaped();
base.InitEnvironment(Connection, CmdEnv);
}
public static bool LocalTempStorageExists(CommandEnvironment Env, string StorageBlockName, bool bQuiet = false)
{
var LocalManifest = LocalTempStorageManifestFilename(Env, StorageBlockName);
if (FileExists_NoExceptions(bQuiet, LocalManifest))
{
return true;
}
return false;
}
internal P4Environment(P4Connection Connection, CommandEnvironment CmdEnv)
{
InitEnvironment(Connection, CmdEnv);
}
public static bool SharedTempStorageExists(CommandEnvironment Env, string StorageBlockName, string GameFolder = "", bool bQuiet = false)
{
var SharedManifest = SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder);
if (FileExists_NoExceptions(bQuiet, SharedManifest))
{
return true;
}
return false;
}
/// <summary>
/// Builds a target using UBT. Automatically creates a logfile. When
/// no LogName is specified, the executable name is used as logfile base name.
/// </summary>
/// <param name="Env">BuildEnvironment to use.</param>
/// <param name="Project">Unreal project to build (optional)</param>
/// <param name="Target">Target to build.</param>
/// <param name="Platform">Platform to build for.</param>
/// <param name="Config">Configuration to build.</param>
/// <param name="AdditionalArgs">Additional arguments to pass on to UBT.</param>
/// <param name="LogName">Optional logifle name.</param>
public static void RunUBT(CommandEnvironment Env, string UBTExecutable, FileReference Project, string Target, UnrealTargetPlatform Platform, UnrealTargetConfiguration Config, string AdditionalArgs = "")
{
RunUBT(Env, UBTExecutable, UBTCommandline(Project, Target, Platform, Config, AdditionalArgs));
}
public static void StoreToTempStorage(CommandEnvironment Env, string StorageBlockName, List<string> Files, bool bLocalOnly = false, string GameFolder = "", string BaseFolder = "")
{
if (String.IsNullOrEmpty(BaseFolder))
{
BaseFolder = Env.LocalRoot;
}
BaseFolder = CombinePaths(BaseFolder, "/");
if (!BaseFolder.EndsWith("/") && !BaseFolder.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BaseFolder);
}
var Local = SaveLocalTempStorageManifest(Env, BaseFolder, StorageBlockName, Files);
if (!bLocalOnly)
{
var StartTime = DateTime.UtcNow;
var BlockPath = SharedTempStorageDirectory(StorageBlockName, GameFolder);
Log("Storing to {0}", BlockPath);
if (DirectoryExists_NoExceptions(BlockPath))
{
throw new AutomationException("Storage Block Already Exists! {0}", BlockPath);
}
CreateDirectory(true, BlockPath);
if (!DirectoryExists_NoExceptions(BlockPath))
{
throw new AutomationException("Storage Block Could Not Be Created! {0}", BlockPath);
}
var DestFiles = new List<string>();
if (ThreadsToCopyWith() < 2)
{
foreach (string InFilename in Files)
{
var Filename = CombinePaths(InFilename);
Robust_FileExists_NoExceptions(false, Filename, "Could not add {0} to manifest because it does not exist");
if (!Filename.StartsWith(BaseFolder, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BaseFolder);
}
var RelativeFile = Filename.Substring(BaseFolder.Length);
var DestFile = CombinePaths(BlockPath, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
throw new AutomationException("Dest file {0} already exists.", DestFile);
}
CopyFile(Filename, DestFile, true);
Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}");
DestFiles.Add(DestFile);
}
}
else
{
var SrcFiles = new List<string>();
foreach (string InFilename in Files)
{
var Filename = CombinePaths(InFilename);
Robust_FileExists_NoExceptions(false, Filename, "Could not add {0} to manifest because it does not exist");
if (!Filename.StartsWith(BaseFolder, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BaseFolder);
}
var RelativeFile = Filename.Substring(BaseFolder.Length);
var DestFile = CombinePaths(BlockPath, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
throw new AutomationException("Dest file {0} already exists.", DestFile);
}
SrcFiles.Add(Filename);
DestFiles.Add(DestFile);
}
ThreadedCopyFiles(SrcFiles.ToArray(), DestFiles.ToArray(), ThreadsToCopyWith());
foreach (string DestFile in DestFiles)
{
Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}");
}
}
var Shared = SaveSharedTempStorageManifest(Env, StorageBlockName, GameFolder, DestFiles);
if (!Local.Compare(Shared))
{
// we will rename this so it can't be used, but leave it around for inspection
RenameFile_NoExceptions(SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder), SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder) + ".broken");
throw new AutomationException("Shared and Local manifest mismatch.");
}
float BuildDuration = (float)((DateTime.UtcNow - StartTime).TotalSeconds);
if (BuildDuration > 60.0f && Shared.GetTotalSize() > 0)
{
var MBSec = (((float)(Shared.GetTotalSize())) / (1024.0f * 1024.0f)) / BuildDuration;
Log("Wrote to shared temp storage at {0} MB/s {1}B {2}s", MBSec, Shared.GetTotalSize(), BuildDuration);
}
}
}
/// <summary>
/// Runs external program and writes the output to a logfile.
/// </summary>
/// <param name="Env">Environment to use.</param>
/// <param name="App">Executable to run</param>
/// <param name="CommandLine">Commandline to pass on to the executable</param>
/// <param name="LogName">Name of the logfile ( if null, executable name is used )</param>
/// <param name="Input">Optional Input for the program (will be provided as stdin)</param>
/// <param name="Options">Defines the options how to run. See ERunOptions.</param>
/// <param name="FilterCallback">Callback to filter log spew before output.</param>
public static void RunAndLog(CommandEnvironment Env, string App, string CommandLine, string LogName = null, int MaxSuccessCode = 0, string Input = null, ERunOptions Options = ERunOptions.Default, Dictionary <string, string> EnvVars = null, ProcessResult.SpewFilterCallbackType SpewFilterCallback = null)
{
RunAndLog(App, CommandLine, GetRunAndLogOnlyName(Env, App, LogName), MaxSuccessCode, Input, Options, EnvVars, SpewFilterCallback);
}
public static List<string> RetrieveFromTempStorage(CommandEnvironment Env, string StorageBlockName, out bool WasLocal, string GameFolder = "", string BaseFolder = "")
{
if (String.IsNullOrEmpty(BaseFolder))
{
BaseFolder = Env.LocalRoot;
}
BaseFolder = CombinePaths(BaseFolder, "/");
if (!BaseFolder.EndsWith("/") && !BaseFolder.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BaseFolder);
}
var Files = new List<string>();
var LocalManifest = LocalTempStorageManifestFilename(Env, StorageBlockName);
if (FileExists_NoExceptions(LocalManifest))
{
Log("Found local manifest {0}", LocalManifest);
var Local = new TempStorageManifest();
Local.Load(LocalManifest);
Files = Local.GetFiles(BaseFolder);
var LocalTest = new TempStorageManifest();
LocalTest.Create(Files, BaseFolder);
if (!Local.Compare(LocalTest))
{
throw new AutomationException("Local files in manifest {0} were tampered with.", LocalManifest);
}
WasLocal = true;
return Files;
}
WasLocal = false;
var StartTime = DateTime.UtcNow;
var BlockPath = CombinePaths(SharedTempStorageDirectory(StorageBlockName, GameFolder), "/");
if (!BlockPath.EndsWith("/") && !BlockPath.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BlockPath);
}
Log("Attempting to retrieve from {0}", BlockPath);
if (!DirectoryExists_NoExceptions(BlockPath))
{
throw new AutomationException("Storage Block Does Not Exists! {0}", BlockPath);
}
var SharedManifest = SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder);
Robust_FileExists_NoExceptions(SharedManifest, "Storage Block Manifest Does Not Exists! {0}");
var Shared = new TempStorageManifest();
Shared.Load(SharedManifest);
var SharedFiles = Shared.GetFiles(BlockPath);
var DestFiles = new List<string>();
if (ThreadsToCopyWith() < 2)
{
foreach (string InFilename in SharedFiles)
{
var Filename = CombinePaths(InFilename);
Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist");
if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath);
}
var RelativeFile = Filename.Substring(BlockPath.Length);
var DestFile = CombinePaths(BaseFolder, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
Log("Dest file {0} already exists, deleting and overwriting", DestFile);
DeleteFile(DestFile);
}
CopyFile(Filename, DestFile, true);
Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}");
if (UnrealBuildTool.Utils.IsRunningOnMono)
{
FixUnixFilePermissions(DestFile);
}
FileInfo Info = new FileInfo(DestFile);
DestFiles.Add(Info.FullName);
}
}
else
{
var SrcFiles = new List<string>();
foreach (string InFilename in SharedFiles)
{
var Filename = CombinePaths(InFilename);
//Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist");
if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath);
}
var RelativeFile = Filename.Substring(BlockPath.Length);
var DestFile = CombinePaths(BaseFolder, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
Log("Dest file {0} already exists, deleting and overwriting", DestFile);
DeleteFile(DestFile);
}
SrcFiles.Add(Filename);
DestFiles.Add(DestFile);
}
ThreadedCopyFiles(SrcFiles.ToArray(), DestFiles.ToArray(), ThreadsToCopyWith());
var NewDestFiles = new List<string>();
foreach (string DestFile in DestFiles)
{
Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}");
if (UnrealBuildTool.Utils.IsRunningOnMono)
{
FixUnixFilePermissions(DestFile);
}
FileInfo Info = new FileInfo(DestFile);
NewDestFiles.Add(Info.FullName);
}
DestFiles = NewDestFiles;
}
var NewLocal = SaveLocalTempStorageManifest(Env, BaseFolder, StorageBlockName, DestFiles);
if (!NewLocal.Compare(Shared))
{
// we will rename this so it can't be used, but leave it around for inspection
RenameFile_NoExceptions(LocalManifest, LocalManifest + ".broken");
throw new AutomationException("Shared and Local manifest mismatch.");
}
float BuildDuration = (float)((DateTime.UtcNow - StartTime).TotalSeconds);
if (BuildDuration > 60.0f && Shared.GetTotalSize() > 0)
{
var MBSec = (((float)(Shared.GetTotalSize())) / (1024.0f * 1024.0f)) / BuildDuration;
Log("Read from shared temp storage at {0} MB/s {1}B {2}s", MBSec, Shared.GetTotalSize(), BuildDuration);
}
return DestFiles;
}
/// <summary>
/// Runs external program and writes the output to a logfile.
/// </summary>
/// <param name="Env">Environment to use.</param>
/// <param name="App">Executable to run</param>
/// <param name="CommandLine">Commandline to pass on to the executable</param>
/// <param name="LogName">Name of the logfile ( if null, executable name is used )</param>
/// <param name="FilterCallback">Callback to filter log spew before output.</param>
/// <returns>Whether the program executed successfully or not.</returns>
public static string RunAndLog(CommandEnvironment Env, string App, string CommandLine, out int SuccessCode, string LogName = null, Dictionary <string, string> EnvVars = null, ProcessResult.SpewFilterCallbackType SpewFilterCallback = null)
{
return(RunAndLog(App, CommandLine, out SuccessCode, GetRunAndLogOnlyName(Env, App, LogName), EnvVars, SpewFilterCallback));
}