/// <summary>
/// Runs a commandlet using Engine/Binaries/Win64/UE4Editor-Cmd.exe.
/// </summary>
/// <param name="ProjectName">Project name.</param>
/// <param name="UE4Exe">The name of the UE4 Editor executable to use.</param>
/// <param name="Commandlet">Commandlet name.</param>
/// <param name="Parameters">Command line parameters (without -run=)</param>
public static void RunCommandlet(string ProjectName, string UE4Exe, string Commandlet, string Parameters = null)
{
Log("Running UE4Editor {0} for project {1}", Commandlet, ProjectName);
var CWD = Path.GetDirectoryName(UE4Exe);
string EditorExe = UE4Exe;
if (String.IsNullOrEmpty(CWD))
{
EditorExe = HostPlatform.Current.GetUE4ExePath(UE4Exe);
CWD = CombinePaths(CmdEnv.LocalRoot, HostPlatform.Current.RelativeBinariesFolder);
}
PushDir(CWD);
string LocalLogFile = LogUtils.GetUniqueLogName(CombinePaths(CmdEnv.EngineSavedFolder, Commandlet));
Log("Commandlet log file is {0}", LocalLogFile);
string Args = String.Format(
"{0} -run={1} {2} -abslog={3} -stdout -FORCELOGFLUSH -CrashForUAT -unattended -AllowStdOutLogVerbosity {4}",
(ProjectName == null) ? "" : CommandUtils.MakePathSafeToUseWithCommandLine(ProjectName),
Commandlet,
String.IsNullOrEmpty(Parameters) ? "" : Parameters,
CommandUtils.MakePathSafeToUseWithCommandLine(LocalLogFile),
IsBuildMachine ? "-buildmachine" : ""
);
ERunOptions Opts = ERunOptions.Default;
if (GlobalCommandLine.UTF8Output)
{
Args += " -UTF8Output";
Opts |= ERunOptions.UTF8Output;
}
var RunResult = Run(EditorExe, Args, Options: Opts);
PopDir();
// Copy the local commandlet log to the destination folder.
string DestLogFile = LogUtils.GetUniqueLogName(CombinePaths(CmdEnv.LogFolder, Commandlet));
if (!CommandUtils.CopyFile_NoExceptions(LocalLogFile, DestLogFile))
{
CommandUtils.LogWarning("Commandlet {0} failed to copy the local log file from {1} to {2}. The log file will be lost.", Commandlet, LocalLogFile, DestLogFile);
}
// Whether it was copied correctly or not, delete the local log as it was only a temporary file.
CommandUtils.DeleteFile_NoExceptions(LocalLogFile);
if (RunResult.ExitCode != 0)
{
throw new AutomationException("BUILD FAILED: Failed while running {0} for {1}; see log {2}", Commandlet, ProjectName, DestLogFile);
}
}
public static void Robust_CopyFile(string InputFileName, string OutputFileName)
{
if (OutputFileName.StartsWith("/Volumes/"))
{
int Retry = 0;
int NumRetries = 60;
bool bCopied = false;
while (!bCopied && Retry < NumRetries)
{
if (Retry > 0)
{
//@todo: These retries should be reported so we can track how often they are occurring.
CommandUtils.LogInformation("*** Mac temp storage retry {0}", OutputFileName);
System.Threading.Thread.Sleep(1000);
}
bCopied = CommandUtils.CopyFile_NoExceptions(InputFileName, OutputFileName, true);
Retry++;
}
}
else
{
CommandUtils.CopyFile(InputFileName, OutputFileName);
}
}
/// <summary>
/// Runs a commandlet using Engine/Binaries/Win64/UE4Editor-Cmd.exe.
/// </summary>
/// <param name="ProjectFile">Project name.</param>
/// <param name="UE4Exe">The name of the UE4 Editor executable to use.</param>
/// <param name="Commandlet">Commandlet name.</param>
/// <param name="Parameters">Command line parameters (without -run=)</param>
public static void RunCommandlet(FileReference ProjectName, string UE4Exe, string Commandlet, string Parameters = null)
{
Log("Running UE4Editor {0} for project {1}", Commandlet, ProjectName);
var CWD = Path.GetDirectoryName(UE4Exe);
string EditorExe = UE4Exe;
if (String.IsNullOrEmpty(CWD))
{
EditorExe = HostPlatform.Current.GetUE4ExePath(UE4Exe);
CWD = CombinePaths(CmdEnv.LocalRoot, HostPlatform.Current.RelativeBinariesFolder);
}
PushDir(CWD);
DateTime StartTime = DateTime.UtcNow;
string LocalLogFile = LogUtils.GetUniqueLogName(CombinePaths(CmdEnv.EngineSavedFolder, Commandlet));
Log("Commandlet log file is {0}", LocalLogFile);
string Args = String.Format(
"{0} -run={1} {2} -abslog={3} -stdout -FORCELOGFLUSH -CrashForUAT -unattended {5}{4}",
(ProjectName == null) ? "" : CommandUtils.MakePathSafeToUseWithCommandLine(ProjectName.FullName),
Commandlet,
String.IsNullOrEmpty(Parameters) ? "" : Parameters,
CommandUtils.MakePathSafeToUseWithCommandLine(LocalLogFile),
IsBuildMachine ? "-buildmachine" : "",
(GlobalCommandLine.Verbose || GlobalCommandLine.AllowStdOutLogVerbosity) ? "-AllowStdOutLogVerbosity " : ""
);
ERunOptions Opts = ERunOptions.Default;
if (GlobalCommandLine.UTF8Output)
{
Args += " -UTF8Output";
Opts |= ERunOptions.UTF8Output;
}
var RunResult = Run(EditorExe, Args, Options: Opts);
PopDir();
// Draw attention to signal exit codes on Posix systems, rather than just printing the exit code
if (RunResult.ExitCode > 128 && RunResult.ExitCode < 128 + 32)
{
if (RunResult.ExitCode == 139)
{
CommandUtils.LogError("Editor terminated abnormally due to a segmentation fault");
}
else
{
CommandUtils.LogError("Editor terminated abnormally with signal {0}", RunResult.ExitCode - 128);
}
}
// If we're running on a Mac, dump all the *.crash files that were generated while the editor was running.
if (HostPlatform.Current.HostEditorPlatform == UnrealTargetPlatform.Mac)
{
// If the exit code indicates the main process crashed, introduce a small delay because the crash report is written asynchronously.
// If we exited normally, still check without waiting in case SCW or some other child process crashed.
if (RunResult.ExitCode > 128)
{
CommandUtils.Log("Pausing before checking for crash logs...");
Thread.Sleep(10 * 1000);
}
// Create a list of directories containing crash logs, and add the system log folder
List <string> CrashDirs = new List <string>();
CrashDirs.Add("/Library/Logs/DiagnosticReports");
// Add the user's log directory too
string HomeDir = Environment.GetEnvironmentVariable("HOME");
if (!String.IsNullOrEmpty(HomeDir))
{
CrashDirs.Add(Path.Combine(HomeDir, "Library/Logs/DiagnosticReports"));
}
// Check each directory for crash logs
List <FileInfo> CrashFileInfos = new List <FileInfo>();
foreach (string CrashDir in CrashDirs)
{
DirectoryInfo CrashDirInfo = new DirectoryInfo(CrashDir);
if (CrashDirInfo.Exists)
{
CrashFileInfos.AddRange(CrashDirInfo.EnumerateFiles("*.crash", SearchOption.TopDirectoryOnly).Where(x => x.LastWriteTimeUtc >= StartTime));
}
}
// Dump them all to the log
foreach (FileInfo CrashFileInfo in CrashFileInfos)
{
// snmpd seems to often crash (suspect due to it being starved of CPU cycles during cooks)
if (!CrashFileInfo.Name.StartsWith("snmpd_"))
{
CommandUtils.Log("Found crash log - {0}", CrashFileInfo.FullName);
try
{
string[] Lines = File.ReadAllLines(CrashFileInfo.FullName);
foreach (string Line in Lines)
{
CommandUtils.Log("Crash: {0}", Line);
}
}
catch (Exception Ex)
{
CommandUtils.LogWarning("Failed to read file ({0})", Ex.Message);
}
}
}
}
// Copy the local commandlet log to the destination folder.
string DestLogFile = LogUtils.GetUniqueLogName(CombinePaths(CmdEnv.LogFolder, Commandlet));
if (!CommandUtils.CopyFile_NoExceptions(LocalLogFile, DestLogFile))
{
CommandUtils.LogWarning("Commandlet {0} failed to copy the local log file from {1} to {2}. The log file will be lost.", Commandlet, LocalLogFile, DestLogFile);
}
string ProjectStatsDirectory = CombinePaths((ProjectName == null)? CombinePaths(CmdEnv.LocalRoot, "Engine") : Path.GetDirectoryName(ProjectName.FullName), "Saved", "Stats");
if (Directory.Exists(ProjectStatsDirectory))
{
string DestCookerStats = CmdEnv.LogFolder;
foreach (var StatsFile in Directory.EnumerateFiles(ProjectStatsDirectory, "*.csv"))
{
if (!CommandUtils.CopyFile_NoExceptions(StatsFile, CombinePaths(DestCookerStats, Path.GetFileName(StatsFile))))
{
CommandUtils.LogWarning("Commandlet {0} failed to copy the local log file from {1} to {2}. The log file will be lost.", Commandlet, StatsFile, CombinePaths(DestCookerStats, Path.GetFileName(StatsFile)));
}
}
}
// else
// {
// CommandUtils.LogWarning("Failed to find directory {0} will not save stats", ProjectStatsDirectory);
// }
// Whether it was copied correctly or not, delete the local log as it was only a temporary file.
CommandUtils.DeleteFile_NoExceptions(LocalLogFile);
if (RunResult.ExitCode != 0)
{
throw new AutomationException(DestLogFile, RunResult.ExitCode, "BUILD FAILED: Failed while running {0} for {1}; see log {2}", Commandlet, ProjectName, DestLogFile);
}
}
/// <summary>
/// Runs a commandlet using Engine/Binaries/Win64/UE4Editor-Cmd.exe.
/// </summary>
/// <param name="ProjectFile">Project name.</param>
/// <param name="UE4Exe">The name of the UE4 Editor executable to use.</param>
/// <param name="Commandlet">Commandlet name.</param>
/// <param name="Parameters">Command line parameters (without -run=)</param>
/// <param name="DestLogFile">Log file after completion</param>
public static void RunCommandlet(FileReference ProjectName, string UE4Exe, string Commandlet, string Parameters, out string DestLogFile)
{
LogInformation("Running UE4Editor {0} for project {1}", Commandlet, ProjectName);
var CWD = Path.GetDirectoryName(UE4Exe);
string EditorExe = UE4Exe;
if (String.IsNullOrEmpty(CWD))
{
EditorExe = HostPlatform.Current.GetUE4ExePath(UE4Exe);
CWD = CombinePaths(CmdEnv.LocalRoot, HostPlatform.Current.RelativeBinariesFolder);
}
PushDir(CWD);
DateTime StartTime = DateTime.UtcNow;
string LocalLogFile = LogUtils.GetUniqueLogName(CombinePaths(CmdEnv.EngineSavedFolder, Commandlet));
LogInformation("Commandlet log file is {0}", LocalLogFile);
string Args = String.Format(
"{0} -run={1} {2} -abslog={3} -stdout -CrashForUAT -unattended -NoLogTimes {5}{4}",
(ProjectName == null) ? "" : CommandUtils.MakePathSafeToUseWithCommandLine(ProjectName.FullName),
Commandlet,
String.IsNullOrEmpty(Parameters) ? "" : Parameters,
CommandUtils.MakePathSafeToUseWithCommandLine(LocalLogFile),
IsBuildMachine ? "-buildmachine" : "",
(GlobalCommandLine.Verbose || GlobalCommandLine.AllowStdOutLogVerbosity) ? "-AllowStdOutLogVerbosity " : ""
);
ERunOptions Opts = ERunOptions.Default;
if (GlobalCommandLine.UTF8Output)
{
Args += " -UTF8Output";
Opts |= ERunOptions.UTF8Output;
}
IProcessResult RunResult = Run(EditorExe, Args, Options: Opts, Identifier: Commandlet);
PopDir();
// If we're running on a Windows build machine, copy any crash dumps into the log folder
if (HostPlatform.Current.HostEditorPlatform == UnrealTargetPlatform.Win64 && IsBuildMachine)
{
DirectoryInfo CrashesDir = new DirectoryInfo(DirectoryReference.Combine(DirectoryReference.FromFile(ProjectName) ?? CommandUtils.EngineDirectory, "Saved", "Crashes").FullName);
if (CrashesDir.Exists)
{
foreach (DirectoryInfo CrashDir in CrashesDir.EnumerateDirectories())
{
if (CrashDir.LastWriteTimeUtc > StartTime)
{
DirectoryInfo OutputCrashesDir = new DirectoryInfo(Path.Combine(CmdEnv.LogFolder, "Crashes", CrashDir.Name));
try
{
CommandUtils.LogInformation("Copying crash data to {0}...", OutputCrashesDir.FullName);
OutputCrashesDir.Create();
foreach (FileInfo CrashFile in CrashDir.EnumerateFiles())
{
CrashFile.CopyTo(Path.Combine(OutputCrashesDir.FullName, CrashFile.Name));
}
}
catch (Exception Ex)
{
CommandUtils.LogWarning("Unable to copy crash data; skipping. See log for exception details.");
CommandUtils.LogVerbose(Tools.DotNETCommon.ExceptionUtils.FormatExceptionDetails(Ex));
}
}
}
}
}
// If we're running on a Mac, dump all the *.crash files that were generated while the editor was running.
if (HostPlatform.Current.HostEditorPlatform == UnrealTargetPlatform.Mac)
{
// If the exit code indicates the main process crashed, introduce a small delay because the crash report is written asynchronously.
// If we exited normally, still check without waiting in case SCW or some other child process crashed.
if (RunResult.ExitCode > 128)
{
CommandUtils.LogInformation("Pausing before checking for crash logs...");
Thread.Sleep(10 * 1000);
}
// Create a list of directories containing crash logs, and add the system log folder
List <string> CrashDirs = new List <string>();
CrashDirs.Add("/Library/Logs/DiagnosticReports");
// Add the user's log directory too
string HomeDir = Environment.GetEnvironmentVariable("HOME");
if (!String.IsNullOrEmpty(HomeDir))
{
CrashDirs.Add(Path.Combine(HomeDir, "Library/Logs/DiagnosticReports"));
}
// Check each directory for crash logs
List <FileInfo> CrashFileInfos = new List <FileInfo>();
foreach (string CrashDir in CrashDirs)
{
try
{
DirectoryInfo CrashDirInfo = new DirectoryInfo(CrashDir);
if (CrashDirInfo.Exists)
{
CrashFileInfos.AddRange(CrashDirInfo.EnumerateFiles("*.crash", SearchOption.TopDirectoryOnly).Where(x => x.LastWriteTimeUtc >= StartTime));
}
}
catch (UnauthorizedAccessException)
{
// Not all account types can access /Library/Logs/DiagnosticReports
}
}
// Dump them all to the log
foreach (FileInfo CrashFileInfo in CrashFileInfos)
{
// snmpd seems to often crash (suspect due to it being starved of CPU cycles during cooks)
// also ignore spotlight crash with the excel plugin
if (!CrashFileInfo.Name.StartsWith("snmpd_") && !CrashFileInfo.Name.StartsWith("mdworker32_") && !CrashFileInfo.Name.StartsWith("Dock_"))
{
CommandUtils.LogInformation("Found crash log - {0}", CrashFileInfo.FullName);
try
{
string[] Lines = File.ReadAllLines(CrashFileInfo.FullName);
foreach (string Line in Lines)
{
CommandUtils.LogInformation("Crash: {0}", Line);
}
}
catch (Exception Ex)
{
CommandUtils.LogWarning("Failed to read file ({0})", Ex.Message);
}
}
}
}
// Copy the local commandlet log to the destination folder.
DestLogFile = LogUtils.GetUniqueLogName(CombinePaths(CmdEnv.LogFolder, Commandlet));
if (!CommandUtils.CopyFile_NoExceptions(LocalLogFile, DestLogFile))
{
CommandUtils.LogWarning("Commandlet {0} failed to copy the local log file from {1} to {2}. The log file will be lost.", Commandlet, LocalLogFile, DestLogFile);
}
string ProjectStatsDirectory = CombinePaths((ProjectName == null)? CombinePaths(CmdEnv.LocalRoot, "Engine") : Path.GetDirectoryName(ProjectName.FullName), "Saved", "Stats");
if (Directory.Exists(ProjectStatsDirectory))
{
string DestCookerStats = CmdEnv.LogFolder;
foreach (var StatsFile in Directory.EnumerateFiles(ProjectStatsDirectory, "*.csv"))
{
if (!CommandUtils.CopyFile_NoExceptions(StatsFile, CombinePaths(DestCookerStats, Path.GetFileName(StatsFile))))
{
CommandUtils.LogWarning("Commandlet {0} failed to copy the local log file from {1} to {2}. The log file will be lost.", Commandlet, StatsFile, CombinePaths(DestCookerStats, Path.GetFileName(StatsFile)));
}
}
}
// else
// {
// CommandUtils.LogWarning("Failed to find directory {0} will not save stats", ProjectStatsDirectory);
// }
// Whether it was copied correctly or not, delete the local log as it was only a temporary file.
CommandUtils.DeleteFile_NoExceptions(LocalLogFile);
// Throw an exception if the execution failed. Draw attention to signal exit codes on Posix systems, rather than just printing the exit code
if (RunResult.ExitCode != 0)
{
string ExitCodeDesc = "";
if (RunResult.ExitCode > 128 && RunResult.ExitCode < 128 + 32)
{
if (RunResult.ExitCode == 139)
{
ExitCodeDesc = " (segmentation fault)";
}
else
{
ExitCodeDesc = String.Format(" (signal {0})", RunResult.ExitCode - 128);
}
}
throw new CommandletException(DestLogFile, RunResult.ExitCode, "Editor terminated with exit code {0}{1} while running {2}{3}; see log {4}", RunResult.ExitCode, ExitCodeDesc, Commandlet, (ProjectName == null)? "" : String.Format(" for {0}", ProjectName), DestLogFile)
{
OutputFormat = AutomationExceptionOutputFormat.Minimal
};
}
}