/// <summary> /// Runs a commandlet using Engine/Binaries/Win64/UE4Editor-Cmd.exe. /// </summary> /// <param name="ProjectName">Project name.</param> /// <param name="UE4Exe">The name of the UE4 Editor executable to use.</param> /// <param name="Commandlet">Commandlet name.</param> /// <param name="Parameters">Command line parameters (without -run=)</param> public static void RunCommandlet(string ProjectName, string UE4Exe, string Commandlet, string Parameters = null) { Log("Running UE4Editor {0} for project {1}", Commandlet, ProjectName); var CWD = Path.GetDirectoryName(UE4Exe); string EditorExe = UE4Exe; if (String.IsNullOrEmpty(CWD)) { EditorExe = HostPlatform.Current.GetUE4ExePath(UE4Exe); CWD = CombinePaths(CmdEnv.LocalRoot, HostPlatform.Current.RelativeBinariesFolder); } PushDir(CWD); string LocalLogFile = LogUtils.GetUniqueLogName(CombinePaths(CmdEnv.EngineSavedFolder, Commandlet)); Log("Commandlet log file is {0}", LocalLogFile); string Args = String.Format( "{0} -run={1} {2} -abslog={3} -stdout -FORCELOGFLUSH -CrashForUAT -unattended -AllowStdOutLogVerbosity {4}", (ProjectName == null) ? "" : CommandUtils.MakePathSafeToUseWithCommandLine(ProjectName), Commandlet, String.IsNullOrEmpty(Parameters) ? "" : Parameters, CommandUtils.MakePathSafeToUseWithCommandLine(LocalLogFile), IsBuildMachine ? "-buildmachine" : "" ); ERunOptions Opts = ERunOptions.Default; if (GlobalCommandLine.UTF8Output) { Args += " -UTF8Output"; Opts |= ERunOptions.UTF8Output; } var RunResult = Run(EditorExe, Args, Options: Opts); PopDir(); // Copy the local commandlet log to the destination folder. string DestLogFile = LogUtils.GetUniqueLogName(CombinePaths(CmdEnv.LogFolder, Commandlet)); if (!CommandUtils.CopyFile_NoExceptions(LocalLogFile, DestLogFile)) { CommandUtils.LogWarning("Commandlet {0} failed to copy the local log file from {1} to {2}. The log file will be lost.", Commandlet, LocalLogFile, DestLogFile); } // Whether it was copied correctly or not, delete the local log as it was only a temporary file. CommandUtils.DeleteFile_NoExceptions(LocalLogFile); if (RunResult.ExitCode != 0) { throw new AutomationException("BUILD FAILED: Failed while running {0} for {1}; see log {2}", Commandlet, ProjectName, DestLogFile); } }
public static void Robust_CopyFile(string InputFileName, string OutputFileName) { if (OutputFileName.StartsWith("/Volumes/")) { int Retry = 0; int NumRetries = 60; bool bCopied = false; while (!bCopied && Retry < NumRetries) { if (Retry > 0) { //@todo: These retries should be reported so we can track how often they are occurring. CommandUtils.LogInformation("*** Mac temp storage retry {0}", OutputFileName); System.Threading.Thread.Sleep(1000); } bCopied = CommandUtils.CopyFile_NoExceptions(InputFileName, OutputFileName, true); Retry++; } } else { CommandUtils.CopyFile(InputFileName, OutputFileName); } }
/// <summary> /// Runs a commandlet using Engine/Binaries/Win64/UE4Editor-Cmd.exe. /// </summary> /// <param name="ProjectFile">Project name.</param> /// <param name="UE4Exe">The name of the UE4 Editor executable to use.</param> /// <param name="Commandlet">Commandlet name.</param> /// <param name="Parameters">Command line parameters (without -run=)</param> public static void RunCommandlet(FileReference ProjectName, string UE4Exe, string Commandlet, string Parameters = null) { Log("Running UE4Editor {0} for project {1}", Commandlet, ProjectName); var CWD = Path.GetDirectoryName(UE4Exe); string EditorExe = UE4Exe; if (String.IsNullOrEmpty(CWD)) { EditorExe = HostPlatform.Current.GetUE4ExePath(UE4Exe); CWD = CombinePaths(CmdEnv.LocalRoot, HostPlatform.Current.RelativeBinariesFolder); } PushDir(CWD); DateTime StartTime = DateTime.UtcNow; string LocalLogFile = LogUtils.GetUniqueLogName(CombinePaths(CmdEnv.EngineSavedFolder, Commandlet)); Log("Commandlet log file is {0}", LocalLogFile); string Args = String.Format( "{0} -run={1} {2} -abslog={3} -stdout -FORCELOGFLUSH -CrashForUAT -unattended {5}{4}", (ProjectName == null) ? "" : CommandUtils.MakePathSafeToUseWithCommandLine(ProjectName.FullName), Commandlet, String.IsNullOrEmpty(Parameters) ? "" : Parameters, CommandUtils.MakePathSafeToUseWithCommandLine(LocalLogFile), IsBuildMachine ? "-buildmachine" : "", (GlobalCommandLine.Verbose || GlobalCommandLine.AllowStdOutLogVerbosity) ? "-AllowStdOutLogVerbosity " : "" ); ERunOptions Opts = ERunOptions.Default; if (GlobalCommandLine.UTF8Output) { Args += " -UTF8Output"; Opts |= ERunOptions.UTF8Output; } var RunResult = Run(EditorExe, Args, Options: Opts); PopDir(); // Draw attention to signal exit codes on Posix systems, rather than just printing the exit code if (RunResult.ExitCode > 128 && RunResult.ExitCode < 128 + 32) { if (RunResult.ExitCode == 139) { CommandUtils.LogError("Editor terminated abnormally due to a segmentation fault"); } else { CommandUtils.LogError("Editor terminated abnormally with signal {0}", RunResult.ExitCode - 128); } } // If we're running on a Mac, dump all the *.crash files that were generated while the editor was running. if (HostPlatform.Current.HostEditorPlatform == UnrealTargetPlatform.Mac) { // If the exit code indicates the main process crashed, introduce a small delay because the crash report is written asynchronously. // If we exited normally, still check without waiting in case SCW or some other child process crashed. if (RunResult.ExitCode > 128) { CommandUtils.Log("Pausing before checking for crash logs..."); Thread.Sleep(10 * 1000); } // Create a list of directories containing crash logs, and add the system log folder List <string> CrashDirs = new List <string>(); CrashDirs.Add("/Library/Logs/DiagnosticReports"); // Add the user's log directory too string HomeDir = Environment.GetEnvironmentVariable("HOME"); if (!String.IsNullOrEmpty(HomeDir)) { CrashDirs.Add(Path.Combine(HomeDir, "Library/Logs/DiagnosticReports")); } // Check each directory for crash logs List <FileInfo> CrashFileInfos = new List <FileInfo>(); foreach (string CrashDir in CrashDirs) { DirectoryInfo CrashDirInfo = new DirectoryInfo(CrashDir); if (CrashDirInfo.Exists) { CrashFileInfos.AddRange(CrashDirInfo.EnumerateFiles("*.crash", SearchOption.TopDirectoryOnly).Where(x => x.LastWriteTimeUtc >= StartTime)); } } // Dump them all to the log foreach (FileInfo CrashFileInfo in CrashFileInfos) { // snmpd seems to often crash (suspect due to it being starved of CPU cycles during cooks) if (!CrashFileInfo.Name.StartsWith("snmpd_")) { CommandUtils.Log("Found crash log - {0}", CrashFileInfo.FullName); try { string[] Lines = File.ReadAllLines(CrashFileInfo.FullName); foreach (string Line in Lines) { CommandUtils.Log("Crash: {0}", Line); } } catch (Exception Ex) { CommandUtils.LogWarning("Failed to read file ({0})", Ex.Message); } } } } // Copy the local commandlet log to the destination folder. string DestLogFile = LogUtils.GetUniqueLogName(CombinePaths(CmdEnv.LogFolder, Commandlet)); if (!CommandUtils.CopyFile_NoExceptions(LocalLogFile, DestLogFile)) { CommandUtils.LogWarning("Commandlet {0} failed to copy the local log file from {1} to {2}. The log file will be lost.", Commandlet, LocalLogFile, DestLogFile); } string ProjectStatsDirectory = CombinePaths((ProjectName == null)? CombinePaths(CmdEnv.LocalRoot, "Engine") : Path.GetDirectoryName(ProjectName.FullName), "Saved", "Stats"); if (Directory.Exists(ProjectStatsDirectory)) { string DestCookerStats = CmdEnv.LogFolder; foreach (var StatsFile in Directory.EnumerateFiles(ProjectStatsDirectory, "*.csv")) { if (!CommandUtils.CopyFile_NoExceptions(StatsFile, CombinePaths(DestCookerStats, Path.GetFileName(StatsFile)))) { CommandUtils.LogWarning("Commandlet {0} failed to copy the local log file from {1} to {2}. The log file will be lost.", Commandlet, StatsFile, CombinePaths(DestCookerStats, Path.GetFileName(StatsFile))); } } } // else // { // CommandUtils.LogWarning("Failed to find directory {0} will not save stats", ProjectStatsDirectory); // } // Whether it was copied correctly or not, delete the local log as it was only a temporary file. CommandUtils.DeleteFile_NoExceptions(LocalLogFile); if (RunResult.ExitCode != 0) { throw new AutomationException(DestLogFile, RunResult.ExitCode, "BUILD FAILED: Failed while running {0} for {1}; see log {2}", Commandlet, ProjectName, DestLogFile); } }
/// <summary> /// Runs a commandlet using Engine/Binaries/Win64/UE4Editor-Cmd.exe. /// </summary> /// <param name="ProjectFile">Project name.</param> /// <param name="UE4Exe">The name of the UE4 Editor executable to use.</param> /// <param name="Commandlet">Commandlet name.</param> /// <param name="Parameters">Command line parameters (without -run=)</param> /// <param name="DestLogFile">Log file after completion</param> public static void RunCommandlet(FileReference ProjectName, string UE4Exe, string Commandlet, string Parameters, out string DestLogFile) { LogInformation("Running UE4Editor {0} for project {1}", Commandlet, ProjectName); var CWD = Path.GetDirectoryName(UE4Exe); string EditorExe = UE4Exe; if (String.IsNullOrEmpty(CWD)) { EditorExe = HostPlatform.Current.GetUE4ExePath(UE4Exe); CWD = CombinePaths(CmdEnv.LocalRoot, HostPlatform.Current.RelativeBinariesFolder); } PushDir(CWD); DateTime StartTime = DateTime.UtcNow; string LocalLogFile = LogUtils.GetUniqueLogName(CombinePaths(CmdEnv.EngineSavedFolder, Commandlet)); LogInformation("Commandlet log file is {0}", LocalLogFile); string Args = String.Format( "{0} -run={1} {2} -abslog={3} -stdout -CrashForUAT -unattended -NoLogTimes {5}{4}", (ProjectName == null) ? "" : CommandUtils.MakePathSafeToUseWithCommandLine(ProjectName.FullName), Commandlet, String.IsNullOrEmpty(Parameters) ? "" : Parameters, CommandUtils.MakePathSafeToUseWithCommandLine(LocalLogFile), IsBuildMachine ? "-buildmachine" : "", (GlobalCommandLine.Verbose || GlobalCommandLine.AllowStdOutLogVerbosity) ? "-AllowStdOutLogVerbosity " : "" ); ERunOptions Opts = ERunOptions.Default; if (GlobalCommandLine.UTF8Output) { Args += " -UTF8Output"; Opts |= ERunOptions.UTF8Output; } IProcessResult RunResult = Run(EditorExe, Args, Options: Opts, Identifier: Commandlet); PopDir(); // If we're running on a Windows build machine, copy any crash dumps into the log folder if (HostPlatform.Current.HostEditorPlatform == UnrealTargetPlatform.Win64 && IsBuildMachine) { DirectoryInfo CrashesDir = new DirectoryInfo(DirectoryReference.Combine(DirectoryReference.FromFile(ProjectName) ?? CommandUtils.EngineDirectory, "Saved", "Crashes").FullName); if (CrashesDir.Exists) { foreach (DirectoryInfo CrashDir in CrashesDir.EnumerateDirectories()) { if (CrashDir.LastWriteTimeUtc > StartTime) { DirectoryInfo OutputCrashesDir = new DirectoryInfo(Path.Combine(CmdEnv.LogFolder, "Crashes", CrashDir.Name)); try { CommandUtils.LogInformation("Copying crash data to {0}...", OutputCrashesDir.FullName); OutputCrashesDir.Create(); foreach (FileInfo CrashFile in CrashDir.EnumerateFiles()) { CrashFile.CopyTo(Path.Combine(OutputCrashesDir.FullName, CrashFile.Name)); } } catch (Exception Ex) { CommandUtils.LogWarning("Unable to copy crash data; skipping. See log for exception details."); CommandUtils.LogVerbose(Tools.DotNETCommon.ExceptionUtils.FormatExceptionDetails(Ex)); } } } } } // If we're running on a Mac, dump all the *.crash files that were generated while the editor was running. if (HostPlatform.Current.HostEditorPlatform == UnrealTargetPlatform.Mac) { // If the exit code indicates the main process crashed, introduce a small delay because the crash report is written asynchronously. // If we exited normally, still check without waiting in case SCW or some other child process crashed. if (RunResult.ExitCode > 128) { CommandUtils.LogInformation("Pausing before checking for crash logs..."); Thread.Sleep(10 * 1000); } // Create a list of directories containing crash logs, and add the system log folder List <string> CrashDirs = new List <string>(); CrashDirs.Add("/Library/Logs/DiagnosticReports"); // Add the user's log directory too string HomeDir = Environment.GetEnvironmentVariable("HOME"); if (!String.IsNullOrEmpty(HomeDir)) { CrashDirs.Add(Path.Combine(HomeDir, "Library/Logs/DiagnosticReports")); } // Check each directory for crash logs List <FileInfo> CrashFileInfos = new List <FileInfo>(); foreach (string CrashDir in CrashDirs) { try { DirectoryInfo CrashDirInfo = new DirectoryInfo(CrashDir); if (CrashDirInfo.Exists) { CrashFileInfos.AddRange(CrashDirInfo.EnumerateFiles("*.crash", SearchOption.TopDirectoryOnly).Where(x => x.LastWriteTimeUtc >= StartTime)); } } catch (UnauthorizedAccessException) { // Not all account types can access /Library/Logs/DiagnosticReports } } // Dump them all to the log foreach (FileInfo CrashFileInfo in CrashFileInfos) { // snmpd seems to often crash (suspect due to it being starved of CPU cycles during cooks) // also ignore spotlight crash with the excel plugin if (!CrashFileInfo.Name.StartsWith("snmpd_") && !CrashFileInfo.Name.StartsWith("mdworker32_") && !CrashFileInfo.Name.StartsWith("Dock_")) { CommandUtils.LogInformation("Found crash log - {0}", CrashFileInfo.FullName); try { string[] Lines = File.ReadAllLines(CrashFileInfo.FullName); foreach (string Line in Lines) { CommandUtils.LogInformation("Crash: {0}", Line); } } catch (Exception Ex) { CommandUtils.LogWarning("Failed to read file ({0})", Ex.Message); } } } } // Copy the local commandlet log to the destination folder. DestLogFile = LogUtils.GetUniqueLogName(CombinePaths(CmdEnv.LogFolder, Commandlet)); if (!CommandUtils.CopyFile_NoExceptions(LocalLogFile, DestLogFile)) { CommandUtils.LogWarning("Commandlet {0} failed to copy the local log file from {1} to {2}. The log file will be lost.", Commandlet, LocalLogFile, DestLogFile); } string ProjectStatsDirectory = CombinePaths((ProjectName == null)? CombinePaths(CmdEnv.LocalRoot, "Engine") : Path.GetDirectoryName(ProjectName.FullName), "Saved", "Stats"); if (Directory.Exists(ProjectStatsDirectory)) { string DestCookerStats = CmdEnv.LogFolder; foreach (var StatsFile in Directory.EnumerateFiles(ProjectStatsDirectory, "*.csv")) { if (!CommandUtils.CopyFile_NoExceptions(StatsFile, CombinePaths(DestCookerStats, Path.GetFileName(StatsFile)))) { CommandUtils.LogWarning("Commandlet {0} failed to copy the local log file from {1} to {2}. The log file will be lost.", Commandlet, StatsFile, CombinePaths(DestCookerStats, Path.GetFileName(StatsFile))); } } } // else // { // CommandUtils.LogWarning("Failed to find directory {0} will not save stats", ProjectStatsDirectory); // } // Whether it was copied correctly or not, delete the local log as it was only a temporary file. CommandUtils.DeleteFile_NoExceptions(LocalLogFile); // Throw an exception if the execution failed. Draw attention to signal exit codes on Posix systems, rather than just printing the exit code if (RunResult.ExitCode != 0) { string ExitCodeDesc = ""; if (RunResult.ExitCode > 128 && RunResult.ExitCode < 128 + 32) { if (RunResult.ExitCode == 139) { ExitCodeDesc = " (segmentation fault)"; } else { ExitCodeDesc = String.Format(" (signal {0})", RunResult.ExitCode - 128); } } throw new CommandletException(DestLogFile, RunResult.ExitCode, "Editor terminated with exit code {0}{1} while running {2}{3}; see log {4}", RunResult.ExitCode, ExitCodeDesc, Commandlet, (ProjectName == null)? "" : String.Format(" for {0}", ProjectName), DestLogFile) { OutputFormat = AutomationExceptionOutputFormat.Minimal }; } }