/// <summary>
/// Retrieves and saves all artifacts from the provided session role. Artifacts are saved to the destination path
/// </summary>
/// <param name="InContext"></param>
/// <param name="InRunningRole"></param>
/// <param name="InDestArtifactPath"></param>
/// <returns></returns>
public UnrealRoleArtifacts SaveRoleArtifacts(UnrealTestContext InContext, UnrealSessionInstance.RoleInstance InRunningRole, string InDestArtifactPath)
{
bool IsServer = InRunningRole.Role.RoleType.IsServer();
string RoleName = (InRunningRole.Role.IsDummy() ? "Dummy" : "") + InRunningRole.Role.RoleType.ToString();
UnrealTargetPlatform?Platform = InRunningRole.Role.Platform;
string RoleConfig = InRunningRole.Role.Configuration.ToString();
Directory.CreateDirectory(InDestArtifactPath);
// Don't archive editor data, there can be a *lot* of stuff in that saved folder!
bool IsEditor = InRunningRole.Role.RoleType.UsesEditor();
bool IsDevBuild = InContext.TestParams.ParseParam("dev");
string DestSavedDir = Path.Combine(InDestArtifactPath, "Saved");
string SourceSavedDir = "";
// save the contents of the saved directory
SourceSavedDir = InRunningRole.AppInstance.ArtifactPath;
// save the output from TTY
string ArtifactLogPath = Path.Combine(InDestArtifactPath, RoleName + "Output.log");
// Write a brief Gauntlet header to aid debugging
StringBuilder LogOut = new StringBuilder();
LogOut.AppendLine("------ Gauntlet Test ------");
LogOut.AppendFormat("Role: {0}\r\n", InRunningRole.Role);
LogOut.AppendFormat("Automation Command: {0}\r\n", Environment.CommandLine);
LogOut.AppendLine("---------------------------");
// Write instance stdout stream
LogOut.Append(InRunningRole.AppInstance.StdOut);
File.WriteAllText(ArtifactLogPath, LogOut.ToString());
Log.Info("Wrote Log to {0}", ArtifactLogPath);
if (IsServer == false)
{
// gif-ify and jpeg-ify any screenshots
try
{
string ScreenshotPath = Path.Combine(SourceSavedDir, "Screenshots", Platform.ToString()).ToLower();
if (Directory.Exists(ScreenshotPath) && Directory.GetFiles(ScreenshotPath).Length > 0)
{
Log.Info("Downsizing and gifying session images at {0}", ScreenshotPath);
// downsize first so gif-step is quicker and takes less resoruces.
Utils.Image.ConvertImages(ScreenshotPath, ScreenshotPath, "jpg", true);
string GifPath = Path.Combine(InDestArtifactPath, RoleName + "Test.gif");
if (Utils.Image.SaveImagesAsGif(ScreenshotPath, GifPath))
{
Log.Info("Saved gif to {0}", GifPath);
}
}
}
catch (Exception Ex)
{
Log.Info("Failed to downsize and gif-ify images! {0}", Ex);
}
}
// don't archive data in dev mode, because peoples saved data could be huuuuuuuge!
if (IsEditor == false)
{
LogLevel OldLevel = Log.Level;
Log.Level = LogLevel.Normal;
if (Directory.Exists(SourceSavedDir))
{
Utils.SystemHelpers.CopyDirectory(SourceSavedDir, DestSavedDir);
Log.Info("Archived artifacts to to {0}", DestSavedDir);
}
else
{
Log.Info("Archive path '{0}' was not found!", SourceSavedDir);
}
Log.Level = OldLevel;
}
else
{
if (IsEditor)
{
Log.Info("Skipping archival of assets for editor {0}", RoleName);
}
else if (IsDevBuild)
{
Log.Info("Skipping archival of assets for dev build");
}
}
foreach (EIntendedBaseCopyDirectory ArtifactDir in InRunningRole.Role.AdditionalArtifactDirectories)
{
if (InRunningRole.AppInstance.Device.GetPlatformDirectoryMappings().ContainsKey(ArtifactDir))
{
string SourcePath = InRunningRole.AppInstance.Device.GetPlatformDirectoryMappings()[ArtifactDir];
var DirToCopy = new DirectoryInfo(SourcePath);
if (DirToCopy.Exists)
{
// Grab the final dir name to copy everything into so everything's not just going into root artifact dir.
string IntendedCopyLocation = Path.Combine(InDestArtifactPath, DirToCopy.Name);
Utils.SystemHelpers.CopyDirectory(SourcePath, IntendedCopyLocation);
}
}
}
// TODO REMOVEME- this should go elsewhere, likely a util that can be called or inserted by relevant test nodes.
if (IsServer == false)
{
// Copy over PSOs
try
{
if (InContext.Options.LogPSO)
{
foreach (var ThisFile in CommandUtils.FindFiles_NoExceptions(true, "*.rec.upipelinecache", true, DestSavedDir))
{
bool Copied = false;
var JustFile = Path.GetFileName(ThisFile);
if (JustFile.StartsWith("++"))
{
var Parts = JustFile.Split(new Char[] { '+', '-' }).Where(A => A != "").ToArray();
if (Parts.Count() >= 2)
{
string ProjectName = Parts[0].ToString();
string BuildRoot = CommandUtils.CombinePaths(CommandUtils.RootBuildStorageDirectory());
string SrcBuildPath = CommandUtils.CombinePaths(BuildRoot, ProjectName);
string SrcBuildPath2 = CommandUtils.CombinePaths(BuildRoot, ProjectName.Replace("Game", "").Replace("game", ""));
if (!CommandUtils.DirectoryExists(SrcBuildPath))
{
SrcBuildPath = SrcBuildPath2;
}
if (CommandUtils.DirectoryExists(SrcBuildPath))
{
var JustBuildFolder = JustFile.Replace("-" + Parts.Last(), "");
string PlatformStr = Platform.ToString();
string SrcCLMetaPath = CommandUtils.CombinePaths(SrcBuildPath, JustBuildFolder, PlatformStr, "MetaData");
if (CommandUtils.DirectoryExists(SrcCLMetaPath))
{
string SrcCLMetaPathCollected = CommandUtils.CombinePaths(SrcCLMetaPath, "CollectedPSOs");
if (!CommandUtils.DirectoryExists(SrcCLMetaPathCollected))
{
Log.Info("Creating Directory {0}", SrcCLMetaPathCollected);
CommandUtils.CreateDirectory(SrcCLMetaPathCollected);
}
if (CommandUtils.DirectoryExists(SrcCLMetaPathCollected))
{
string DestFile = CommandUtils.CombinePaths(SrcCLMetaPathCollected, JustFile);
CommandUtils.CopyFile_NoExceptions(ThisFile, DestFile, true);
if (CommandUtils.FileExists(true, DestFile))
{
Log.Info("Deleting local file, copied to {0}", DestFile);
CommandUtils.DeleteFile_NoExceptions(ThisFile, true);
Copied = true;
}
}
}
}
}
}
if (!Copied)
{
Log.Warning("Could not find anywhere to put this file {0}", JustFile);
}
}
}
}
catch (Exception Ex)
{
Log.Info("Failed to copy upipelinecaches to the network {0}", Ex);
}
}
// END REMOVEME
UnrealLogParser LogParser = new UnrealLogParser(InRunningRole.AppInstance.StdOut);
int ExitCode = InRunningRole.AppInstance.ExitCode;
LogParser.GetTestExitCode(out ExitCode);
UnrealRoleArtifacts Artifacts = new UnrealRoleArtifacts(InRunningRole.Role, InRunningRole.AppInstance, InDestArtifactPath, ArtifactLogPath, LogParser);
return(Artifacts);
}
public DeploymentContext(
FileReference RawProjectPathOrName,
string InLocalRoot,
string BaseStageDirectory,
string BaseArchiveDirectory,
Platform InSourcePlatform,
Platform InTargetPlatform,
List <UnrealTargetConfiguration> InTargetConfigurations,
IEnumerable <StageTarget> InStageTargets,
List <String> InStageExecutables,
bool InServer,
bool InCooked,
bool InStageCrashReporter,
bool InStage,
bool InCookOnTheFly,
bool InArchive,
bool InProgram,
bool IsClientInsteadOfNoEditor,
bool InForceChunkManifests,
bool bInUseWebsocketNetDriver = false
)
{
bStageCrashReporter = InStageCrashReporter;
RawProjectPath = RawProjectPathOrName;
DedicatedServer = InServer;
LocalRoot = CommandUtils.CombinePaths(InLocalRoot);
CookSourcePlatform = InSourcePlatform;
StageTargetPlatform = InTargetPlatform;
StageTargetConfigurations = new List <UnrealTargetConfiguration>(InTargetConfigurations);
StageTargets = new List <StageTarget>(InStageTargets);
StageExecutables = InStageExecutables;
IsCodeBasedProject = ProjectUtils.IsCodeBasedUProjectFile(RawProjectPath);
ShortProjectName = ProjectUtils.GetShortProjectName(RawProjectPath);
Stage = InStage;
Archive = InArchive;
bUseWebsocketNetDriver = bInUseWebsocketNetDriver;
if (CookSourcePlatform != null && InCooked)
{
CookPlatform = CookSourcePlatform.GetCookPlatform(DedicatedServer, IsClientInsteadOfNoEditor);
}
else if (CookSourcePlatform != null && InProgram)
{
CookPlatform = CookSourcePlatform.GetCookPlatform(false, false);
}
else
{
CookPlatform = "";
}
if (StageTargetPlatform != null && InCooked)
{
FinalCookPlatform = StageTargetPlatform.GetCookPlatform(DedicatedServer, IsClientInsteadOfNoEditor);
}
else if (StageTargetPlatform != null && InProgram)
{
FinalCookPlatform = StageTargetPlatform.GetCookPlatform(false, false);
}
else
{
FinalCookPlatform = "";
}
PlatformDir = StageTargetPlatform.PlatformType.ToString();
StageDirectory = CommandUtils.CombinePaths(BaseStageDirectory, FinalCookPlatform);
ArchiveDirectory = CommandUtils.CombinePaths(BaseArchiveDirectory, FinalCookPlatform);
if (!CommandUtils.FileExists(RawProjectPath.FullName))
{
throw new AutomationException("Can't find uproject file {0}.", RawProjectPathOrName);
}
ProjectRoot = CommandUtils.CombinePaths(CommandUtils.GetDirectoryName(RawProjectPath.FullName));
if (!CommandUtils.DirectoryExists(ProjectRoot))
{
throw new AutomationException("Project Directory {0} doesn't exist.", ProjectRoot);
}
RelativeProjectRootForStage = ShortProjectName;
ProjectArgForCommandLines = CommandUtils.MakePathSafeToUseWithCommandLine(RawProjectPath.FullName);
CookSourceRuntimeRootDir = RuntimeRootDir = LocalRoot;
RuntimeProjectRootDir = ProjectRoot;
RelativeProjectRootForUnrealPak = CommandUtils.CombinePaths(RelativeProjectRootForStage).Replace("\\", "/");
if (RelativeProjectRootForUnrealPak.StartsWith("/"))
{
RelativeProjectRootForUnrealPak = RelativeProjectRootForUnrealPak.Substring(1);
RelativeProjectRootForStage = RelativeProjectRootForStage.Substring(1);
}
SourceRelativeProjectRoot = RelativeProjectRootForStage; // for foreign projects this doesn't make much sense, but it turns into a noop on staging files
if (ProjectRoot.StartsWith(LocalRoot, StringComparison.InvariantCultureIgnoreCase))
{
SourceRelativeProjectRoot = ProjectRoot.Substring(LocalRoot.Length);
}
if (SourceRelativeProjectRoot.StartsWith("/") || SourceRelativeProjectRoot.StartsWith("\\"))
{
SourceRelativeProjectRoot = SourceRelativeProjectRoot.Substring(1);
}
if (Stage)
{
CommandUtils.CreateDirectory(StageDirectory);
StageProjectRoot = CommandUtils.CombinePaths(StageDirectory, RelativeProjectRootForStage);
RuntimeRootDir = StageDirectory;
CookSourceRuntimeRootDir = CommandUtils.CombinePaths(BaseStageDirectory, CookPlatform);
RuntimeProjectRootDir = StageProjectRoot;
ProjectArgForCommandLines = CommandUtils.MakePathSafeToUseWithCommandLine(UProjectCommandLineArgInternalRoot + RelativeProjectRootForStage + "/" + ShortProjectName + ".uproject");
}
if (Archive)
{
CommandUtils.CreateDirectory(ArchiveDirectory);
}
ProjectArgForCommandLines = ProjectArgForCommandLines.Replace("\\", "/");
ProjectBinariesFolder = CommandUtils.CombinePaths(ProjectUtils.GetClientProjectBinariesRootPath(RawProjectPath, TargetRules.TargetType.Game, IsCodeBasedProject), PlatformDir);
// If we were configured to use manifests across the whole project, then this platform should use manifests.
// Otherwise, read whether we are generating chunks from the ProjectPackagingSettings ini.
if (InForceChunkManifests)
{
PlatformUsesChunkManifests = true;
}
else
{
ConfigCacheIni GameIni = ConfigCacheIni.CreateConfigCacheIni(InTargetPlatform.PlatformType, "Game", RawProjectPath.Directory);
String IniPath = "/Script/UnrealEd.ProjectPackagingSettings";
bool bSetting = false;
if (GameIni.GetBool(IniPath, "bGenerateChunks", out bSetting))
{
PlatformUsesChunkManifests = bSetting;
}
}
}
public override void ExecuteBuild()
{
// Get the list of platform names
string[] FeaturePacks = ParseParamValue("FeaturePacks").Split(';');
string TempDir = ParseParamValue("TempDir");
UnrealTargetPlatform HostPlatform = BuildHostPlatform.Current.Platform;
string TargetPlatforms = ParseParamValue("TargetPlatforms");
string SavedDir = ParseParamValue("SavedDir");
string BackendName = ParseParamValue("BackendName", "CreateInstalledEnginePak");
string RelativePakPath = ParseParamValue("RelativePakPath", "Engine/DerivedDataCache/Compressed.ddp");
bool bSkipEngine = ParseParam("SkipEngine");
// Get paths to everything within the temporary directory
string EditorExe = CommandUtils.GetEditorCommandletExe(TempDir, HostPlatform);
string OutputPakFile = CommandUtils.CombinePaths(TempDir, RelativePakPath);
string OutputCsvFile = Path.ChangeExtension(OutputPakFile, ".csv");
List <string> ProjectPakFiles = new List <string>();
List <string> FeaturePackPaths = new List <string>();
// loop through all the projects first and bail out if one of them doesn't exist.
foreach (string FeaturePack in FeaturePacks)
{
if (!String.IsNullOrWhiteSpace(FeaturePack))
{
string FeaturePackPath = CommandUtils.CombinePaths(CommandUtils.RootDirectory.FullName, FeaturePack);
if (!CommandUtils.FileExists(FeaturePackPath))
{
throw new AutomationException("Could not find project: " + FeaturePack);
}
FeaturePackPaths.Add(FeaturePackPath);
}
}
// loop through all the paths and generate ddc data for them
foreach (string FeaturePackPath in FeaturePackPaths)
{
string ProjectSpecificPlatforms = TargetPlatforms;
FileReference FileRef = new FileReference(FeaturePackPath);
string GameName = FileRef.GetFileNameWithoutAnyExtensions();
ProjectDescriptor Project = ProjectDescriptor.FromFile(FileRef);
if (Project.TargetPlatforms != null && Project.TargetPlatforms.Length > 0)
{
// Restrict target platforms used to those specified in project file
List <string> FilteredPlatforms = new List <string>();
// Always include the editor platform for cooking
string EditorCookPlatform = Platform.GetPlatform(HostPlatform).GetEditorCookPlatform();
if (TargetPlatforms.Contains(EditorCookPlatform))
{
FilteredPlatforms.Add(EditorCookPlatform);
}
foreach (string TargetPlatform in Project.TargetPlatforms)
{
if (TargetPlatforms.Contains(TargetPlatform))
{
FilteredPlatforms.Add(TargetPlatform);
}
}
if (FilteredPlatforms.Count == 0)
{
LogInformation("Did not find any project specific platforms for FeaturePack {0} out of supplied TargetPlatforms {1}, skipping it!", GameName, ProjectSpecificPlatforms);
continue;
}
ProjectSpecificPlatforms = CommandUtils.CombineCommandletParams(FilteredPlatforms.Distinct().ToArray());
}
CommandUtils.LogInformation("Generating DDC data for {0} on {1}", GameName, ProjectSpecificPlatforms);
CommandUtils.DDCCommandlet(FileRef, EditorExe, null, ProjectSpecificPlatforms, String.Format("-fill -DDC={0} -ProjectOnly", BackendName));
string ProjectPakFile = CommandUtils.CombinePaths(Path.GetDirectoryName(OutputPakFile), String.Format("Compressed-{0}.ddp", GameName));
CommandUtils.DeleteFile(ProjectPakFile);
CommandUtils.RenameFile(OutputPakFile, ProjectPakFile);
string ProjectCsvFile = Path.ChangeExtension(ProjectPakFile, ".csv");
CommandUtils.DeleteFile(ProjectCsvFile);
CommandUtils.RenameFile(OutputCsvFile, ProjectCsvFile);
ProjectPakFiles.Add(Path.GetFileName(ProjectPakFile));
}
// Generate DDC for the editor, and merge all the other PAK files in
CommandUtils.LogInformation("Generating DDC data for engine content on {0}", TargetPlatforms);
CommandUtils.DDCCommandlet(null, EditorExe, null, TargetPlatforms, String.Format("-fill -DDC={0} -MergePaks={1}{2}", BackendName, CommandUtils.MakePathSafeToUseWithCommandLine(String.Join("+", ProjectPakFiles)), bSkipEngine? " -projectonly" : ""));
string SavedPakFile = CommandUtils.CombinePaths(SavedDir, RelativePakPath);
CommandUtils.CopyFile(OutputPakFile, SavedPakFile);
}
public DeploymentContext(
string RawProjectPathOrName,
string InLocalRoot,
string BaseStageDirectory,
string BaseArchiveDirectory,
string CookFlavor,
Platform InSourcePlatform,
Platform InTargetPlatform,
List <UnrealTargetConfiguration> InTargetConfigurations,
List <String> InStageExecutables,
bool InServer,
bool InCooked,
bool InStageCrashReporter,
bool InStage,
bool InCookOnTheFly,
bool InArchive,
bool InProgram,
bool bHasDedicatedServerAndClient
)
{
bStageCrashReporter = InStageCrashReporter;
RawProjectPath = RawProjectPathOrName;
DedicatedServer = InServer;
LocalRoot = CommandUtils.CombinePaths(InLocalRoot);
CookSourcePlatform = InSourcePlatform;
StageTargetPlatform = InTargetPlatform;
StageTargetConfigurations = new List <UnrealTargetConfiguration>(InTargetConfigurations);
StageExecutables = InStageExecutables;
IsCodeBasedProject = ProjectUtils.IsCodeBasedUProjectFile(RawProjectPath);
ShortProjectName = ProjectUtils.GetShortProjectName(RawProjectPath);
Stage = InStage;
Archive = InArchive;
if (CookSourcePlatform != null && InCooked)
{
CookPlatform = CookSourcePlatform.GetCookPlatform(DedicatedServer, bHasDedicatedServerAndClient, CookFlavor);
}
else if (CookSourcePlatform != null && InProgram)
{
CookPlatform = CookSourcePlatform.GetCookPlatform(false, false, "");
}
else
{
CookPlatform = "";
}
if (StageTargetPlatform != null && InCooked)
{
FinalCookPlatform = StageTargetPlatform.GetCookPlatform(DedicatedServer, bHasDedicatedServerAndClient, CookFlavor);
}
else if (StageTargetPlatform != null && InProgram)
{
FinalCookPlatform = StageTargetPlatform.GetCookPlatform(false, false, "");
}
else
{
FinalCookPlatform = "";
}
PlatformDir = StageTargetPlatform.PlatformType.ToString();
StageDirectory = CommandUtils.CombinePaths(BaseStageDirectory, FinalCookPlatform);
ArchiveDirectory = CommandUtils.CombinePaths(BaseArchiveDirectory, FinalCookPlatform);
if (!CommandUtils.FileExists(RawProjectPath))
{
throw new AutomationException("Can't find uproject file {0}.", RawProjectPathOrName);
}
ProjectRoot = CommandUtils.CombinePaths(CommandUtils.GetDirectoryName(Path.GetFullPath(RawProjectPath)));
if (!CommandUtils.DirectoryExists(ProjectRoot))
{
throw new AutomationException("Project Directory {0} doesn't exist.", ProjectRoot);
}
RelativeProjectRootForStage = ShortProjectName;
ProjectArgForCommandLines = CommandUtils.MakePathSafeToUseWithCommandLine(RawProjectPath);
CookSourceRuntimeRootDir = RuntimeRootDir = LocalRoot;
RuntimeProjectRootDir = ProjectRoot;
RelativeProjectRootForUnrealPak = CommandUtils.CombinePaths(RelativeProjectRootForStage).Replace("\\", "/");
if (RelativeProjectRootForUnrealPak.StartsWith("/"))
{
RelativeProjectRootForUnrealPak = RelativeProjectRootForUnrealPak.Substring(1);
RelativeProjectRootForStage = RelativeProjectRootForStage.Substring(1);
}
SourceRelativeProjectRoot = RelativeProjectRootForStage; // for foreign projects this doesn't make much sense, but it turns into a noop on staging files
if (ProjectRoot.StartsWith(LocalRoot, StringComparison.InvariantCultureIgnoreCase))
{
SourceRelativeProjectRoot = ProjectRoot.Substring(LocalRoot.Length);
}
if (SourceRelativeProjectRoot.StartsWith("/") || SourceRelativeProjectRoot.StartsWith("\\"))
{
SourceRelativeProjectRoot = SourceRelativeProjectRoot.Substring(1);
}
if (Stage)
{
CommandUtils.CreateDirectory(StageDirectory);
StageProjectRoot = CommandUtils.CombinePaths(StageDirectory, RelativeProjectRootForStage);
RuntimeRootDir = StageDirectory;
CookSourceRuntimeRootDir = CommandUtils.CombinePaths(BaseStageDirectory, CookPlatform);
RuntimeProjectRootDir = StageProjectRoot;
ProjectArgForCommandLines = CommandUtils.MakePathSafeToUseWithCommandLine(UProjectCommandLineArgInternalRoot + RelativeProjectRootForStage + "/" + ShortProjectName + ".uproject");
}
if (Archive)
{
CommandUtils.CreateDirectory(ArchiveDirectory);
}
ProjectArgForCommandLines = ProjectArgForCommandLines.Replace("\\", "/");
}
