/// <summary>
/// Saves all artifacts from the provided session to the specified output path.
/// </summary>
/// <param name="Context"></param>
/// <param name="TestInstance"></param>
/// <param name="OutputPath"></param>
/// <returns></returns>
public IEnumerable <UnrealRoleArtifacts> SaveRoleArtifacts(UnrealTestContext Context, UnrealSessionInstance TestInstance, string OutputPath)
{
int ClientCount = 1;
int DummyClientCount = 1;
List <UnrealRoleArtifacts> AllArtifacts = new List <UnrealRoleArtifacts>();
foreach (UnrealSessionInstance.RoleInstance App in TestInstance.RunningRoles)
{
bool IsServer = App.Role.RoleType.IsServer();
string RoleName = (App.Role.IsDummy() ? "Dummy" : "") + App.Role.RoleType.ToString();
string FolderName = RoleName;
if (IsServer == false)
{
if ((!App.Role.IsDummy() && ClientCount++ > 1))
{
FolderName += string.Format("_{0:00}", ClientCount);
}
else if (App.Role.IsDummy() && DummyClientCount++ > 1)
{
FolderName += string.Format("_{0:00}", DummyClientCount);
}
}
string DestPath = Path.Combine(OutputPath, FolderName);
var Artifacts = SaveRoleArtifacts(Context, App, DestPath);
AllArtifacts.Add(Artifacts);
}
return(AllArtifacts);
}
/// <summary>
/// Create the list of tests specified by the context.
/// </summary>
/// <param name="Context"></param>
/// <returns></returns>
IEnumerable <ITestNode> CreateTestList(UnrealTestContext Context, Params DefaultParams, ArgumentWithParams PlatformParams = null)
{
List <ITestNode> NodeList = new List <ITestNode>();
IEnumerable <string> Namespaces = Context.Options.Namespaces.Split(',').Select(S => S.Trim());
List <string> BuildIssues = new List <string>();
UnrealTargetPlatform UnrealPlatform = UnrealTargetPlatform.Unknown;
if (!Enum.TryParse(PlatformParams.Argument, true, out UnrealPlatform))
{
throw new AutomationException("Could not convert platform {0} to a valid UnrealTargetPlatform", PlatformParams.Argument);
}
//List<string> Platforms = Globals.Params.ParseValue("platform")
// Create an instance of each test and add it to the executor
foreach (var Test in Context.Options.TestList)
{
// create a copy of the context for this test
UnrealTestContext TestContext = (UnrealTestContext)Context.Clone();
// if test specifies platforms, filter for this context
if (Test.Platforms.Count() > 0 && Test.Platforms.Where(Plat => Plat.Argument == PlatformParams.Argument).Count() == 0)
{
continue;
}
if (Blacklist.Instance.IsTestBlacklisted(Test.TestName, UnrealPlatform, TestContext.BuildInfo.Branch))
{
Gauntlet.Log.Info("Test {0} is currently blacklisted on {1} in branch {2}", Test.TestName, UnrealPlatform, TestContext.BuildInfo.Branch);
continue;
}
// combine global and test-specific params
Params CombinedParams = new Params(DefaultParams.AllArguments.Concat(Test.TestParams.AllArguments).ToArray());
// parse any target constraints
List <string> PerfSpecArgs = CombinedParams.ParseValues("PerfSpec", false);
string PerfSpecArg = PerfSpecArgs.Count > 0 ? PerfSpecArgs.Last() : "Unspecified";
EPerfSpec PerfSpec;
if (!Enum.TryParse <EPerfSpec>(PerfSpecArg, true, out PerfSpec))
{
throw new AutomationException("Unable to convert perfspec '{0}' into an EPerfSpec", PerfSpec);
}
TestContext.Constraint = new UnrealTargetConstraint(UnrealPlatform, PerfSpec);
TestContext.TestParams = CombinedParams;
// This will throw if the test cannot be created
ITestNode NewTest = Utils.TestConstructor.ConstructTest <ITestNode, UnrealTestContext>(Test.TestName, TestContext, Namespaces);
NodeList.Add(NewTest);
}
return(NodeList);
}
public object Clone()
{
UnrealTestContext Copy = (UnrealTestContext)this.MemberwiseClone();
// todo - what else shou;d be unique?
Copy.RoleContext = this.RoleContext.ToDictionary(entry => entry.Key,
entry => (UnrealTestRoleContext)entry.Value.Clone());
return(Copy);
}
public override void CreateReport(TestResult Result, UnrealTestContext Context, UnrealBuildSource Build, IEnumerable <UnrealRoleArtifacts> Artifacts, string ArtifactPath)
{
if (Result != TestResult.Passed)
{
return;
}
// Gather results and merge PGO data
Log.Info("Gathering profiling results...");
PGOPlatform.GatherResults(TestInstance.ClientApps[0].ArtifactPath);
}
/// <summary>
/// Get a ConfigHierarchy of the type you want with platform-specific config for the given Role. This object can be used to read .ini config values.
/// Default params return the client platform's game config.
/// This looks for the workspace files for the project that you are trying to run your test on. The config directory for that project must exist to find valid results.
/// </summary>
public static ConfigHierarchy GetConfigHierarchy(UnrealTestContext TestContext, ConfigHierarchyType ConfigType = ConfigHierarchyType.Game, UnrealTargetRole TargetRole = UnrealTargetRole.Client)
{
string ProjectPath = Path.Combine(Environment.CurrentDirectory, TestContext.BuildInfo.ProjectName);
if (!Directory.Exists(ProjectPath))
{
Log.Warning(string.Format("Directory does not exist at {0}! Returned ConfigHierarchy will not contain any config values. Make sure to sync the config directory for the project you are trying to run.", ProjectPath));
}
return(ConfigCache.ReadHierarchy(ConfigType, new DirectoryReference(ProjectPath), TestContext.GetRoleContext(TargetRole).Platform));
}
// End UnrealTestNode properties and members
// UnrealTestNode member functions
public UnrealTestNode(UnrealTestContext InContext)
{
Context = InContext;
UnrealTestResult = TestResult.Invalid;
MissingProcesses = new List <IAppInstance>();
TimeToWaitForProcesses = 5;
LastLogCount = 0;
CurrentPass = 0;
NumPasses = 0;
}
public UnrealTestNode(UnrealTestContext InContext)
{
Context = InContext;
UnrealTestResult = TestResult.Invalid;
MissingProcesses = new List <IAppInstance>();
TimeToWaitForProcesses = 5;
LastLogCount = 0;
CurrentPass = 0;
NumPasses = 0;
TestVersion = new Version("1.0.0");
ArtifactPath = string.Empty;
}
/// <summary>
/// Called when report creation fails, by default logs warning with failure message
/// </summary>
protected virtual void HandleCreateReportFailure(UnrealTestContext Context, string Message = "")
{
if (string.IsNullOrEmpty(Message))
{
Message = string.Format("See Gauntlet.log for details");
}
if (Globals.IsWorker)
{
// log for worker to parse context
Log.Info("GauntletWorker:CreateReportFailure:{0}", Context.WorkerJobID);
}
Log.Warning("CreateReport Failed: {0}", Message);
}
/// <summary>
/// Saves all artifacts from the provided session to the specified output path.
/// </summary>
/// <param name="Context"></param>
/// <param name="TestInstance"></param>
/// <param name="OutputPath"></param>
/// <returns></returns>
public IEnumerable <UnrealRoleArtifacts> SaveRoleArtifacts(UnrealTestContext Context, UnrealSessionInstance TestInstance, string OutputPath)
{
Dictionary <UnrealTargetRole, int> RoleCounts = new Dictionary <UnrealTargetRole, int>();
int DummyClientCount = 0;
List <UnrealRoleArtifacts> AllArtifacts = new List <UnrealRoleArtifacts>();
foreach (UnrealSessionInstance.RoleInstance App in TestInstance.RunningRoles)
{
string RoleName = (App.Role.IsDummy() ? "Dummy" : "") + App.Role.RoleType.ToString();
string FolderName = RoleName;
int RoleCount = 1;
if (App.Role.IsDummy())
{
DummyClientCount++;
RoleCount = DummyClientCount;
}
else
{
if (!RoleCounts.ContainsKey(App.Role.RoleType))
{
RoleCounts.Add(App.Role.RoleType, 1);
}
else
{
RoleCounts[App.Role.RoleType]++;
}
RoleCount = RoleCounts[App.Role.RoleType];
}
if (RoleCount > 1)
{
FolderName += string.Format("_{0:00}", RoleCount);
}
string DestPath = Path.Combine(OutputPath, FolderName);
var Artifacts = SaveRoleArtifacts(Context, App, DestPath);
AllArtifacts.Add(Artifacts);
}
return(AllArtifacts);
}
/// <summary>
/// Optional function that is called on test completion and gives an opportunity to create a report
/// </summary>
/// <param name="Result"></param>
/// <param name="Contex"></param>
/// <param name="Build"></param>
public virtual void CreateReport(TestResult Result, UnrealTestContext Contex, UnrealBuildSource Build, IEnumerable <UnrealRoleArtifacts> Artifacts, string ArtifactPath)
{
}
/// <summary>
/// Execute all tests according to the provided context
/// </summary>
/// <param name="Context"></param>
/// <returns></returns>
public ExitCode RunTests(UnrealTestOptions ContextOptions)
{
if (ContextOptions.Verbose)
{
Gauntlet.Log.Level = Gauntlet.LogLevel.Verbose;
}
if (ContextOptions.VeryVerbose)
{
Gauntlet.Log.Level = Gauntlet.LogLevel.VeryVerbose;
}
if (ParseParam("log"))
{
if (!Directory.Exists(ContextOptions.LogDir))
{
Directory.CreateDirectory(ContextOptions.LogDir);
}
// include test names and timestamp in log filename as multiple (parallel or sequential) Gauntlet tests may be outputting to same directory
string LogPath = Path.Combine(ContextOptions.LogDir, string.Format("GauntletLog{0}-{1}.txt", ContextOptions.TestList.Aggregate(new StringBuilder(), (SB, T) => SB.AppendFormat("-{0}", T.ToString())).ToString(), DateTime.Now.ToString(@"yyyy.MM.dd.HH.mm.ss")));
Gauntlet.Log.Verbose("Writing Gauntlet log to {0}", LogPath);
Gauntlet.Log.SaveToFile(LogPath);
}
// prune our temp folder
Utils.SystemHelpers.CleanupMarkedDirectories(ContextOptions.TempDir, 7);
if (string.IsNullOrEmpty(ContextOptions.Build))
{
throw new AutomationException("No builds specified. Use -builds=p:\\path\\to\\build");
}
if (typeof(UnrealBuildSource).IsAssignableFrom(ContextOptions.BuildSourceType) == false)
{
throw new AutomationException("Provided BuildSource type does not inherit from UnrealBuildSource");
}
// make -test=none implicit if no test is supplied
if (ContextOptions.TestList.Count == 0)
{
Gauntlet.Log.Info("No test specified, creating default test node");
ContextOptions.TestList.Add(TestRequest.CreateRequest("DefaultTest"));
}
bool EditorForAllRoles = Globals.Params.ParseParam("editor") || string.Equals(Globals.Params.ParseValue("build", ""), "editor", StringComparison.OrdinalIgnoreCase);
if (EditorForAllRoles)
{
Gauntlet.Log.Verbose("Will use Editor for all roles");
}
Dictionary <UnrealTargetRole, UnrealTestRoleContext> RoleContexts = new Dictionary <UnrealTargetRole, UnrealTestRoleContext>();
// Default platform to the current os
UnrealTargetPlatform DefaultPlatform = BuildHostPlatform.Current.Platform;
UnrealTargetConfiguration DefaultConfiguration = UnrealTargetConfiguration.Development;
// todo, pass this in as a BuildSource and remove the COntextOption params specific to finding builds
UnrealBuildSource BuildInfo = (UnrealBuildSource)Activator.CreateInstance(ContextOptions.BuildSourceType, new object[] { ContextOptions.Project, ContextOptions.UsesSharedBuildType, Environment.CurrentDirectory, ContextOptions.Build, ContextOptions.SearchPaths });
// Setup accounts
SetupAccounts();
List <ITestNode> AllTestNodes = new List <ITestNode>();
bool InitializedDevices = false;
HashSet <UnrealTargetPlatform> UsedPlatforms = new HashSet <UnrealTargetPlatform>();
// for all platforms we want to test...
foreach (ArgumentWithParams PlatformWithParams in ContextOptions.PlatformList)
{
string PlatformString = PlatformWithParams.Argument;
// combine global and platform-specific params
Params CombinedParams = new Params(ContextOptions.Params.AllArguments.Concat(PlatformWithParams.AllArguments).ToArray());
UnrealTargetPlatform PlatformType;
if (!Enum.TryParse <UnrealTargetPlatform>(PlatformString, true, out PlatformType))
{
throw new AutomationException("Unable to convert platform '{0}' into an UnrealTargetPlatform", PlatformString);
}
if (!InitializedDevices)
{
// Setup the devices and assign them to the executor
SetupDevices(PlatformType, ContextOptions);
InitializedDevices = true;
}
// Create a context for each process type to operate as
foreach (UnrealTargetRole Type in Enum.GetValues(typeof(UnrealTargetRole)))
{
UnrealTestRoleContext Role = new UnrealTestRoleContext();
// Default to these
Role.Type = Type;
Role.Platform = DefaultPlatform;
Role.Configuration = DefaultConfiguration;
// globally, what was requested (e.g -platform=PS4 -configuration=Shipping)
UnrealTargetPlatform RequestedPlatform = PlatformType;
UnrealTargetConfiguration RequestedConfiguration = ContextOptions.Configuration;
// look for FooConfiguration, FooPlatform overrides.
// e.g. ServerConfiguration, ServerPlatform
string PlatformRoleString = Globals.Params.ParseValue(Type.ToString() + "Platform", null);
string ConfigString = Globals.Params.ParseValue(Type.ToString() + "Configuration", null);
if (string.IsNullOrEmpty(PlatformRoleString) == false)
{
RequestedPlatform = (UnrealTargetPlatform)Enum.Parse(typeof(UnrealTargetPlatform), PlatformRoleString, true);
}
if (string.IsNullOrEmpty(ConfigString) == false)
{
RequestedConfiguration = (UnrealTargetConfiguration)Enum.Parse(typeof(UnrealTargetConfiguration), ConfigString, true);
}
// look for -clientargs= and -editorclient etc
Role.ExtraArgs = Globals.Params.ParseValue(Type.ToString() + "Args", "");
bool UsesEditor = EditorForAllRoles || Globals.Params.ParseParam("Editor" + Type.ToString());
if (UsesEditor)
{
Gauntlet.Log.Verbose("Will use Editor for role {0}", Type);
}
Role.Skip = Globals.Params.ParseParam("Skip" + Type.ToString());
if (Role.Skip)
{
Gauntlet.Log.Verbose("Will use NullPlatform to skip role {0}", Type);
}
// TODO - the below is a bit rigid, but maybe that's good enough since the "actually use the editor.." option
// is specific to clients and servers
// client can override platform and config
if (Type.IsClient())
{
Role.Platform = RequestedPlatform;
Role.Configuration = RequestedConfiguration;
if (UsesEditor)
{
Role.Type = UnrealTargetRole.EditorGame;
Role.Platform = DefaultPlatform;
Role.Configuration = UnrealTargetConfiguration.Development;
}
}
else if (Type.IsServer())
{
// server can only override config
Role.Configuration = RequestedConfiguration;
if (UsesEditor)
{
Role.Type = UnrealTargetRole.EditorServer;
Role.Platform = DefaultPlatform;
Role.Configuration = UnrealTargetConfiguration.Development;
}
}
Gauntlet.Log.Verbose("Mapped Role {0} to RoleContext {1}", Type, Role);
RoleContexts[Type] = Role;
UsedPlatforms.Add(Role.Platform);
}
UnrealTestContext Context = new UnrealTestContext(BuildInfo, RoleContexts, ContextOptions);
IEnumerable <ITestNode> TestNodes = CreateTestList(Context, CombinedParams, PlatformWithParams);
AllTestNodes.AddRange(TestNodes);
}
bool AllTestsPassed = ExecuteTests(ContextOptions, AllTestNodes);
// dispose now, not during shutdown gc, because this runs commands...
DevicePool.Instance.Dispose();
DoCleanup(UsedPlatforms);
return(AllTestsPassed ? ExitCode.Success : ExitCode.Error_TestFailure);
}
/// <summary>
/// Retrieves and saves all artifacts from the provided session role. Artifacts are saved to the destination path
/// </summary>
/// <param name="InContext"></param>
/// <param name="InRunningRole"></param>
/// <param name="InDestArtifactPath"></param>
/// <returns></returns>
public UnrealRoleArtifacts SaveRoleArtifacts(UnrealTestContext InContext, UnrealSessionInstance.RoleInstance InRunningRole, string InDestArtifactPath)
{
bool IsServer = InRunningRole.Role.RoleType.IsServer();
string RoleName = (InRunningRole.Role.IsDummy() ? "Dummy" : "") + InRunningRole.Role.RoleType.ToString();
UnrealTargetPlatform?Platform = InRunningRole.Role.Platform;
string RoleConfig = InRunningRole.Role.Configuration.ToString();
Directory.CreateDirectory(InDestArtifactPath);
// Don't archive editor data, there can be a *lot* of stuff in that saved folder!
bool IsEditor = InRunningRole.Role.RoleType.UsesEditor();
bool IsDevBuild = InContext.TestParams.ParseParam("dev");
string DestSavedDir = Path.Combine(InDestArtifactPath, "Saved");
string SourceSavedDir = "";
// save the contents of the saved directory
SourceSavedDir = InRunningRole.AppInstance.ArtifactPath;
// save the output from TTY
string ArtifactLogPath = Path.Combine(InDestArtifactPath, RoleName + "Output.log");
// Write a brief Gauntlet header to aid debugging
StringBuilder LogOut = new StringBuilder();
LogOut.AppendLine("------ Gauntlet Test ------");
LogOut.AppendFormat("Role: {0}\r\n", InRunningRole.Role);
LogOut.AppendFormat("Automation Command: {0}\r\n", Environment.CommandLine);
LogOut.AppendLine("---------------------------");
// Write instance stdout stream
LogOut.Append(InRunningRole.AppInstance.StdOut);
File.WriteAllText(ArtifactLogPath, LogOut.ToString());
Log.Info("Wrote Log to {0}", ArtifactLogPath);
if (IsServer == false)
{
// gif-ify and jpeg-ify any screenshots
try
{
string ScreenshotPath = Path.Combine(SourceSavedDir, "Screenshots", Platform.ToString()).ToLower();
if (Directory.Exists(ScreenshotPath) && Directory.GetFiles(ScreenshotPath).Length > 0)
{
Log.Info("Downsizing and gifying session images at {0}", ScreenshotPath);
// downsize first so gif-step is quicker and takes less resoruces.
Utils.Image.ConvertImages(ScreenshotPath, ScreenshotPath, "jpg", true);
string GifPath = Path.Combine(InDestArtifactPath, RoleName + "Test.gif");
if (Utils.Image.SaveImagesAsGif(ScreenshotPath, GifPath))
{
Log.Info("Saved gif to {0}", GifPath);
}
}
}
catch (Exception Ex)
{
Log.Info("Failed to downsize and gif-ify images! {0}", Ex);
}
}
// don't archive data in dev mode, because peoples saved data could be huuuuuuuge!
if (IsEditor == false)
{
LogLevel OldLevel = Log.Level;
Log.Level = LogLevel.Normal;
if (Directory.Exists(SourceSavedDir))
{
Utils.SystemHelpers.CopyDirectory(SourceSavedDir, DestSavedDir);
Log.Info("Archived artifacts to to {0}", DestSavedDir);
}
else
{
Log.Info("Archive path '{0}' was not found!", SourceSavedDir);
}
Log.Level = OldLevel;
}
else
{
if (IsEditor)
{
Log.Info("Skipping archival of assets for editor {0}", RoleName);
}
else if (IsDevBuild)
{
Log.Info("Skipping archival of assets for dev build");
}
}
foreach (EIntendedBaseCopyDirectory ArtifactDir in InRunningRole.Role.AdditionalArtifactDirectories)
{
if (InRunningRole.AppInstance.Device.GetPlatformDirectoryMappings().ContainsKey(ArtifactDir))
{
string SourcePath = InRunningRole.AppInstance.Device.GetPlatformDirectoryMappings()[ArtifactDir];
var DirToCopy = new DirectoryInfo(SourcePath);
if (DirToCopy.Exists)
{
// Grab the final dir name to copy everything into so everything's not just going into root artifact dir.
string IntendedCopyLocation = Path.Combine(InDestArtifactPath, DirToCopy.Name);
Utils.SystemHelpers.CopyDirectory(SourcePath, IntendedCopyLocation);
}
}
}
// TODO REMOVEME- this should go elsewhere, likely a util that can be called or inserted by relevant test nodes.
if (IsServer == false)
{
// Copy over PSOs
try
{
if (InContext.Options.LogPSO)
{
foreach (var ThisFile in CommandUtils.FindFiles_NoExceptions(true, "*.rec.upipelinecache", true, DestSavedDir))
{
bool Copied = false;
var JustFile = Path.GetFileName(ThisFile);
if (JustFile.StartsWith("++"))
{
var Parts = JustFile.Split(new Char[] { '+', '-' }).Where(A => A != "").ToArray();
if (Parts.Count() >= 2)
{
string ProjectName = Parts[0].ToString();
string BuildRoot = CommandUtils.CombinePaths(CommandUtils.RootBuildStorageDirectory());
string SrcBuildPath = CommandUtils.CombinePaths(BuildRoot, ProjectName);
string SrcBuildPath2 = CommandUtils.CombinePaths(BuildRoot, ProjectName.Replace("Game", "").Replace("game", ""));
if (!CommandUtils.DirectoryExists(SrcBuildPath))
{
SrcBuildPath = SrcBuildPath2;
}
if (CommandUtils.DirectoryExists(SrcBuildPath))
{
var JustBuildFolder = JustFile.Replace("-" + Parts.Last(), "");
string PlatformStr = Platform.ToString();
string SrcCLMetaPath = CommandUtils.CombinePaths(SrcBuildPath, JustBuildFolder, PlatformStr, "MetaData");
if (CommandUtils.DirectoryExists(SrcCLMetaPath))
{
string SrcCLMetaPathCollected = CommandUtils.CombinePaths(SrcCLMetaPath, "CollectedPSOs");
if (!CommandUtils.DirectoryExists(SrcCLMetaPathCollected))
{
Log.Info("Creating Directory {0}", SrcCLMetaPathCollected);
CommandUtils.CreateDirectory(SrcCLMetaPathCollected);
}
if (CommandUtils.DirectoryExists(SrcCLMetaPathCollected))
{
string DestFile = CommandUtils.CombinePaths(SrcCLMetaPathCollected, JustFile);
CommandUtils.CopyFile_NoExceptions(ThisFile, DestFile, true);
if (CommandUtils.FileExists(true, DestFile))
{
Log.Info("Deleting local file, copied to {0}", DestFile);
CommandUtils.DeleteFile_NoExceptions(ThisFile, true);
Copied = true;
}
}
}
}
}
}
if (!Copied)
{
Log.Warning("Could not find anywhere to put this file {0}", JustFile);
}
}
}
}
catch (Exception Ex)
{
Log.Info("Failed to copy upipelinecaches to the network {0}", Ex);
}
}
// END REMOVEME
UnrealLogParser LogParser = new UnrealLogParser(InRunningRole.AppInstance.StdOut);
int ExitCode = InRunningRole.AppInstance.ExitCode;
LogParser.GetTestExitCode(out ExitCode);
UnrealRoleArtifacts Artifacts = new UnrealRoleArtifacts(InRunningRole.Role, InRunningRole.AppInstance, InDestArtifactPath, ArtifactLogPath, LogParser);
return(Artifacts);
}
public BootTest(Gauntlet.UnrealTestContext InContext)
: base(InContext)
{
}
public RunAutomatedTests(Gauntlet.UnrealTestContext InContext)
: base(InContext)
{
}
public Benchmark(Gauntlet.UnrealTestContext InContext)
: base(InContext)
{
}
public FortGPUTestbedPerfTest(Gauntlet.UnrealTestContext InContext)
: base(InContext)
{
}
/// <summary>
/// Optional function that is called on test completion and gives an opportunity to create a report
/// </summary>
/// <param name="Result"></param>
/// <param name="Contex"></param>
/// <param name="Build"></param>
public virtual void SubmitToDashboard(TestResult Result, UnrealTestContext Contex, UnrealBuildSource Build, IEnumerable <UnrealRoleArtifacts> Artifacts, string ArtifactPath)
{
}
public PGONode(UnrealTestContext InContext) : base(InContext)
{
}
public ElementalDemoTest(Gauntlet.UnrealTestContext InContext)
: base(InContext)
{
}