/// <summary> /// Saves all artifacts from the provided session to the specified output path. /// </summary> /// <param name="Context"></param> /// <param name="TestInstance"></param> /// <param name="OutputPath"></param> /// <returns></returns> public IEnumerable <UnrealRoleArtifacts> SaveRoleArtifacts(UnrealTestContext Context, UnrealSessionInstance TestInstance, string OutputPath) { int ClientCount = 1; int DummyClientCount = 1; List <UnrealRoleArtifacts> AllArtifacts = new List <UnrealRoleArtifacts>(); foreach (UnrealSessionInstance.RoleInstance App in TestInstance.RunningRoles) { bool IsServer = App.Role.RoleType.IsServer(); string RoleName = (App.Role.IsDummy() ? "Dummy" : "") + App.Role.RoleType.ToString(); string FolderName = RoleName; if (IsServer == false) { if ((!App.Role.IsDummy() && ClientCount++ > 1)) { FolderName += string.Format("_{0:00}", ClientCount); } else if (App.Role.IsDummy() && DummyClientCount++ > 1) { FolderName += string.Format("_{0:00}", DummyClientCount); } } string DestPath = Path.Combine(OutputPath, FolderName); var Artifacts = SaveRoleArtifacts(Context, App, DestPath); AllArtifacts.Add(Artifacts); } return(AllArtifacts); }
/// <summary> /// Create the list of tests specified by the context. /// </summary> /// <param name="Context"></param> /// <returns></returns> IEnumerable <ITestNode> CreateTestList(UnrealTestContext Context, Params DefaultParams, ArgumentWithParams PlatformParams = null) { List <ITestNode> NodeList = new List <ITestNode>(); IEnumerable <string> Namespaces = Context.Options.Namespaces.Split(',').Select(S => S.Trim()); List <string> BuildIssues = new List <string>(); UnrealTargetPlatform UnrealPlatform = UnrealTargetPlatform.Unknown; if (!Enum.TryParse(PlatformParams.Argument, true, out UnrealPlatform)) { throw new AutomationException("Could not convert platform {0} to a valid UnrealTargetPlatform", PlatformParams.Argument); } //List<string> Platforms = Globals.Params.ParseValue("platform") // Create an instance of each test and add it to the executor foreach (var Test in Context.Options.TestList) { // create a copy of the context for this test UnrealTestContext TestContext = (UnrealTestContext)Context.Clone(); // if test specifies platforms, filter for this context if (Test.Platforms.Count() > 0 && Test.Platforms.Where(Plat => Plat.Argument == PlatformParams.Argument).Count() == 0) { continue; } if (Blacklist.Instance.IsTestBlacklisted(Test.TestName, UnrealPlatform, TestContext.BuildInfo.Branch)) { Gauntlet.Log.Info("Test {0} is currently blacklisted on {1} in branch {2}", Test.TestName, UnrealPlatform, TestContext.BuildInfo.Branch); continue; } // combine global and test-specific params Params CombinedParams = new Params(DefaultParams.AllArguments.Concat(Test.TestParams.AllArguments).ToArray()); // parse any target constraints List <string> PerfSpecArgs = CombinedParams.ParseValues("PerfSpec", false); string PerfSpecArg = PerfSpecArgs.Count > 0 ? PerfSpecArgs.Last() : "Unspecified"; EPerfSpec PerfSpec; if (!Enum.TryParse <EPerfSpec>(PerfSpecArg, true, out PerfSpec)) { throw new AutomationException("Unable to convert perfspec '{0}' into an EPerfSpec", PerfSpec); } TestContext.Constraint = new UnrealTargetConstraint(UnrealPlatform, PerfSpec); TestContext.TestParams = CombinedParams; // This will throw if the test cannot be created ITestNode NewTest = Utils.TestConstructor.ConstructTest <ITestNode, UnrealTestContext>(Test.TestName, TestContext, Namespaces); NodeList.Add(NewTest); } return(NodeList); }
public object Clone() { UnrealTestContext Copy = (UnrealTestContext)this.MemberwiseClone(); // todo - what else shou;d be unique? Copy.RoleContext = this.RoleContext.ToDictionary(entry => entry.Key, entry => (UnrealTestRoleContext)entry.Value.Clone()); return(Copy); }
public override void CreateReport(TestResult Result, UnrealTestContext Context, UnrealBuildSource Build, IEnumerable <UnrealRoleArtifacts> Artifacts, string ArtifactPath) { if (Result != TestResult.Passed) { return; } // Gather results and merge PGO data Log.Info("Gathering profiling results..."); PGOPlatform.GatherResults(TestInstance.ClientApps[0].ArtifactPath); }
/// <summary> /// Get a ConfigHierarchy of the type you want with platform-specific config for the given Role. This object can be used to read .ini config values. /// Default params return the client platform's game config. /// This looks for the workspace files for the project that you are trying to run your test on. The config directory for that project must exist to find valid results. /// </summary> public static ConfigHierarchy GetConfigHierarchy(UnrealTestContext TestContext, ConfigHierarchyType ConfigType = ConfigHierarchyType.Game, UnrealTargetRole TargetRole = UnrealTargetRole.Client) { string ProjectPath = Path.Combine(Environment.CurrentDirectory, TestContext.BuildInfo.ProjectName); if (!Directory.Exists(ProjectPath)) { Log.Warning(string.Format("Directory does not exist at {0}! Returned ConfigHierarchy will not contain any config values. Make sure to sync the config directory for the project you are trying to run.", ProjectPath)); } return(ConfigCache.ReadHierarchy(ConfigType, new DirectoryReference(ProjectPath), TestContext.GetRoleContext(TargetRole).Platform)); }
// End UnrealTestNode properties and members // UnrealTestNode member functions public UnrealTestNode(UnrealTestContext InContext) { Context = InContext; UnrealTestResult = TestResult.Invalid; MissingProcesses = new List <IAppInstance>(); TimeToWaitForProcesses = 5; LastLogCount = 0; CurrentPass = 0; NumPasses = 0; }
public UnrealTestNode(UnrealTestContext InContext) { Context = InContext; UnrealTestResult = TestResult.Invalid; MissingProcesses = new List <IAppInstance>(); TimeToWaitForProcesses = 5; LastLogCount = 0; CurrentPass = 0; NumPasses = 0; TestVersion = new Version("1.0.0"); ArtifactPath = string.Empty; }
/// <summary> /// Called when report creation fails, by default logs warning with failure message /// </summary> protected virtual void HandleCreateReportFailure(UnrealTestContext Context, string Message = "") { if (string.IsNullOrEmpty(Message)) { Message = string.Format("See Gauntlet.log for details"); } if (Globals.IsWorker) { // log for worker to parse context Log.Info("GauntletWorker:CreateReportFailure:{0}", Context.WorkerJobID); } Log.Warning("CreateReport Failed: {0}", Message); }
/// <summary> /// Saves all artifacts from the provided session to the specified output path. /// </summary> /// <param name="Context"></param> /// <param name="TestInstance"></param> /// <param name="OutputPath"></param> /// <returns></returns> public IEnumerable <UnrealRoleArtifacts> SaveRoleArtifacts(UnrealTestContext Context, UnrealSessionInstance TestInstance, string OutputPath) { Dictionary <UnrealTargetRole, int> RoleCounts = new Dictionary <UnrealTargetRole, int>(); int DummyClientCount = 0; List <UnrealRoleArtifacts> AllArtifacts = new List <UnrealRoleArtifacts>(); foreach (UnrealSessionInstance.RoleInstance App in TestInstance.RunningRoles) { string RoleName = (App.Role.IsDummy() ? "Dummy" : "") + App.Role.RoleType.ToString(); string FolderName = RoleName; int RoleCount = 1; if (App.Role.IsDummy()) { DummyClientCount++; RoleCount = DummyClientCount; } else { if (!RoleCounts.ContainsKey(App.Role.RoleType)) { RoleCounts.Add(App.Role.RoleType, 1); } else { RoleCounts[App.Role.RoleType]++; } RoleCount = RoleCounts[App.Role.RoleType]; } if (RoleCount > 1) { FolderName += string.Format("_{0:00}", RoleCount); } string DestPath = Path.Combine(OutputPath, FolderName); var Artifacts = SaveRoleArtifacts(Context, App, DestPath); AllArtifacts.Add(Artifacts); } return(AllArtifacts); }
/// <summary> /// Optional function that is called on test completion and gives an opportunity to create a report /// </summary> /// <param name="Result"></param> /// <param name="Contex"></param> /// <param name="Build"></param> public virtual void CreateReport(TestResult Result, UnrealTestContext Contex, UnrealBuildSource Build, IEnumerable <UnrealRoleArtifacts> Artifacts, string ArtifactPath) { }
/// <summary> /// Execute all tests according to the provided context /// </summary> /// <param name="Context"></param> /// <returns></returns> public ExitCode RunTests(UnrealTestOptions ContextOptions) { if (ContextOptions.Verbose) { Gauntlet.Log.Level = Gauntlet.LogLevel.Verbose; } if (ContextOptions.VeryVerbose) { Gauntlet.Log.Level = Gauntlet.LogLevel.VeryVerbose; } if (ParseParam("log")) { if (!Directory.Exists(ContextOptions.LogDir)) { Directory.CreateDirectory(ContextOptions.LogDir); } // include test names and timestamp in log filename as multiple (parallel or sequential) Gauntlet tests may be outputting to same directory string LogPath = Path.Combine(ContextOptions.LogDir, string.Format("GauntletLog{0}-{1}.txt", ContextOptions.TestList.Aggregate(new StringBuilder(), (SB, T) => SB.AppendFormat("-{0}", T.ToString())).ToString(), DateTime.Now.ToString(@"yyyy.MM.dd.HH.mm.ss"))); Gauntlet.Log.Verbose("Writing Gauntlet log to {0}", LogPath); Gauntlet.Log.SaveToFile(LogPath); } // prune our temp folder Utils.SystemHelpers.CleanupMarkedDirectories(ContextOptions.TempDir, 7); if (string.IsNullOrEmpty(ContextOptions.Build)) { throw new AutomationException("No builds specified. Use -builds=p:\\path\\to\\build"); } if (typeof(UnrealBuildSource).IsAssignableFrom(ContextOptions.BuildSourceType) == false) { throw new AutomationException("Provided BuildSource type does not inherit from UnrealBuildSource"); } // make -test=none implicit if no test is supplied if (ContextOptions.TestList.Count == 0) { Gauntlet.Log.Info("No test specified, creating default test node"); ContextOptions.TestList.Add(TestRequest.CreateRequest("DefaultTest")); } bool EditorForAllRoles = Globals.Params.ParseParam("editor") || string.Equals(Globals.Params.ParseValue("build", ""), "editor", StringComparison.OrdinalIgnoreCase); if (EditorForAllRoles) { Gauntlet.Log.Verbose("Will use Editor for all roles"); } Dictionary <UnrealTargetRole, UnrealTestRoleContext> RoleContexts = new Dictionary <UnrealTargetRole, UnrealTestRoleContext>(); // Default platform to the current os UnrealTargetPlatform DefaultPlatform = BuildHostPlatform.Current.Platform; UnrealTargetConfiguration DefaultConfiguration = UnrealTargetConfiguration.Development; // todo, pass this in as a BuildSource and remove the COntextOption params specific to finding builds UnrealBuildSource BuildInfo = (UnrealBuildSource)Activator.CreateInstance(ContextOptions.BuildSourceType, new object[] { ContextOptions.Project, ContextOptions.UsesSharedBuildType, Environment.CurrentDirectory, ContextOptions.Build, ContextOptions.SearchPaths }); // Setup accounts SetupAccounts(); List <ITestNode> AllTestNodes = new List <ITestNode>(); bool InitializedDevices = false; HashSet <UnrealTargetPlatform> UsedPlatforms = new HashSet <UnrealTargetPlatform>(); // for all platforms we want to test... foreach (ArgumentWithParams PlatformWithParams in ContextOptions.PlatformList) { string PlatformString = PlatformWithParams.Argument; // combine global and platform-specific params Params CombinedParams = new Params(ContextOptions.Params.AllArguments.Concat(PlatformWithParams.AllArguments).ToArray()); UnrealTargetPlatform PlatformType; if (!Enum.TryParse <UnrealTargetPlatform>(PlatformString, true, out PlatformType)) { throw new AutomationException("Unable to convert platform '{0}' into an UnrealTargetPlatform", PlatformString); } if (!InitializedDevices) { // Setup the devices and assign them to the executor SetupDevices(PlatformType, ContextOptions); InitializedDevices = true; } // Create a context for each process type to operate as foreach (UnrealTargetRole Type in Enum.GetValues(typeof(UnrealTargetRole))) { UnrealTestRoleContext Role = new UnrealTestRoleContext(); // Default to these Role.Type = Type; Role.Platform = DefaultPlatform; Role.Configuration = DefaultConfiguration; // globally, what was requested (e.g -platform=PS4 -configuration=Shipping) UnrealTargetPlatform RequestedPlatform = PlatformType; UnrealTargetConfiguration RequestedConfiguration = ContextOptions.Configuration; // look for FooConfiguration, FooPlatform overrides. // e.g. ServerConfiguration, ServerPlatform string PlatformRoleString = Globals.Params.ParseValue(Type.ToString() + "Platform", null); string ConfigString = Globals.Params.ParseValue(Type.ToString() + "Configuration", null); if (string.IsNullOrEmpty(PlatformRoleString) == false) { RequestedPlatform = (UnrealTargetPlatform)Enum.Parse(typeof(UnrealTargetPlatform), PlatformRoleString, true); } if (string.IsNullOrEmpty(ConfigString) == false) { RequestedConfiguration = (UnrealTargetConfiguration)Enum.Parse(typeof(UnrealTargetConfiguration), ConfigString, true); } // look for -clientargs= and -editorclient etc Role.ExtraArgs = Globals.Params.ParseValue(Type.ToString() + "Args", ""); bool UsesEditor = EditorForAllRoles || Globals.Params.ParseParam("Editor" + Type.ToString()); if (UsesEditor) { Gauntlet.Log.Verbose("Will use Editor for role {0}", Type); } Role.Skip = Globals.Params.ParseParam("Skip" + Type.ToString()); if (Role.Skip) { Gauntlet.Log.Verbose("Will use NullPlatform to skip role {0}", Type); } // TODO - the below is a bit rigid, but maybe that's good enough since the "actually use the editor.." option // is specific to clients and servers // client can override platform and config if (Type.IsClient()) { Role.Platform = RequestedPlatform; Role.Configuration = RequestedConfiguration; if (UsesEditor) { Role.Type = UnrealTargetRole.EditorGame; Role.Platform = DefaultPlatform; Role.Configuration = UnrealTargetConfiguration.Development; } } else if (Type.IsServer()) { // server can only override config Role.Configuration = RequestedConfiguration; if (UsesEditor) { Role.Type = UnrealTargetRole.EditorServer; Role.Platform = DefaultPlatform; Role.Configuration = UnrealTargetConfiguration.Development; } } Gauntlet.Log.Verbose("Mapped Role {0} to RoleContext {1}", Type, Role); RoleContexts[Type] = Role; UsedPlatforms.Add(Role.Platform); } UnrealTestContext Context = new UnrealTestContext(BuildInfo, RoleContexts, ContextOptions); IEnumerable <ITestNode> TestNodes = CreateTestList(Context, CombinedParams, PlatformWithParams); AllTestNodes.AddRange(TestNodes); } bool AllTestsPassed = ExecuteTests(ContextOptions, AllTestNodes); // dispose now, not during shutdown gc, because this runs commands... DevicePool.Instance.Dispose(); DoCleanup(UsedPlatforms); return(AllTestsPassed ? ExitCode.Success : ExitCode.Error_TestFailure); }
/// <summary> /// Retrieves and saves all artifacts from the provided session role. Artifacts are saved to the destination path /// </summary> /// <param name="InContext"></param> /// <param name="InRunningRole"></param> /// <param name="InDestArtifactPath"></param> /// <returns></returns> public UnrealRoleArtifacts SaveRoleArtifacts(UnrealTestContext InContext, UnrealSessionInstance.RoleInstance InRunningRole, string InDestArtifactPath) { bool IsServer = InRunningRole.Role.RoleType.IsServer(); string RoleName = (InRunningRole.Role.IsDummy() ? "Dummy" : "") + InRunningRole.Role.RoleType.ToString(); UnrealTargetPlatform?Platform = InRunningRole.Role.Platform; string RoleConfig = InRunningRole.Role.Configuration.ToString(); Directory.CreateDirectory(InDestArtifactPath); // Don't archive editor data, there can be a *lot* of stuff in that saved folder! bool IsEditor = InRunningRole.Role.RoleType.UsesEditor(); bool IsDevBuild = InContext.TestParams.ParseParam("dev"); string DestSavedDir = Path.Combine(InDestArtifactPath, "Saved"); string SourceSavedDir = ""; // save the contents of the saved directory SourceSavedDir = InRunningRole.AppInstance.ArtifactPath; // save the output from TTY string ArtifactLogPath = Path.Combine(InDestArtifactPath, RoleName + "Output.log"); // Write a brief Gauntlet header to aid debugging StringBuilder LogOut = new StringBuilder(); LogOut.AppendLine("------ Gauntlet Test ------"); LogOut.AppendFormat("Role: {0}\r\n", InRunningRole.Role); LogOut.AppendFormat("Automation Command: {0}\r\n", Environment.CommandLine); LogOut.AppendLine("---------------------------"); // Write instance stdout stream LogOut.Append(InRunningRole.AppInstance.StdOut); File.WriteAllText(ArtifactLogPath, LogOut.ToString()); Log.Info("Wrote Log to {0}", ArtifactLogPath); if (IsServer == false) { // gif-ify and jpeg-ify any screenshots try { string ScreenshotPath = Path.Combine(SourceSavedDir, "Screenshots", Platform.ToString()).ToLower(); if (Directory.Exists(ScreenshotPath) && Directory.GetFiles(ScreenshotPath).Length > 0) { Log.Info("Downsizing and gifying session images at {0}", ScreenshotPath); // downsize first so gif-step is quicker and takes less resoruces. Utils.Image.ConvertImages(ScreenshotPath, ScreenshotPath, "jpg", true); string GifPath = Path.Combine(InDestArtifactPath, RoleName + "Test.gif"); if (Utils.Image.SaveImagesAsGif(ScreenshotPath, GifPath)) { Log.Info("Saved gif to {0}", GifPath); } } } catch (Exception Ex) { Log.Info("Failed to downsize and gif-ify images! {0}", Ex); } } // don't archive data in dev mode, because peoples saved data could be huuuuuuuge! if (IsEditor == false) { LogLevel OldLevel = Log.Level; Log.Level = LogLevel.Normal; if (Directory.Exists(SourceSavedDir)) { Utils.SystemHelpers.CopyDirectory(SourceSavedDir, DestSavedDir); Log.Info("Archived artifacts to to {0}", DestSavedDir); } else { Log.Info("Archive path '{0}' was not found!", SourceSavedDir); } Log.Level = OldLevel; } else { if (IsEditor) { Log.Info("Skipping archival of assets for editor {0}", RoleName); } else if (IsDevBuild) { Log.Info("Skipping archival of assets for dev build"); } } foreach (EIntendedBaseCopyDirectory ArtifactDir in InRunningRole.Role.AdditionalArtifactDirectories) { if (InRunningRole.AppInstance.Device.GetPlatformDirectoryMappings().ContainsKey(ArtifactDir)) { string SourcePath = InRunningRole.AppInstance.Device.GetPlatformDirectoryMappings()[ArtifactDir]; var DirToCopy = new DirectoryInfo(SourcePath); if (DirToCopy.Exists) { // Grab the final dir name to copy everything into so everything's not just going into root artifact dir. string IntendedCopyLocation = Path.Combine(InDestArtifactPath, DirToCopy.Name); Utils.SystemHelpers.CopyDirectory(SourcePath, IntendedCopyLocation); } } } // TODO REMOVEME- this should go elsewhere, likely a util that can be called or inserted by relevant test nodes. if (IsServer == false) { // Copy over PSOs try { if (InContext.Options.LogPSO) { foreach (var ThisFile in CommandUtils.FindFiles_NoExceptions(true, "*.rec.upipelinecache", true, DestSavedDir)) { bool Copied = false; var JustFile = Path.GetFileName(ThisFile); if (JustFile.StartsWith("++")) { var Parts = JustFile.Split(new Char[] { '+', '-' }).Where(A => A != "").ToArray(); if (Parts.Count() >= 2) { string ProjectName = Parts[0].ToString(); string BuildRoot = CommandUtils.CombinePaths(CommandUtils.RootBuildStorageDirectory()); string SrcBuildPath = CommandUtils.CombinePaths(BuildRoot, ProjectName); string SrcBuildPath2 = CommandUtils.CombinePaths(BuildRoot, ProjectName.Replace("Game", "").Replace("game", "")); if (!CommandUtils.DirectoryExists(SrcBuildPath)) { SrcBuildPath = SrcBuildPath2; } if (CommandUtils.DirectoryExists(SrcBuildPath)) { var JustBuildFolder = JustFile.Replace("-" + Parts.Last(), ""); string PlatformStr = Platform.ToString(); string SrcCLMetaPath = CommandUtils.CombinePaths(SrcBuildPath, JustBuildFolder, PlatformStr, "MetaData"); if (CommandUtils.DirectoryExists(SrcCLMetaPath)) { string SrcCLMetaPathCollected = CommandUtils.CombinePaths(SrcCLMetaPath, "CollectedPSOs"); if (!CommandUtils.DirectoryExists(SrcCLMetaPathCollected)) { Log.Info("Creating Directory {0}", SrcCLMetaPathCollected); CommandUtils.CreateDirectory(SrcCLMetaPathCollected); } if (CommandUtils.DirectoryExists(SrcCLMetaPathCollected)) { string DestFile = CommandUtils.CombinePaths(SrcCLMetaPathCollected, JustFile); CommandUtils.CopyFile_NoExceptions(ThisFile, DestFile, true); if (CommandUtils.FileExists(true, DestFile)) { Log.Info("Deleting local file, copied to {0}", DestFile); CommandUtils.DeleteFile_NoExceptions(ThisFile, true); Copied = true; } } } } } } if (!Copied) { Log.Warning("Could not find anywhere to put this file {0}", JustFile); } } } } catch (Exception Ex) { Log.Info("Failed to copy upipelinecaches to the network {0}", Ex); } } // END REMOVEME UnrealLogParser LogParser = new UnrealLogParser(InRunningRole.AppInstance.StdOut); int ExitCode = InRunningRole.AppInstance.ExitCode; LogParser.GetTestExitCode(out ExitCode); UnrealRoleArtifacts Artifacts = new UnrealRoleArtifacts(InRunningRole.Role, InRunningRole.AppInstance, InDestArtifactPath, ArtifactLogPath, LogParser); return(Artifacts); }
public BootTest(Gauntlet.UnrealTestContext InContext) : base(InContext) { }
public RunAutomatedTests(Gauntlet.UnrealTestContext InContext) : base(InContext) { }
public Benchmark(Gauntlet.UnrealTestContext InContext) : base(InContext) { }
public FortGPUTestbedPerfTest(Gauntlet.UnrealTestContext InContext) : base(InContext) { }
/// <summary> /// Optional function that is called on test completion and gives an opportunity to create a report /// </summary> /// <param name="Result"></param> /// <param name="Contex"></param> /// <param name="Build"></param> public virtual void SubmitToDashboard(TestResult Result, UnrealTestContext Contex, UnrealBuildSource Build, IEnumerable <UnrealRoleArtifacts> Artifacts, string ArtifactPath) { }
public PGONode(UnrealTestContext InContext) : base(InContext) { }
public ElementalDemoTest(Gauntlet.UnrealTestContext InContext) : base(InContext) { }