/// <summary> /// Execute all tests according to the provided context /// </summary> /// <param name="Context"></param> /// <returns></returns> public ExitCode RunTests(UnrealTestOptions ContextOptions) { if (ContextOptions.Verbose) { Gauntlet.Log.Level = Gauntlet.LogLevel.Verbose; } if (ContextOptions.VeryVerbose) { Gauntlet.Log.Level = Gauntlet.LogLevel.VeryVerbose; } if (ParseParam("log")) { if (!Directory.Exists(ContextOptions.LogDir)) { Directory.CreateDirectory(ContextOptions.LogDir); } // include test names and timestamp in log filename as multiple (parallel or sequential) Gauntlet tests may be outputting to same directory string LogPath = Path.Combine(ContextOptions.LogDir, string.Format("GauntletLog{0}-{1}.txt", ContextOptions.TestList.Aggregate(new StringBuilder(), (SB, T) => SB.AppendFormat("-{0}", T.ToString())).ToString(), DateTime.Now.ToString(@"yyyy.MM.dd.HH.mm.ss"))); Gauntlet.Log.Verbose("Writing Gauntlet log to {0}", LogPath); Gauntlet.Log.SaveToFile(LogPath); } // prune our temp folder Utils.SystemHelpers.CleanupMarkedDirectories(ContextOptions.TempDir, 7); if (string.IsNullOrEmpty(ContextOptions.Build)) { throw new AutomationException("No builds specified. Use -builds=p:\\path\\to\\build"); } if (typeof(UnrealBuildSource).IsAssignableFrom(ContextOptions.BuildSourceType) == false) { throw new AutomationException("Provided BuildSource type does not inherit from UnrealBuildSource"); } // make -test=none implicit if no test is supplied if (ContextOptions.TestList.Count == 0) { Gauntlet.Log.Info("No test specified, creating default test node"); ContextOptions.TestList.Add(TestRequest.CreateRequest("DefaultTest")); } bool EditorForAllRoles = Globals.Params.ParseParam("editor") || string.Equals(Globals.Params.ParseValue("build", ""), "editor", StringComparison.OrdinalIgnoreCase); if (EditorForAllRoles) { Gauntlet.Log.Verbose("Will use Editor for all roles"); } Dictionary <UnrealTargetRole, UnrealTestRoleContext> RoleContexts = new Dictionary <UnrealTargetRole, UnrealTestRoleContext>(); // Default platform to the current os UnrealTargetPlatform DefaultPlatform = BuildHostPlatform.Current.Platform; UnrealTargetConfiguration DefaultConfiguration = UnrealTargetConfiguration.Development; // todo, pass this in as a BuildSource and remove the COntextOption params specific to finding builds UnrealBuildSource BuildInfo = (UnrealBuildSource)Activator.CreateInstance(ContextOptions.BuildSourceType, new object[] { ContextOptions.Project, ContextOptions.UsesSharedBuildType, Environment.CurrentDirectory, ContextOptions.Build, ContextOptions.SearchPaths }); // Setup accounts SetupAccounts(); List <ITestNode> AllTestNodes = new List <ITestNode>(); bool InitializedDevices = false; HashSet <UnrealTargetPlatform> UsedPlatforms = new HashSet <UnrealTargetPlatform>(); // for all platforms we want to test... foreach (ArgumentWithParams PlatformWithParams in ContextOptions.PlatformList) { string PlatformString = PlatformWithParams.Argument; // combine global and platform-specific params Params CombinedParams = new Params(ContextOptions.Params.AllArguments.Concat(PlatformWithParams.AllArguments).ToArray()); UnrealTargetPlatform PlatformType; if (!Enum.TryParse <UnrealTargetPlatform>(PlatformString, true, out PlatformType)) { throw new AutomationException("Unable to convert platform '{0}' into an UnrealTargetPlatform", PlatformString); } if (!InitializedDevices) { // Setup the devices and assign them to the executor SetupDevices(PlatformType, ContextOptions); InitializedDevices = true; } // Create a context for each process type to operate as foreach (UnrealTargetRole Type in Enum.GetValues(typeof(UnrealTargetRole))) { UnrealTestRoleContext Role = new UnrealTestRoleContext(); // Default to these Role.Type = Type; Role.Platform = DefaultPlatform; Role.Configuration = DefaultConfiguration; // globally, what was requested (e.g -platform=PS4 -configuration=Shipping) UnrealTargetPlatform RequestedPlatform = PlatformType; UnrealTargetConfiguration RequestedConfiguration = ContextOptions.Configuration; // look for FooConfiguration, FooPlatform overrides. // e.g. ServerConfiguration, ServerPlatform string PlatformRoleString = Globals.Params.ParseValue(Type.ToString() + "Platform", null); string ConfigString = Globals.Params.ParseValue(Type.ToString() + "Configuration", null); if (string.IsNullOrEmpty(PlatformRoleString) == false) { RequestedPlatform = (UnrealTargetPlatform)Enum.Parse(typeof(UnrealTargetPlatform), PlatformRoleString, true); } if (string.IsNullOrEmpty(ConfigString) == false) { RequestedConfiguration = (UnrealTargetConfiguration)Enum.Parse(typeof(UnrealTargetConfiguration), ConfigString, true); } // look for -clientargs= and -editorclient etc Role.ExtraArgs = Globals.Params.ParseValue(Type.ToString() + "Args", ""); bool UsesEditor = EditorForAllRoles || Globals.Params.ParseParam("Editor" + Type.ToString()); if (UsesEditor) { Gauntlet.Log.Verbose("Will use Editor for role {0}", Type); } Role.Skip = Globals.Params.ParseParam("Skip" + Type.ToString()); if (Role.Skip) { Gauntlet.Log.Verbose("Will use NullPlatform to skip role {0}", Type); } // TODO - the below is a bit rigid, but maybe that's good enough since the "actually use the editor.." option // is specific to clients and servers // client can override platform and config if (Type.IsClient()) { Role.Platform = RequestedPlatform; Role.Configuration = RequestedConfiguration; if (UsesEditor) { Role.Type = UnrealTargetRole.EditorGame; Role.Platform = DefaultPlatform; Role.Configuration = UnrealTargetConfiguration.Development; } } else if (Type.IsServer()) { // server can only override config Role.Configuration = RequestedConfiguration; if (UsesEditor) { Role.Type = UnrealTargetRole.EditorServer; Role.Platform = DefaultPlatform; Role.Configuration = UnrealTargetConfiguration.Development; } } Gauntlet.Log.Verbose("Mapped Role {0} to RoleContext {1}", Type, Role); RoleContexts[Type] = Role; UsedPlatforms.Add(Role.Platform); } UnrealTestContext Context = new UnrealTestContext(BuildInfo, RoleContexts, ContextOptions); IEnumerable <ITestNode> TestNodes = CreateTestList(Context, CombinedParams, PlatformWithParams); AllTestNodes.AddRange(TestNodes); } bool AllTestsPassed = ExecuteTests(ContextOptions, AllTestNodes); // dispose now, not during shutdown gc, because this runs commands... DevicePool.Instance.Dispose(); DoCleanup(UsedPlatforms); return(AllTestsPassed ? ExitCode.Success : ExitCode.Error_TestFailure); }
/// <summary> /// Called after command line params are applied. Perform any checks / fixup /// </summary> /// <param name="InParams"></param> public virtual void ParametersWereApplied(string[] InParams) { // save params this.Params = new Params(InParams); if (string.IsNullOrEmpty(TempDir)) { TempDir = Globals.TempDir; } else { Globals.TempDir = TempDir; } if (string.IsNullOrEmpty(LogDir)) { LogDir = Globals.LogDir; } else { Globals.LogDir = LogDir; } if (string.IsNullOrEmpty(Sandbox)) { Sandbox = Project; } // parse platforms. These will be in the format of Win64(param1,param2='foo,bar') etc // check for old-style -platform=Win64(params) List <string> PlatformArgStrings = Params.ParseValues("Platform="); // check for convenience flags of -Win64(params) (TODO - need to think about this..) /*foreach (UnrealTargetPlatform Plat in Enum.GetValues(typeof(UnrealTargetPlatform))) * { * IEnumerable<string> RawPlatformArgs = InParams.Where(P => P.ToLower().StartsWith(Plat.ToString().ToLower())); * * PlatformArgStrings.AddRange(RawPlatformArgs); * }*/ // now turn the Plat(param1,param2) etc into an argument/parm pair PlatformList = PlatformArgStrings.SelectMany(P => ArgumentWithParams.CreateFromString(P)).ToList(); List <string> TestArgStrings = Params.ParseValues("test="); // clear all tests incase this is called multiple times TestList.Clear(); if (TestArgStrings.Count == 0) { TestArgStrings = Params.ParseValues("tests="); } foreach (string TestArg in TestArgStrings) { foreach (ArgumentWithParams TestWithParms in ArgumentWithParams.CreateFromString(TestArg)) { TestRequest Test = new TestRequest() { TestName = TestWithParms.Argument, TestParams = new Params(TestWithParms.AllArguments) }; // parse any specified platforms foreach (string PlatformArg in Test.TestParams.ParseValues("Platform=")) { List <ArgumentWithParams> PlatParams = ArgumentWithParams.CreateFromString(PlatformArg); Test.Platforms.AddRange(PlatParams); // register platform in test options PlatParams.ForEach(TestPlat => { if (PlatformList.Where(Plat => Plat.Argument == TestPlat.Argument).Count() == 0) { PlatformList.Add(TestPlat); } }); } TestList.Add(Test); } } if (PlatformList.Count == 0) { // Default to local platform PlatformList.Add(new ArgumentWithParams(BuildHostPlatform.Current.Platform.ToString(), new string[0])); } // do we have any tests? Need to check the global test list bool HaveTests = TestList.Count > 0 || PlatformList.Where(Plat => Plat.ParseValues("test").Count() > 0).Count() > 0; // turn -device=BobsKit,BobsKit(PS4) into a device list List <string> DeviceArgStrings = Params.ParseValues("device="); if (DeviceArgStrings.Count == 0) { DeviceArgStrings = Params.ParseValues("devices="); } DeviceList = DeviceArgStrings.SelectMany(D => ArgumentWithParams.CreateFromString(D)).ToList(); if (DeviceList.Count == 0) { // Add the default test DeviceList.Add(new ArgumentWithParams("default", new string[0])); } // remote all -test and -platform arguments from our params. Nothing else should be looking at these now... string[] CleanArgs = Params.AllArguments .Where(Arg => !Arg.StartsWith("test=", StringComparison.OrdinalIgnoreCase) && !Arg.StartsWith("platform=", StringComparison.OrdinalIgnoreCase) && !Arg.StartsWith("device=", StringComparison.OrdinalIgnoreCase)) .ToArray(); Params = new Params(CleanArgs); }