/// <summary>
/// Test entry point
/// </summary>
public override void TickTest()
{
// create the build source
UnrealBuildSource BuildSource = new UnrealBuildSource(this.ProjectName, ProjectFile, this.UnrealPath, UsesSharedBuildType, BuildPath, new string[] { "" });
// check editor and statged info is valid
CheckResult(BuildSource.EditorValid, "Editor build was invalid");
CheckResult(BuildSource.BuildCount > 0, "staged build was invalid");
// simple check with an editor role
UnrealSessionRole EditorRole = new UnrealSessionRole(UnrealTargetRole.Editor, BuildHostPlatform.Current.Platform, UnrealTargetConfiguration.Development);
List <string> Reasons = new List <string>();
// Check the build source can support this role
bool ContainsEditor = BuildSource.CanSupportRole(EditorRole, ref Reasons);
CheckResult(ContainsEditor, "{0", string.Join(", ", Reasons));
// now actually try to create it
UnrealAppConfig Config = BuildSource.CreateConfiguration(EditorRole, new UnrealSessionRole[] { });
CheckResult(Config != null, "Build source did not return a config for {0}", EditorRole.ToString());
ValidateEditorConfig(Config, BuildSource);
// Check all editor types (game, server, etc)
TestBuildSourceForEditorTypes(BuildSource);
// Test all monolithics that our base test says we support
TestBuildSourceForMonolithics(BuildSource);
MarkComplete();
}
public override void TickTest()
{
// create a new build
UnrealBuildSource Build = new UnrealBuildSource(this.GameName, this.UsesSharedBuildType, Environment.CurrentDirectory, this.BuildPath, new string[] { "" });
// check it's valid
if (!CheckResult(Build.BuildCount > 0, "staged build was invalid"))
{
MarkComplete();
return;
}
// Create devices to run the client and server
ITargetDevice PCDevice = new TargetDeviceWindows("Local PC", Globals.TempDir);
// Monolithic servers
TestInstallThenRun(Build, UnrealTargetRole.Server, PCDevice, UnrealTargetConfiguration.Development);
TestInstallThenRun(Build, UnrealTargetRole.Server, PCDevice, UnrealTargetConfiguration.Test);
// Monolithic clients
TestInstallThenRun(Build, UnrealTargetRole.Client, PCDevice, UnrealTargetConfiguration.Development);
TestInstallThenRun(Build, UnrealTargetRole.Client, PCDevice, UnrealTargetConfiguration.Test);
// editor configs
TestInstallThenRun(Build, UnrealTargetRole.EditorGame, PCDevice, UnrealTargetConfiguration.Development);
TestInstallThenRun(Build, UnrealTargetRole.EditorServer, PCDevice, UnrealTargetConfiguration.Development);
TestInstallThenRun(Build, UnrealTargetRole.Editor, PCDevice, UnrealTargetConfiguration.Development);
MarkComplete();
}
protected void TestInstallThenRun(UnrealBuildSource Build, UnrealTargetRole ProcessType, ITargetDevice Device, UnrealTargetConfiguration Config, UnrealOptions InOptions = null)
{
// create a config based on the passed in params
UnrealSessionRole Role = new UnrealSessionRole(ProcessType, Device.Platform, Config, InOptions);
Log.Info("Testing {0}", Role);
UnrealAppConfig AppConfig = Build.CreateConfiguration(Role);
if (!CheckResult(AppConfig != null, "Could not create config for {0} {1} with platform {2} from build.", Config, ProcessType, Device))
{
MarkComplete();
return;
}
// Install the app on this device
IAppInstall AppInstall = Device.InstallApplication(AppConfig);
CheckResult(AppConfig != null, "Could not create AppInstall for {0} {1} with platform {2} from build.", Config, ProcessType, Device);
DateTime StartTime = DateTime.Now;
// Run the app and wait for either a timeout or it to exit
IAppInstance AppProcess = AppInstall.Run();
while (AppProcess.HasExited == false)
{
if ((DateTime.Now - StartTime).TotalSeconds > 60)
{
break;
}
Thread.Sleep(1000);
}
// Check it didn't exit unexpectedly
CheckResult(AppProcess.HasExited == false, "Failed to run {0} {1} with platform {2}", Config, ProcessType, Device);
// but kill it
AppProcess.Kill();
// Check that it left behind some artifacts (minimum should be a log)
int ArtifactCount = new DirectoryInfo(AppProcess.ArtifactPath).GetFiles("*", SearchOption.AllDirectories).Length;
CheckResult(ArtifactCount > 0, "No artifacts on device!");
}
/// <summary>
/// Tests that this BuildSource is capable of providing a config for all supported editor-based roles
/// </summary>
/// <param name="BuildSource"></param>
void TestBuildSourceForEditorTypes(UnrealBuildSource BuildSource)
{
// create a config for all editor based types
foreach (var E in Enum.GetValues(typeof(UnrealTargetRole)).Cast <UnrealTargetRole>())
{
if (E.UsesEditor())
{
UnrealAppConfig Config = BuildSource.CreateConfiguration(new UnrealSessionRole(E, UnrealTargetPlatform.Win64, UnrealTargetConfiguration.Development));
CheckResult(Config != null, "Editor config for {0} returned null!", E);
CheckResult(string.IsNullOrEmpty(Config.Name) == false, "No config name!");
ValidateEditorConfig(Config, BuildSource);
}
}
}
public override void TickTest()
{
// create a new build
UnrealBuildSource Build = new UnrealBuildSource(this.ProjectName, this.ProjectFile, this.UnrealPath, this.UsesSharedBuildType, this.BuildPath, new string[] { "" });
// check it's valid
if (!CheckResult(Build.BuildCount > 0, "staged build was invalid"))
{
MarkComplete();
return;
}
// Create devices to run the client and server
ITargetDevice PCDevice = new TargetDeviceMac("LocalMac", Globals.TempDir);
UnrealOptions GameOptions = new UnrealOptions();
GameOptions.Windowed = true;
GameOptions.ResX = 1280;
GameOptions.ResY = 720;
GameOptions.CommonArgs = "-log";
UnrealOptions OtherOptions = new UnrealOptions();
OtherOptions.CommonArgs = "-log";
// editor configs
TestInstallThenRun(Build, UnrealTargetRole.EditorGame, PCDevice, UnrealTargetConfiguration.Development, GameOptions);
TestInstallThenRun(Build, UnrealTargetRole.EditorServer, PCDevice, UnrealTargetConfiguration.Development, OtherOptions);
TestInstallThenRun(Build, UnrealTargetRole.Editor, PCDevice, UnrealTargetConfiguration.Development, OtherOptions);
// Monolithic clients
//TestInstallThenRun(Build, UnrealRoleType.Client, PCDevice, UnrealTargetConfiguration.Development, GameOptions);
TestInstallThenRun(Build, UnrealTargetRole.Client, PCDevice, UnrealTargetConfiguration.Test, GameOptions);
// Monolithic servers
//TestInstallThenRun(Build, UnrealRoleType.Server, PCDevice, UnrealTargetConfiguration.Development, OtherOptions);
//TestInstallThenRun(Build, UnrealRoleType.Server, PCDevice, UnrealTargetConfiguration.Test, OtherOptions);
MarkComplete();
}
/// <summary>
/// Simple validation of a config that uses
/// </summary>
/// <param name="Config"></param>
/// <param name="BuildSource"></param>
void ValidateEditorConfig(UnrealAppConfig Config, UnrealBuildSource BuildSource)
{
string Args = Config.CommandLine.Trim().ToLower();
CheckResult(Config.ProcessType.UsesEditor(), "Config does not use editor!");
// Check the project name was the first arg
CheckResult(Args.IndexOf(BuildSource.ProjectName.ToLower()) == 0, "Editor-based config for {0} needs to include project name as first argument", Config.ProcessType);
// for clients, check for -game
if (Config.ProcessType.IsClient())
{
CheckResult(Args.Contains("-game"), "Editor-based game needs to include -game");
}
// for servers, check for -server
if (Config.ProcessType.IsServer())
{
CheckResult(Args.Contains("-server"), "Editor-based game needs to include -server");
}
}
public override void TickTest()
{
// Grab the most recent Release build of Orion
UnrealBuildSource Build = new UnrealBuildSource(this.ProjectName, this.ProjectFile, this.UnrealPath, this.UsesSharedBuildType, this.BuildPath);
// create client and server riles
UnrealSessionRole ClientRole = new UnrealSessionRole(UnrealTargetRole.Client, UnrealTargetPlatform.Win64, UnrealTargetConfiguration.Development);
UnrealSessionRole ServerRole = new UnrealSessionRole(UnrealTargetRole.Server, UnrealTargetPlatform.Win64, UnrealTargetConfiguration.Development);
// create configurations from the build
UnrealAppConfig ClientConfig = Build.CreateConfiguration(ClientRole);
UnrealAppConfig ServerConfig = Build.CreateConfiguration(ServerRole);
UnrealOptions Options = new UnrealOptions();
// create some params
string[] Params = new string[] { "nullrhi", "ResX=800", "ResY=600", "Map=FooMap", "epicapp=Prod", "buildidoverride=1111", "commonargs=-somethingcommon", "-clientargs=-somethingclient", "-serverargs=-somethingserver" };
// apply them to options
AutoParam.ApplyParams(Options, Params);
Options.ApplyToConfig(ClientConfig);
Options.ApplyToConfig(ServerConfig);
CheckResult(ClientConfig.CommandLine.Contains("-nullrhi"), "Client Arg not applied!");
CheckResult(ClientConfig.CommandLine.Contains("-ResX=800"), "Client Arg not applied!");
CheckResult(ClientConfig.CommandLine.Contains("-ResY=600"), "Client Arg not applied!");
CheckResult(ClientConfig.CommandLine.Contains("-somethingclient"), "Client Arg not applied!");
CheckResult(ServerConfig.CommandLine.Contains("-Map=FooMap") == false, "Server Arg incorrectly applied!");
CheckResult(ServerConfig.CommandLine.StartsWith("FooMap"), "Server Args not start with map!");
CheckResult(ServerConfig.CommandLine.Contains("-somethingserver"), "Server Arg not applied!");
CheckResult(ClientConfig.CommandLine.Contains("-buildidoverride=1111"), "Common Arg not applied!");
CheckResult(ClientConfig.CommandLine.Contains("-somethingcommon"), "Common Arg not applied!");
CheckResult(ServerConfig.CommandLine.Contains("-buildidoverride=1111"), "Common Arg not applied!");
CheckResult(ServerConfig.CommandLine.Contains("-somethingcommon"), "Common Arg not applied!");
MarkComplete();
}
/// <summary>
/// Tests that this BuildSource is capable of returning configs for all the roles,
/// platforms, and configurations that our base class says it should support
/// </summary>
/// <param name="BuildSource"></param>
/// <returns></returns>
void TestBuildSourceForMonolithics(UnrealBuildSource BuildSource)
{
List <UnrealSessionRole> AllRoles = new List <UnrealSessionRole>();
// Add a role for all supported clients on all supported configurations
foreach (var Platform in SupportedClientPlatforms)
{
foreach (var Config in SupportedConfigurations)
{
AllRoles.Add(new UnrealSessionRole(UnrealTargetRole.Client, Platform, Config));
}
}
// Add a role for all supported servers on all supported configurations
foreach (var Platform in SupportedServerPlatforms)
{
foreach (var Config in SupportedConfigurations)
{
AllRoles.Add(new UnrealSessionRole(UnrealTargetRole.Server, Platform, Config));
}
}
// Now check the build source can create all of these
foreach (var Role in AllRoles)
{
List <string> Issues = new List <string>();
bool Result = BuildSource.CanSupportRole(Role, ref Issues);
Issues.ForEach(S => Log.Error(S));
CheckResult(Result, "Failed to get artifacts for {0}", Role);
// now actually try to create it
UnrealAppConfig Config = BuildSource.CreateConfiguration(Role);
CheckResult(Config != null, "Build source did not return a config for {0}", Role.ToString());
}
}
void TestClientPlatform(UnrealTargetPlatform Platform)
{
string GameName = this.ProjectFile.FullName;
string BuildPath = this.BuildPath;
string DevKit = this.DevkitName;
if (GameName.Equals("OrionGame", StringComparison.OrdinalIgnoreCase) == false)
{
Log.Info("Skipping test {0} due to non-Orion project!", this);
MarkComplete();
return;
}
// create a new build
UnrealBuildSource Build = new UnrealBuildSource(ProjectFile, this.UnrealPath, UsesSharedBuildType, BuildPath);
// check it's valid
if (!CheckResult(Build.BuildCount > 0, "staged build was invalid"))
{
MarkComplete();
return;
}
// Create devices to run the client and server
ITargetDevice ServerDevice = new TargetDeviceWindows("PC Server", Gauntlet.Globals.TempDir);
ITargetDevice ClientDevice = null;
if (Platform == UnrealTargetPlatform.PS4)
{
//ClientDevice = new TargetDevicePS4(this.PS4Name);
}
else
{
ClientDevice = new TargetDeviceWindows("PC Client", Gauntlet.Globals.TempDir);
}
UnrealAppConfig ServerConfig = Build.CreateConfiguration(new UnrealSessionRole(UnrealTargetRole.Server, ServerDevice.Platform, UnrealTargetConfiguration.Development));
UnrealAppConfig ClientConfig = Build.CreateConfiguration(new UnrealSessionRole(UnrealTargetRole.Client, ClientDevice.Platform, UnrealTargetConfiguration.Development));
if (!CheckResult(ServerConfig != null && ServerConfig != null, "Could not create configs!"))
{
MarkComplete();
return;
}
ShortSoloOptions Options = new ShortSoloOptions();
Options.ApplyToConfig(ClientConfig);
Options.ApplyToConfig(ServerConfig);
IAppInstall ClientInstall = ClientDevice.InstallApplication(ClientConfig);
IAppInstall ServerInstall = ServerDevice.InstallApplication(ServerConfig);
if (!CheckResult(ServerConfig != null && ServerConfig != null, "Could not create configs!"))
{
MarkComplete();
return;
}
IAppInstance ClientInstance = ClientInstall.Run();
IAppInstance ServerInstance = ServerInstall.Run();
DateTime StartTime = DateTime.Now;
bool RunWasSuccessful = true;
while (ClientInstance.HasExited == false)
{
if ((DateTime.Now - StartTime).TotalSeconds > 800)
{
RunWasSuccessful = false;
break;
}
}
ClientInstance.Kill();
ServerInstance.Kill();
UnrealLogParser LogParser = new UnrealLogParser(ClientInstance.StdOut);
UnrealLogParser.CallstackMessage ErrorInfo = LogParser.GetFatalError();
if (ErrorInfo != null)
{
CheckResult(false, "FatalError - {0}", ErrorInfo.Message);
}
RunWasSuccessful = LogParser.HasRequestExit();
CheckResult(RunWasSuccessful, "Failed to run for platform {0}", Platform);
}
public override void CreateReport(TestResult Result, UnrealTestContext Contex, UnrealBuildSource Build, IEnumerable <UnrealRoleArtifacts> Artifacts, string ArtifactPath)
{
UnrealRoleArtifacts ClientArtifacts = Artifacts.Where(A => A.SessionRole.RoleType == UnrealTargetRole.Client).FirstOrDefault();
var SnapshotSummary = new UnrealSnapshotSummary <UnrealHealthSnapshot>(ClientArtifacts.AppInstance.StdOut);
Log.Info("Elemental Performance Report");
Log.Info(SnapshotSummary.ToString());
base.CreateReport(Result, Contex, Build, Artifacts, ArtifactPath);
}
/// <summary>
/// Called after a test finishes to create an overall summary based on looking at the artifacts
/// </summary>
/// <param name="Result"></param>
/// <param name="Context"></param>
/// <param name="Build"></param>
/// <param name="Artifacts"></param>
/// <param name="InArtifactPath"></param>
public override void CreateReport(TestResult Result, UnrealTestContext Context, UnrealBuildSource Build, IEnumerable <UnrealRoleArtifacts> Artifacts, string InArtifactPath)
{
// only check for artifacts if the test passed
if (Result != TestResult.Passed)
{
return;
}
if (!DidDetectLaunch)
{
SetUnrealTestResult(TestResult.Failed);
Log.Error("Failed to detect completion of launch");
return;
}
bool MissingFiles = false;
foreach (var RoleArtifact in Artifacts)
{
DirectoryInfo RoleDir = new DirectoryInfo(RoleArtifact.ArtifactPath);
IEnumerable <FileInfo> ArtifactFiles = RoleDir.EnumerateFiles("*.*", SearchOption.AllDirectories);
// user may not have cleared paths between runs, so throw away anything that's older than 2m
ArtifactFiles = ArtifactFiles.Where(F => (DateTime.Now - F.LastWriteTime).TotalMinutes < 2);
if (ArtifactFiles.Any() == false)
{
MissingFiles = true;
Log.Error("No artifact files found for {0}. Were they not retrieved from the device?", RoleArtifact.SessionRole);
}
IEnumerable <FileInfo> LogFiles = ArtifactFiles.Where(F => F.Extension.Equals(".log", StringComparison.OrdinalIgnoreCase));
if (LogFiles.Any() == false)
{
MissingFiles = true;
Log.Error("No log files found for {0}. Were they not retrieved from the device?", RoleArtifact.SessionRole);
}
}
if (MissingFiles)
{
SetUnrealTestResult(TestResult.Failed);
Log.Error("One or more roles did not generated any artifacts");
}
Log.Info("Found valid artifacts for test");
}
public override void TickTest()
{
AccountPool.Instance.RegisterAccount(new EpicAccount("Foo", "Bar"));
// Add three devices to the pool
DevicePool.Instance.RegisterDevices(new ITargetDevice[] {
new TargetDeviceWindows("Local PC1", Globals.TempDir)
, new TargetDeviceWindows("Local PC2", Globals.TempDir)
});
// Create a new build (params come from our base class will be similar to "OrionGame" and "p:\builds\orion\branch-cl")
UnrealBuildSource Build = new UnrealBuildSource(this.GameName, this.UsesSharedBuildType, Environment.CurrentDirectory, this.BuildPath, new string[] { "" });
// create a new options structure
UnrealOptions Options = new UnrealOptions();
// set the mapname, this will be applied automatically to the server
Options.Map = "OrionEntry";
Options.Log = true;
// add some common options.
string ServerArgs = " -nomcp -log";
// We want the client to connect to the server, so get the IP address of this PC and add it to the client args as an ExecCmd
string LocalIP = Dns.GetHostEntry(Dns.GetHostName()).AddressList.Where(o => o.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork).First().ToString();
string ClientArgs = string.Format(" -ExecCmds=\"open {0}\"", LocalIP);
// create a new session with client & server roles
UnrealSession Session = new UnrealSession(Build, new[] {
new UnrealSessionRole(UnrealTargetRole.Client, UnrealTargetPlatform.PS4, UnrealTargetConfiguration.Development, ClientArgs, Options)
, new UnrealSessionRole(UnrealTargetRole.Server, UnrealTargetPlatform.Win64, UnrealTargetConfiguration.Development, ServerArgs, Options)
});
// launch an instance of this session
UnrealSessionInstance SessionInstance = Session.LaunchSession();
// wait for two minutes - long enough for anything to go wrong :)
DateTime StartTime = DateTime.Now;
while (SessionInstance.ClientsRunning && SessionInstance.ServerRunning)
{
if ((DateTime.Now - StartTime).TotalSeconds > 120)
{
break;
}
Thread.Sleep(1000);
}
// check these are both still running
CheckResult(SessionInstance.ClientsRunning, "Clients exited during test");
CheckResult(SessionInstance.ServerRunning, "Server exited during test");
// shutdown the session
SessionInstance.Shutdown();
// shutdown the pools
AccountPool.Shutdown();
DevicePool.Shutdown();
MarkComplete(TestResult.Passed);
}
