public override void ExecuteBuild()
{
Log("Starting 1st timer (1s). This should not crash.");
using (var SafeTimer = new WatchdogTimer(1))
{
// Wait 500ms
Log("Started {0}", SafeTimer.GetProcessName());
Thread.Sleep(500);
}
Log("First timer disposed successfully.");
Log("Starting 2nd timer (2s). This should throw an exception after 1 second.");
try
{
using (var CrashTimer = new WatchdogTimer(2))
{
// Wait 5s (this will trigger the watchdog timer)
Log("Started {0}", CrashTimer.GetProcessName());
Thread.Sleep(1000);
throw new Exception("Test exceptions under WatchdogTimer");
}
}
catch (Exception Ex)
{
Log("Triggered exception guarded by WatchdogTimer:");
Log(System.Diagnostics.TraceEventType.Information, Ex);
}
Log("Starting 3rd timer (2s). This should crash after 2 seconds.");
using (var CrashTimer = new WatchdogTimer(2))
{
// Wait 5s (this will trigger the watchdog timer)
Log("Started {0}", CrashTimer.GetProcessName());
Thread.Sleep(5000);
}
}
private static void RunStandaloneClient(List<DeploymentContext> DeployContextList, string ClientLogFile, ERunOptions ClientRunFlags, string ClientApp, string ClientCmdLine, ProjectParams Params)
{
if (Params.Unattended)
{
int Timeout = 60 * 30;
if (Params.RunAutomationTests)
{
Timeout = 6 * 60 * 60;
}
using (var Watchdog = new WatchdogTimer(Timeout))
{
string AllClientOutput = "";
ProcessResult ClientProcess = null;
FileStream ClientProcessLog = null;
StreamReader ClientLogReader = null;
Log("Starting Client for unattended test....");
ClientProcess = Run(ClientApp, ClientCmdLine + " -FORCELOGFLUSH", null, ClientRunFlags | ERunOptions.NoWaitForExit);
while (!FileExists(ClientLogFile) && !ClientProcess.HasExited)
{
Log("Waiting for client logging process to start...{0}", ClientLogFile);
Thread.Sleep(2000);
}
if (!ClientProcess.HasExited)
{
Thread.Sleep(2000);
Log("Client logging process started...{0}", ClientLogFile);
ClientProcessLog = File.Open(ClientLogFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
ClientLogReader = new StreamReader(ClientProcessLog);
}
if (ClientProcess.HasExited || ClientLogReader == null)
{
throw new AutomationException("Client exited before we asked it to.");
}
bool bKeepReading = true;
bool WelcomedCorrectly = false;
while (!ClientProcess.HasExited && bKeepReading)
{
while (!ClientLogReader.EndOfStream && bKeepReading && !ClientProcess.HasExited)
{
string ClientOutput = ClientLogReader.ReadToEnd();
if (!String.IsNullOrEmpty(ClientOutput))
{
AllClientOutput += ClientOutput;
Console.Write(ClientOutput);
string LookFor = "Bringing up level for play took";
if (Params.RunAutomationTest != "")
{
LookFor = "Automation Test Succeeded";
}
else if (Params.RunAutomationTests)
{
LookFor = "Automation Test Queue Empty";
}
else if (Params.EditorTest)
{
LookFor = "Asset discovery search completed in";
}
if (AllClientOutput.Contains(LookFor))
{
Log("Client loaded, lets wait 30 seconds...");
Thread.Sleep(30000);
if (!ClientProcess.HasExited)
{
WelcomedCorrectly = true;
}
Log("Test complete...");
bKeepReading = false;
}
else if (Params.RunAutomationTests)
{
int FailIndex = AllClientOutput.IndexOf("Automation Test Failed");
int ParenIndex = AllClientOutput.LastIndexOf(")");
if (FailIndex >= 0 && ParenIndex > FailIndex)
{
string Tail = AllClientOutput.Substring(FailIndex);
int CloseParenIndex = Tail.IndexOf(")");
int OpenParenIndex = Tail.IndexOf("(");
string Test = "";
if (OpenParenIndex >= 0 && CloseParenIndex > OpenParenIndex)
{
Test = Tail.Substring(OpenParenIndex + 1, CloseParenIndex - OpenParenIndex - 1);
}
#if true
Log("Stopping client...");
ClientProcess.StopProcess();
if (IsBuildMachine)
{
Thread.Sleep(70000);
}
#endif
throw new AutomationException("Automated test failed ({0}).", Test);
}
}
}
}
}
if (ClientProcess != null && !ClientProcess.HasExited)
{
Log("Stopping client...");
ClientProcess.StopProcess();
}
// this is hack that prevents a hang
if (IsBuildMachine)
{
Thread.Sleep(70000);
}
if (!WelcomedCorrectly)
{
throw new AutomationException("Client exited before we asked it to.");
}
}
}
else
{
var SC = DeployContextList[0];
ProcessResult ClientProcess = SC.StageTargetPlatform.RunClient(ClientRunFlags, ClientApp, ClientCmdLine, Params);
if (ClientProcess != null)
{
// If the client runs without StdOut redirect we're going to read the log output directly from log file on
// a separate thread.
if ((ClientRunFlags & ERunOptions.NoStdOutRedirect) == ERunOptions.NoStdOutRedirect)
{
ClientLogReaderThread = new System.Threading.Thread(ClientLogReaderProc);
ClientLogReaderThread.Start(new object[] { ClientLogFile, ClientProcess });
}
do
{
Thread.Sleep(100);
}
while (ClientProcess.HasExited == false);
}
}
}
public override void ExecuteBuild()
{
Log("Starting the first timer (1s). This should not crash.");
using (var SafeTimer = new WatchdogTimer(1))
{
// Wait 500ms
Thread.Sleep(500);
}
Log("First timer disposed successfully.");
Log("Starting the second timer (2s). This should crash after 2 seconds.");
using (var CrashTimer = new WatchdogTimer(2))
{
// Wait 5s (this will trigger the watchdog timer)
Thread.Sleep(5000);
}
}
private static void RunClientWithServer(string ServerLogFile, ProcessResult ServerProcess, string ClientApp, string ClientCmdLine, ERunOptions ClientRunFlags, string ClientLogFile, ProjectParams Params)
{
ProcessResult ClientProcess = null;
var OtherClients = new List<ProcessResult>();
bool WelcomedCorrectly = false;
int NumClients = Params.NumClients;
string AllClientOutput = "";
if (Params.Unattended)
{
int Timeout = 60 * 5;
if (Params.FakeClient)
{
Timeout = 60 * 15;
}
if (Params.RunAutomationTests)
{
Timeout = 6 * 60 * 60;
}
using (var Watchdog = new WatchdogTimer(Timeout))
{
while (!FileExists(ServerLogFile) && !ServerProcess.HasExited)
{
Log("Waiting for logging process to start...");
Thread.Sleep(2000);
}
Thread.Sleep(1000);
string AllServerOutput = "";
using (FileStream ProcessLog = File.Open(ServerLogFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
StreamReader LogReader = new StreamReader(ProcessLog);
bool bKeepReading = true;
FileStream ClientProcessLog = null;
StreamReader ClientLogReader = null;
// Read until the process has exited.
while (!ServerProcess.HasExited && bKeepReading)
{
while (!LogReader.EndOfStream && bKeepReading && ClientProcess == null)
{
string Output = LogReader.ReadToEnd();
if (!String.IsNullOrEmpty(Output))
{
AllServerOutput += Output;
if (ClientProcess == null &&
(AllServerOutput.Contains("Game Engine Initialized") || AllServerOutput.Contains("Unreal Network File Server is ready")))
{
Log("Starting Client for unattended test....");
ClientProcess = Run(ClientApp, ClientCmdLine + " -FORCELOGFLUSH", null, ClientRunFlags | ERunOptions.NoWaitForExit);
//@todo no testing is done on these
if (NumClients > 1 && NumClients < 9)
{
for (int i = 1; i < NumClients; i++)
{
Log("Starting Extra Client....");
OtherClients.Add(Run(ClientApp, ClientCmdLine, null, ClientRunFlags | ERunOptions.NoWaitForExit));
}
}
while (!FileExists(ClientLogFile) && !ClientProcess.HasExited)
{
Log("Waiting for client logging process to start...{0}", ClientLogFile);
Thread.Sleep(2000);
}
if (!ClientProcess.HasExited)
{
Thread.Sleep(2000);
Log("Client logging process started...{0}", ClientLogFile);
ClientProcessLog = File.Open(ClientLogFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
ClientLogReader = new StreamReader(ClientProcessLog);
}
}
else if (ClientProcess == null && !ServerProcess.HasExited)
{
Log("Waiting for server to start....");
Thread.Sleep(2000);
}
if (ClientProcess != null && ClientProcess.HasExited)
{
ServerProcess.StopProcess();
if (IsBuildMachine)
{
Thread.Sleep(70000);
}
throw new AutomationException("Client exited before we asked it to.");
}
}
}
if (ClientLogReader != null)
{
if (ClientProcess.HasExited)
{
ServerProcess.StopProcess();
if (IsBuildMachine)
{
Thread.Sleep(70000);
}
throw new AutomationException("Client exited or closed the log before we asked it to.");
}
while (!ClientProcess.HasExited && !ServerProcess.HasExited && bKeepReading)
{
while (!ClientLogReader.EndOfStream && bKeepReading && !ServerProcess.HasExited && !ClientProcess.HasExited)
{
string ClientOutput = ClientLogReader.ReadToEnd();
if (!String.IsNullOrEmpty(ClientOutput))
{
AllClientOutput += ClientOutput;
Console.Write(ClientOutput);
string LookFor = "Bringing up level for play took";
if (Params.DedicatedServer)
{
LookFor = "Welcomed by server";
}
else if (Params.RunAutomationTest != "")
{
LookFor = "Automation Test Succeeded";
}
else if (Params.RunAutomationTests)
{
LookFor = "Automation Test Queue Empty";
}
if (AllClientOutput.Contains(LookFor))
{
if (Params.FakeClient)
{
Log("Welcomed by server or client loaded, lets wait ten minutes...");
Thread.Sleep(60000 * 10);
}
else
{
Log("Welcomed by server or client loaded, lets wait 30 seconds...");
Thread.Sleep(30000);
}
if (!ServerProcess.HasExited && !ClientProcess.HasExited)
{
WelcomedCorrectly = true;
}
if (!ServerProcess.HasExited)
{
Log("Test complete");
if (ClientProcess != null && !ClientProcess.HasExited)
{
Log("Stopping client....");
ClientProcess.StopProcess();
}
Log("Stopping server....");
ServerProcess.StopProcess();
if (IsBuildMachine)
{
Thread.Sleep(70000);
}
}
bKeepReading = false;
}
else if (Params.RunAutomationTests)
{
int FailIndex = AllClientOutput.IndexOf("Automation Test Failed");
int ParenIndex = AllClientOutput.LastIndexOf(")");
if (FailIndex >= 0 && ParenIndex > FailIndex)
{
string Tail = AllClientOutput.Substring(FailIndex);
int CloseParenIndex = Tail.IndexOf(")");
int OpenParenIndex = Tail.IndexOf("(");
string Test = "";
if (OpenParenIndex >= 0 && CloseParenIndex > OpenParenIndex)
{
Test = Tail.Substring(OpenParenIndex + 1, CloseParenIndex - OpenParenIndex - 1);
}
#if true
if (!ServerProcess.HasExited)
{
Log("Stopping server....");
ServerProcess.StopProcess();
}
if (ClientProcess != null && !ClientProcess.HasExited)
{
Log("Stopping client....");
ClientProcess.StopProcess();
}
foreach (var OtherClient in OtherClients)
{
if (OtherClient != null && !OtherClient.HasExited)
{
Log("Stopping client....");
OtherClient.StopProcess();
}
}
if (IsBuildMachine)
{
Thread.Sleep(70000);
}
#endif
throw new AutomationException("Automated test failed ({0}).", Test);
}
}
}
}
}
}
}
}
}
}
else
{
LogFileReaderProcess(ServerLogFile, ServerProcess, (string Output) =>
{
bool bKeepReading = true;
if (ClientProcess == null && !String.IsNullOrEmpty(Output))
{
AllClientOutput += Output;
if (ClientProcess == null && (AllClientOutput.Contains("Game Engine Initialized") || AllClientOutput.Contains("Unreal Network File Server is ready")))
{
Log("Starting Client....");
ClientProcess = Run(ClientApp, ClientCmdLine, null, ClientRunFlags | ERunOptions.NoWaitForExit);
if (NumClients > 1 && NumClients < 9)
{
for (int i = 1; i < NumClients; i++)
{
Log("Starting Extra Client....");
OtherClients.Add(Run(ClientApp, ClientCmdLine, null, ClientRunFlags | ERunOptions.NoWaitForExit));
}
}
}
}
else if (ClientProcess == null && !ServerProcess.HasExited)
{
Log("Waiting for server to start....");
Thread.Sleep(2000);
}
if (String.IsNullOrEmpty(Output) == false)
{
Console.Write(Output);
}
if (ClientProcess != null && ClientProcess.HasExited)
{
Log("Client exited, stopping server....");
ServerProcess.StopProcess();
bKeepReading = false;
}
return bKeepReading; // Keep reading
});
}
Log("Server exited....");
if (ClientProcess != null && !ClientProcess.HasExited)
{
ClientProcess.StopProcess();
}
foreach (var OtherClient in OtherClients)
{
if (OtherClient != null && !OtherClient.HasExited)
{
OtherClient.StopProcess();
}
}
if (Params.Unattended)
{
// this is a hack to prevent hangs on exit
if (IsBuildMachine)
{
Thread.Sleep(70000);
}
if (!WelcomedCorrectly)
{
throw new AutomationException("Server or client exited before we asked it to.");
}
}
}