/// <summary>
/// Periodically called while the test is running. A chance for tests to examine their
/// health, log updates etc. Base classes must call this or take all responsibility for
/// setting Status as necessary
/// </summary>
/// <returns></returns>
public override void TickTest()
{
IAppInstance App = null;
if (TestInstance.ClientApps == null)
{
App = TestInstance.ServerApp;
}
else
{
if (TestInstance.ClientApps.Length > 0)
{
App = TestInstance.ClientApps.First();
}
}
if (App != null)
{
UnrealLogParser Parser = new UnrealLogParser(App.StdOut);
// TODO - hardcoded for Orion
List <string> TestLines = Parser.GetLogChannel("Gauntlet").ToList();
TestLines.AddRange(Parser.GetLogChannel("OrionTest"));
for (int i = LastLogCount; i < TestLines.Count; i++)
{
Log.Info(TestLines[i]);
if (Regex.IsMatch(TestLines[i], @".*GauntletHeartbeat\: Active.*"))
{
LastHeartbeatTime = DateTime.Now;
LastActiveHeartbeatTime = DateTime.Now;
}
else if (Regex.IsMatch(TestLines[i], @".*GauntletHeartbeat\: Idle.*"))
{
LastHeartbeatTime = DateTime.Now;
}
}
LastLogCount = TestLines.Count;
// Detect missed heartbeats and fail the test
CheckHeartbeat();
}
// Check status and health after updating logs
if (GetTestStatus() == TestStatus.InProgress && IsTestRunning() == false)
{
MarkTestComplete();
}
}
/// <summary>
/// Returns all results found in our construction content.
/// </summary>
/// <returns></returns>
public IEnumerable <AutomationTestResult> GetResults()
{
IEnumerable <Match> TestStarts = Parser.GetAllMatches(@"LogAutomationController.+Test Started. Name={(.+?)}");
// Find all automation results that succeeded/failed
// [00:10:54.148] LogAutomationController: Display: Test Started. Name={ST_PR04}
// [2019.04.30-18.49.51:329][244]LogAutomationController: Display: Test Completed With Success. Name={ST_PR04} Path={Project.Functional Tests./Game/Tests/Rendering/PlanarReflection.ST_PR04}
// [2019.04.30-18.49.51:330] [244] LogAutomationController: BeginEvents: Project.Functional Tests./Game/Tests/Rendering/PlanarReflection.ST_PR04
// [2019.04.30 - 18.49.51:331][244] LogAutomationController: Screenshot 'ST_PR04' was similar! Global Difference = 0.001377, Max Local Difference = 0.037953
// [2019.04.30 - 18.49.51:332][244]LogAutomationController: EndEvents: Project.Functional Tests./Game/Tests/Rendering/PlanarReflection.ST_PR04
IEnumerable <Match> TestResults = Parser.GetAllMatches(@"LogAutomationController.+Test Completed. Result={(.+?)}\s+Name={(.+?)}\s+Path={(.+?)}");
string[] AutomationChannel = Parser.GetLogChannel("AutomationController").ToArray();
// Convert these lines into results by parsing out the details and then adding the events
IEnumerable <AutomationTestResult> Results = TestStarts.Select(TestMatch =>
{
string DisplayName = TestMatch.Groups[1].ToString();
string LongName = "";
bool Completed = false;
bool Passed = false;
List <string> Events = new List <string>();
Match ResultMatch = TestResults.Where(M => M.Groups[2].ToString() == DisplayName).FirstOrDefault();
if (ResultMatch != null)
{
Completed = true;
Passed = ResultMatch.Groups[1].ToString().ToLower() == "passed" ? true : false;
LongName = ResultMatch.Groups[3].ToString();
string EventName = string.Format("BeginEvents: {0}", LongName);
int EventIndex = Array.FindIndex(AutomationChannel, S => S.Contains(EventName)) + 1;
if (EventIndex > 0)
{
while (EventIndex < AutomationChannel.Length)
{
string Event = AutomationChannel[EventIndex++];
if (Event.Contains("EndEvents"))
{
break;
}
Events.Add(Event);
}
}
}
AutomationTestResult Result = new AutomationTestResult(DisplayName, LongName, Completed, Passed);
Result.Events = Events;
return(Result);
});
return(Results);
}
/// <summary>
/// Periodically called while the test is running. A chance for tests to examine their
/// health, log updates etc. Base classes must call this or take all responsibility for
/// setting Status as necessary
/// </summary>
/// <returns></returns>
public override void TickTest()
{
IAppInstance App = null;
if (TestInstance.ClientApps == null)
{
App = TestInstance.ServerApp;
}
else
{
if (TestInstance.ClientApps.Length > 0)
{
App = TestInstance.ClientApps.First();
}
}
if (App != null)
{
UnrealLogParser Parser = new UnrealLogParser(App.StdOut);
// TODO - hardcoded for Orion
List <string> TestLines = Parser.GetLogChannel("Gauntlet").ToList();
TestLines.AddRange(Parser.GetLogChannel("OrionTest"));
for (int i = LastLogCount; i < TestLines.Count; i++)
{
Log.Info(TestLines[i]);
}
LastLogCount = TestLines.Count;
}
// Check status and health after updating logs
if (GetTestStatus() == TestStatus.InProgress && IsTestRunning() == false)
{
MarkTestComplete();
}
}
/// <summary>
/// Returns all results found in our construction content.
/// </summary>
/// <returns></returns>
public IEnumerable <AutomationTestResult> GetResults()
{
// Find all automation results that succeeded/failed
// [2019.04.30-18.49.51:329][244]LogAutomationController: Display: Automation Test Succeeded (ST_PR04 - Project.Functional Tests./Game/Tests/Rendering/PlanarReflection.ST_PR04)
// [2019.04.30-18.49.51:330] [244] LogAutomationController: BeginEvents: Project.Functional Tests./Game/Tests/Rendering/PlanarReflection.ST_PR04
// [2019.04.30 - 18.49.51:331][244] LogAutomationController: Screenshot 'ST_PR04' was similar! Global Difference = 0.001377, Max Local Difference = 0.037953
// [2019.04.30 - 18.49.51:332][244]LogAutomationController: EndEvents: Project.Functional Tests./Game/Tests/Rendering/PlanarReflection.ST_PR04
IEnumerable <Match> TestMatches = Parser.GetAllMatches(@"LogAutomationController.+Test\s+(Succeeded|Failed)\s+\((.+)\s+-\s+(.+)\)");
string[] AutomationChannel = Parser.GetLogChannel("AutomationController").ToArray();
// Convert these lines into results by parsing out the details and then adding the events
IEnumerable <AutomationTestResult> Results = TestMatches.Select(M =>
{
bool TestPassed = M.Groups[1].ToString().ToLower() == "succeeded" ? true : false;
string DisplayName = M.Groups[2].ToString();
string TestName = M.Groups[3].ToString();
AutomationTestResult Result = new AutomationTestResult(DisplayName, TestName, TestPassed);
string EventName = string.Format("BeginEvents: {0}", TestName);
int EventIndex = Array.FindIndex(AutomationChannel, S => S.Contains(EventName)) + 1;
if (EventIndex > 0)
{
while (EventIndex < AutomationChannel.Length)
{
string Event = AutomationChannel[EventIndex++];
if (Event.Contains("EndEvents"))
{
break;
}
Result.Events.Add(Event);
}
}
return(Result);
});
return(Results);
}