public static async Task Runner(TestInfo testInfo)
{
// Test log
LogHelper.SetLogger(testInfo);
ApplyTestContextLogging(testInfo.Context);
var rerunCount = 1;
var log = LogHelper.GetDefaultLogger();
var execContext = new TestExecutionContext(new TestStorage(), new WrapperHelper(), _testContext);
lock (_wrapperHelpers)
{
_wrapperHelpers.Add(execContext.Wrapper);
}
var testResult = new TestResult();
try
{
log.Info($"***** Running {++_testsRunCounter} of {TotalTests} *****");
testResult = await new TestHelper(execContext).ExecuteTest(testInfo);
if (!testResult.Success)
{
// Rerun test when failed
var retryCount = Constants.AppConfig.RetryCount;
if (retryCount == 0 && testResult.Retry)
{
retryCount++;
}
while (rerunCount <= retryCount)
{
var testLog = LogHelper.GetLogger();
testLog.Info($"\n##### Retrying {testInfo.Name} for {++rerunCount} time #####");
// Resetting cache & summary
execContext.Storage = new TestStorage();
testInfo.Browser = null;
testResult = await new TestHelper(execContext).ExecuteTest(testInfo);
if (testResult.Success)
{
break;
}
}
}
Assert.IsTrue(testResult.Success, testInfo.Name);
}
catch (AssertFailedException)
{
throw;
}
catch (Exception ex)
{
log.Error($"Fatal Error: {ex.Message}\nStackTrace: {ex.StackTrace}");
Assert.Fail($"{testInfo.Name} failed!");
}
finally
{
ReportStatus(execContext, testResult);
log.Info($"***** {testResult.Status} in {rerunCount}/{Constants.AppConfig.RetryCount + 1} attempt(s) " +
$"Status=P:{_successCount},F:{_failureCount} *****");
}
}
private static void ReportStatus(TestExecutionContext context, TestResult result)
{
if (!result.Success)
{
context.Storage.Summary.LogSummary();
_failureCount += 1;
return;
}
_successCount += 1;
}
public async Task <TestResult> ExecuteTest(TestInfo testInfo)
{
var stopWatch = new Stopwatch();
stopWatch.Start();
var testResult = new TestResult {
Success = true
};
var summary = _execContext.Storage.Summary;
// don't log sub flows as the TestName changes
_log.Info($"------- {testInfo.Name} started... -------");
summary.LogData[SummaryFields.TestName] = testInfo.Name;
LoadTasks(testInfo);
summary.LogData[SummaryFields.TaskCount] = testInfo.Tasks.Count.ToString();
try
{
foreach (var taskInfo in testInfo.Tasks)
{
var taskResult = await new TaskHelper(_execContext).ExecuteTask(testInfo, taskInfo);
if (!taskResult.Success)
{
_log.Info($"{taskInfo.TaskName} ({taskInfo.Action}) failed!");
summary.LogData[SummaryFields.FailedTask] = $"{taskInfo.TaskName} ({taskInfo.Action})";
// Saving failed test log for debugging purpose
PersistLog(testInfo);
testResult.Success = false;
// Retry when there is a recoverable error
testResult.Retry = taskResult.IsError;
break;
}
_log.Info($"{taskInfo.TaskName} ({taskInfo.Action}) succeeded.");
}
}
finally
{
testResult.Status = $"{testInfo.Name} {(testResult.Success ? "passed" : "failed")}!";
summary.LogData[SummaryFields.FullName] = testInfo.FullName;
summary.LogData[SummaryFields.Environment] = testInfo.Environment.ToString();
summary.LogData[SummaryFields.Plugin] = testInfo.GetPlugin();
summary.LogData[SummaryFields.TestResult] = testResult.Success ? "Pass" : "Fail";
summary.LogData[SummaryFields.ExecutionTime] = stopWatch.ElapsedMilliseconds.ToString();
SeleniumHelper.Cleanup(testInfo.Browser, testInfo.Language);
stopWatch.Stop();
_log.Info($"------- {testResult.Status} -------");
}
return(testResult);
}