public List <SystemTestResult> Execute(SystemTestCategories systemTestCategory)
{
var results = new List <SystemTestResult>();
int hoursSinceRunStarted = 24;
int scenarioCompleteTimeoutMins = 120;
try
{
using (var scope = _repositoryFactory.BeginRepositoryScope())
{
var runRepository = scope.CreateRepository <IRunRepository>();
// Get any runs started in last N hours
int countFailedRuns = 0;
var runs = runRepository.GetAll().Where(r => r.FirstScenarioStartedDateTime != null && r.FirstScenarioStartedDateTime >= DateTime.UtcNow.AddHours(-hoursSinceRunStarted)).OrderBy(r => r.CreatedDateTime);
// Check runs
foreach (var run in runs)
{
// Count up failures
countFailedRuns += run.Scenarios.Where(s => s.Status == ScenarioStatuses.CompletedError).ToList().Count;
try
{
ISystemTest runOutputTest = new RunOutputTest(run, _category, _repositoryFactory, scenarioCompleteTimeoutMins);
results.AddRange(runOutputTest.Execute(systemTestCategory));
}
catch { }; // Ignore
}
if (!runs.Any())
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Warning, _category, string.Format("No runs were started in the last {0} hours", hoursSinceRunStarted), ""));
}
else if (countFailedRuns == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Information, _category, string.Format("No failed runs in the last {0} hours", hoursSinceRunStarted), ""));
}
}
}
catch (System.Exception exception)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Error checking run history: {0}", exception.Message), ""));
}
return(results);
}
public List <SystemTestResult> Execute(SystemTestCategories systemTestCategory)
{
var results = new List <SystemTestResult>();
bool runCompleted = false;
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, "Executing test run"));
try
{
using (var scope = _repositoryFactory.BeginRepositoryScope())
{
var runRepository = scope.CreateRepository <IRunRepository>();
// Load template run
Run templateRun = runRepository.Find(_templateRunId);
if (templateRun == null) // Template run does not exist, create it
{
CreateRunForTemplate(_templateRunId);
templateRun = runRepository.Find(_templateRunId);
}
if (templateRun != null)
{
// Create run from template
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, string.Format("Creating test run from template {0} ({1})", templateRun.Description, templateRun.Id)));
var run = CreateRunFromTemplate(templateRun, runRepository);
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, string.Format("Created test run (Run ID: {0})", run.Id)));
// Validate that run can be started
IEnumerable <SystemMessage> validationMessages = null;
try
{
validationMessages = _runManager.ValidateForStartRun(run);
}
catch (System.Exception exception)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Cannot perform test run because validation failed: {0}", exception.Message), ""));
}
if (!validationMessages.Any())
{
// Start run
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, string.Format("Starting test run {0} ({1})", run.Description, run.Id)));
var runInstances = StartRun(run, runRepository);
bool isRunStarted = runInstances.Any();
try
{
// If run not started (E.g. Provisioning, no free AutoBooks, no working AutoBooks) then try and do something to get them to start. Most likely then
// there's an AutoBook provisioning, in which case we just have to wait, but we may as well restart crashed instances too.
if (!isRunStarted)
{
// Restart non-working AutoBooks
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, "Restarting any non-working AutoBooks"));
int countRestarted = _autoBooks.RestartNonWorking().Count;
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, string.Format("Restarted {0} non-working AutoBooks", countRestarted)));
// Wait for provisioning AutoBooks
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, "Waiting for any AutoBooks that are provisioning"));
int countProvisioned = _autoBooks.WaitForProvisioned().Count;
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, string.Format("Waited for {0} AutoBooks that are provisioning", countProvisioned)));
// Wait for an AutoBook to pick up the run
if (countRestarted > 0 || countProvisioned > 0)
{
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, "Waiting for any AutoBooks to start run"));
DateTime wait = DateTime.UtcNow.AddSeconds(30);
do
{
System.Threading.Thread.Sleep(200);
} while (DateTime.UtcNow < wait);
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, "Waited for any AutoBooks to start run"));
}
// Refresh run details
using (var runScope = scope.BeginRepositoryScope())
{
runRepository = runScope.CreateRepository <IRunRepository>();
run = runRepository.Find(run.Id);
}
// Check if any scenario is running
isRunStarted = run.Scenarios.Where(s => (s.IsScheduledOrRunning && s.Status != ScenarioStatuses.Scheduled) || (s.IsCompleted)).Any();
}
if (isRunStarted) // Running
{
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, string.Format("Started test run (Scenarios started={0})", runInstances.Count)));
// Wait for run to complete
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, "Waiting for run to complete"));
runCompleted = WaitForRunCompleted(run.Id, TimeSpan.FromMinutes(_scenarioCompleteTimeoutMins));
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, string.Format("Waited for run to complete (Completed={0})", runCompleted)));
// Refresh run details
using (var runScope = scope.BeginRepositoryScope())
{
runRepository = runScope.CreateRepository <IRunRepository>();
run = runRepository.Find(run.Id);
}
// Check run results
ISystemTest runOutputTest = new RunOutputTest(run, _category, _repositoryFactory, _scenarioCompleteTimeoutMins);
results.AddRange(runOutputTest.Execute(systemTestCategory));
}
else // Run not started (E.g. Provisioning AutoBook, no working AutoBooks)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "Unable to start test run", ""));
}
}
finally
{
// Refresh run details
using (var runScope = scope.BeginRepositoryScope())
{
runRepository = runScope.CreateRepository <IRunRepository>();
run = runRepository.Find(run.Id);
}
// Reset scenario status to not in progress otherwise other runs can't start
bool resetRun = false;
foreach (var scenario in run.Scenarios)
{
if (Array.IndexOf(new ScenarioStatuses[] { ScenarioStatuses.Pending, ScenarioStatuses.CompletedError, ScenarioStatuses.CompletedSuccess, ScenarioStatuses.Deleted }, scenario.Status) == -1)
{
resetRun = true;
scenario.ResetToPendingStatus();
}
}
if (resetRun)
{
runRepository.Add(run);
runRepository.SaveChanges();
}
}
}
else // Cannot execute run, validation message
{
foreach (var validationMessage in validationMessages)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Cannot execute test run because validation failed: {0}", validationMessage.Description[Globals.SupportedLanguages[0]]), ""));
}
}
}
}
}
catch (System.Exception exception)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Error executing test run: {0}", exception.Message), ""));
}
if (results.Where(r => r.ResultType == SystemTestResult.ResultTypes.Error).Any())
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "Test run failed", ""));
}
else
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Information, _category, "Test run OK", ""));
}
return(results);
}