public List <SystemTestResult> Execute(SystemTestCategories systemTestCategory)
{
var results = new List <SystemTestResult>();
try
{
string connectionString = _applicationConfiguration["db:Master:connectionString"];
using (IDocumentStore documentStore = DocumentStoreFactory.CreateStore(connectionString, System.Reflection.Assembly.GetExecutingAssembly()))
{
var databaseConfiguration = documentStore.DatabaseCommands.Admin.GetDatabaseConfiguration();
if (databaseConfiguration != null)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Information, _category, "Database connectivity test OK", ""));
}
else
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Database doesn't exist. Please check the connection string"), ""));
}
}
}
catch (Exception exception)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Unable to connect to database and run a query: {0}. Please check the connection to the Database Server", exception.Message), ""));
}
return(results);
}
public List <SystemTestResult> Execute(SystemTestCategories systemTestCategory)
{
var results = new List <SystemTestResult>();
MailMessage mailMessage = new MailMessage()
{
Subject = "Test email",
Sender = new MailAddress(Globals.EngineeringEmailAddress),
From = new MailAddress(Globals.EngineeringEmailAddress),
Body = "Test email",
IsBodyHtml = false,
};
mailMessage.To.Add(new MailAddress(Globals.EngineeringEmailAddress));
try
{
IEmailConnection emailConnection = EmailUtilities.GetEmailConnection(_applicationConfiguration);
emailConnection.SendEmail(mailMessage);
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Warning, _category, "Email test OK.", "")); // Email passed to server, we don't know for certain that it will arrive
}
catch (System.Exception exception)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Warning, _category, String.Format("Error sending test email: {0}. Notifications will not work. Please check the configuration.", exception.Message), ""));
}
return(results);
}
public List <SystemTestResult> Execute(SystemTestCategories systemTestCategory)
{
var results = new List <SystemTestResult>();
try
{
// Create test task instance
TaskInstance taskInstance = TaskInstanceFactory.CreateTestTaskInstance(0, 3);
// Execute task, wait for completion, will fail if exe missing, insufficient folder/file permissions
//taskResult not used anywhere in this test - so commented out
//TaskResult taskResult = new ProcessTaskExecutor(_rootFolder, _repositoryFactory, TimeSpan.FromSeconds(10)).Execute(taskInstance);
if (taskInstance.Status != TaskInstanceStatues.CompletedSuccess)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "Error testing task executor. It may cause runs to fail.", ""));
}
}
catch (System.Exception exception)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Error {0} testing task executor. It may cause runs to fail.", exception.Message), ""));
}
if (!results.Where(r => r.ResultType == SystemTestResult.ResultTypes.Error).Any())
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Information, _category, "Task executor test OK", ""));
}
return(results);
}
public List <SystemTestResult> Execute(SystemTestCategories systemTestCategory)
{
var results = new List <SystemTestResult>();
try
{
// Get API version
APIVersionModel apiVersion = GetVersion();
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Information, _category, string.Format("API version: {0}", apiVersion.Version), ""));
// Check binaries version
if (String.IsNullOrEmpty(_autoBooks.Settings.BinariesVersion))
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "Binaries version: Not set. It will not be possible to start any runs", ""));
}
else
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Information, _category, string.Format("Binaries version: {0}", _autoBooks.Settings.BinariesVersion), ""));
}
// Check AutoBook API version
if (String.IsNullOrEmpty(_autoBooks.Settings.ApplicationVersion))
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "AutoBook API (Default) version: Not set. It will not be possible to start any runs.", ""));
}
else
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Information, _category, string.Format("AutoBook API (Default) version: {0}", _autoBooks.Settings.ApplicationVersion), ""));
}
// Check Provisioning API version
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Warning, _category, "Provisioning API version: Unknown (Not currently checked).", ""));
// TODO: Add API to frontend
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Warning, _category, "Frontend version: Unknown (It is not currently possible to automatically determine the version).", ""));
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Warning, _category, "Database version: Unknown (It is not currently possible to automatically determine the version).", ""));
results.AddRange(TestComponentCompatibility("", "", "", "", "", ""));
}
catch (System.Exception exception)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Error checking component versions: {0}", exception.Message), ""));
}
finally
{
if (!results.Where(r => r.ResultType == SystemTestResult.ResultTypes.Error).Any())
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Information, _category, "Version test OK.", ""));
}
}
return(results);
}
public List <SystemTestResult> Execute(SystemTestCategories systemTestCategory)
{
var results = new List <SystemTestResult>();
try
{
// Execute logging persistence check
results.AddRange(ExecuteLoggingTest());
}
catch (System.Exception exception)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Error testing event logging: {0}", exception.Message), ""));
}
return(results);
}
public List <SystemTestResult> Execute(SystemTestCategories systemTestCategory)
{
List <SystemTestResult> results = new List <SystemTestResult>();
try
{
// Execute run notification HTTP tests
results.AddRange(ExecuteRunNotificationHTTPTests());
}
catch (System.Exception exception)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Error checking run notification: {0}", exception.Message), ""));
}
return(results);
}
public List <SystemTestResult> Execute(SystemTestCategories systemTestCategory)
{
var results = new List <SystemTestResult>();
int hoursSinceRunStarted = 24;
int scenarioCompleteTimeoutMins = 120;
try
{
using (var scope = _repositoryFactory.BeginRepositoryScope())
{
var runRepository = scope.CreateRepository <IRunRepository>();
// Get any runs started in last N hours
int countFailedRuns = 0;
var runs = runRepository.GetAll().Where(r => r.FirstScenarioStartedDateTime != null && r.FirstScenarioStartedDateTime >= DateTime.UtcNow.AddHours(-hoursSinceRunStarted)).OrderBy(r => r.CreatedDateTime);
// Check runs
foreach (var run in runs)
{
// Count up failures
countFailedRuns += run.Scenarios.Where(s => s.Status == ScenarioStatuses.CompletedError).ToList().Count;
try
{
ISystemTest runOutputTest = new RunOutputTest(run, _category, _repositoryFactory, scenarioCompleteTimeoutMins);
results.AddRange(runOutputTest.Execute(systemTestCategory));
}
catch { }; // Ignore
}
if (!runs.Any())
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Warning, _category, string.Format("No runs were started in the last {0} hours", hoursSinceRunStarted), ""));
}
else if (countFailedRuns == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Information, _category, string.Format("No failed runs in the last {0} hours", hoursSinceRunStarted), ""));
}
}
}
catch (System.Exception exception)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Error checking run history: {0}", exception.Message), ""));
}
return(results);
}
/// <summary>
/// Returns list of system tests
/// </summary>
/// <returns></returns>
private List <ISystemTest> GetSystemTests(SystemTestCategories systemTestCategory)
{
var systemTests = new List <ISystemTest>();
systemTests.Add(new AuditEventTest(_auditEventRepository));
systemTests.Add(new DatabaseConnectivityTest(_applicationConfiguration));
systemTests.Add(new StaticDataTest(_repositoryFactory, _applicationConfiguration));
systemTests.Add(new ScheduleDataTest(_repositoryFactory));
systemTests.Add(new AutoBookTest(_autoBooks, _repositoryFactory));
systemTests.Add(new ScenarioPassLibraryTest(_repositoryFactory));
systemTests.Add(new RunExecuteTest(_runManager, _autoBooks, _repositoryFactory, RunExecuteTest.DefaultTemplateRunId, _auditEventRepository, 15, _identityGeneratorResolver));
systemTests.Add(new RunHistoryTest(_repositoryFactory));
systemTests.Add(new RunNotificationTest(_repositoryFactory));
systemTests.Add(new EmailTest(_applicationConfiguration));
systemTests.Add(new CloudTest(_cloudStorage, _awsSettings));
systemTests.Add(new ComponentVersionTest(_autoBooks));
return(systemTests.Where(st => st.CanExecute(systemTestCategory)).ToList());
}
/// <summary>
/// Executes tests
/// </summary>
/// <returns></returns>
public List <SystemTestResult> ExecuteTests(SystemTestCategories systemTestCategory)
{
// Get all system tests to run for category
var systemTests = GetSystemTests(systemTestCategory);
// Run tests
var allSystemTestResults = new List <SystemTestResult>();
foreach (var systemTest in systemTests)
{
try
{
allSystemTestResults.AddRange(systemTest.Execute(systemTestCategory));
}
catch { }; // Ignore
}
return(allSystemTestResults);
}
public List <SystemTestResult> Execute(SystemTestCategories systemTestCategory)
{
var results = new List <SystemTestResult>();
try
{
using (var scope = _repositoryFactory.BeginRepositoryScope())
{
// Get list of AutoBooks
var autoBookRepository = scope.CreateRepository <IAutoBookRepository>();
var autoBooks = autoBookRepository.GetAll();
// Check AutoBook provisioning settings
results.AddRange(ExecuteProvisioningTests(autoBooks.ToList()));
// Warn if no AutoBooks exist
if (autoBookRepository.GetAll().ToList().Count == 0)
{
if (!_autoBooks.Settings.AutoProvisioning)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "No AutoBook data exists and auto-provisioning is disabled. It will not be possible to execute any runs.", ""));
}
}
else
{
// Check connectivity to each AutoBook
int countWorkingAutoBooks = 0;
int countFreeAutoBooks = 0;
foreach (var autoBook in autoBooks)
{
try
{
IAutoBook autoBookInterface = _autoBooks.GetInterface(autoBook);
var autoBookStatus = autoBookInterface.GetStatus();
if (AutoBook.IsWorkingStatus(autoBookStatus))
{
countWorkingAutoBooks++;
if (autoBookStatus == AutoBookStatuses.Idle)
{
countFreeAutoBooks++;
}
}
else if (autoBookStatus == AutoBookStatuses.Fatal_Error)
{
if (!_autoBooks.Settings.AutoProvisioning)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Warning, _category, string.Format("AutoBook {0} has indicated that is has encountered a fatal error. It cannot be used for any runs while it is in this state. Please manually restart it.", autoBook.Id), ""));
}
}
}
catch
{
if (autoBook.Status != AutoBookStatuses.Provisioning) // Ignore error if Provisioning
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Warning, _category, string.Format("Error testing connection to AutoBook {0} ({1})", autoBook.Id, autoBook.Api), ""));
}
}
}
if (countWorkingAutoBooks == 0)
{
if (!_autoBooks.Settings.AutoProvisioning)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "There are no working AutoBooks. It will not be possible to start any runs.", ""));
}
}
else if (countFreeAutoBooks == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Warning, _category, string.Format("All {0} working AutoBooks are currently executing a run. It will not be possible to start any runs until existing runs have completed.", countWorkingAutoBooks), ""));
}
else if (results.Count == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Information, _category, string.Format("AutoBook test OK ({0} working AutoBooks)", countWorkingAutoBooks), ""));
}
}
}
}
catch (System.Exception exception)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Error checking AutoBooks: {0}", exception.Message), ""));
}
return(results);
}
public bool CanExecute(SystemTestCategories systemTestCategories)
{
return(true);
}
public List <SystemTestResult> Execute(SystemTestCategories systemTestCategory)
{
List <SystemTestResult> results = new List <SystemTestResult>();
try
{
using (var scope = _repositoryFactory.BeginRepositoryScope())
{
// Get sales areas for run
var salesAreaRepository = scope.CreateRepository <ISalesAreaRepository>();
var salesAreas = RunManagement.RunManager.GetSalesAreas(_run, salesAreaRepository.GetAll());
// Check scenarios
foreach (var scenario in _run.Scenarios)
{
// Warn of possibly crashed scenarios, running for too long
if (scenario.StartedDateTime != null && scenario.IsScheduledOrRunning && scenario.Status != ScenarioStatuses.Scheduled)
{
if (scenario.StartedDateTime.Value.AddMinutes(_scenarioCompleteTimeoutMins) <= DateTime.UtcNow) // Hard-coded limit for the moment
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Warning, _category, string.Format("Run {0} (RunID {1}) has a scenario (ScenarioID {2}) that was started at {3} and it is still running after {4} mins." +
"Please check if it has crashed.",
_run.Description, _run.Id, scenario.Id, scenario.StartedDateTime, _scenarioCompleteTimeoutMins), ""));
}
}
switch (scenario.Status)
{
case ScenarioStatuses.CompletedError:
//countFailedRuns++;
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Run {0} (Run ID {1}) has a scenario (Scenario ID {2}) that completed with errors. The Optimiser Report will indicate why it failed.", _run.Description, _run.Id, scenario.Id), ""));
break;
case ScenarioStatuses.Scheduled:
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Run {0} (Run ID {1}) has a scenario (Scenario ID {2}) that has been scheduled but not started. Please check if there is a problem with the AutoBooks.", _run.Description, _run.Id, scenario.Id), ""));
break;
}
// Check recommendations output data
results.AddRange(CheckRecommendationOutputData(_run, salesAreas, scenario));
// Check failures output data
results.AddRange(CheckFailuresOutputData(_run, salesAreas, scenario));
// Check scenario results output data
results.AddRange(CheckScenarioResultsOutputData(_run, salesAreas, scenario));
}
// Check Smooth failures output data
results.AddRange(CheckSmoothFailuresOutputData(_run, salesAreas));
// Check break efficiency output data
results.AddRange(CheckBreakEfficiencyOutputData(_run, salesAreas));
}
}
catch (System.Exception exception)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Error checking run output (Run ID {0}): {1}", _run.Id, exception.Message), ""));
}
finally
{
if (!results.Where(r => r.ResultType == SystemTestResult.ResultTypes.Error).Any() && _run.Scenarios.Where(s => s.Status == ScenarioStatuses.CompletedSuccess).ToList().Count == _run.Scenarios.Count)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Information, _category, "Run output OK", ""));
}
}
return(results);
}
public IHttpActionResult Execute(SystemTestCategories category = SystemTestCategories.Default)
{
var systemTestResults = _systemTestsManager.ExecuteTests(category);
return(Ok(systemTestResults));
}
public List <SystemTestResult> Execute(SystemTestCategories systemTestCategory)
{
var results = new List <SystemTestResult>();
Model.S3UploadComment upload = null;
Model.S3DownloadComment download = null;
bool success = false;
// Upload test file
Guid id = Guid.NewGuid();
string localTempFolder = System.Web.Hosting.HostingEnvironment.MapPath("/Temp");
string localOutputFolder = System.Web.Hosting.HostingEnvironment.MapPath("/Output");
string localTempFile = Path.Combine(localTempFolder, string.Format("{0}.tmp", id));
int action = 0; // 1=Upload, 2=Download
try
{
// Create local file
Directory.CreateDirectory(localTempFolder);
File.WriteAllText(localTempFile, new String('X', 1024 * 10));
// Upload to cloud
action = 1;
upload = new Model.S3UploadComment()
{
BucketName = _awsSettings.S3Bucket,
DestinationFilePath = string.Format("input/Test-{0}.tmp", id.ToString()),
SourceFilePath = localTempFile
};
_cloudStorage.Upload(upload);
// Download from cloud
action = 2;
download = new Model.S3DownloadComment()
{
BucketName = _awsSettings.S3Bucket,
FileName = string.Format("input/Test-{0}.tmp", id.ToString()),
LocalFileFolder = localOutputFolder
};
_cloudStorage.Download(download);
success = true;
}
catch (System.Exception exception)
{
switch (action)
{
case 0:
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Error creating file for testing the cloud storage upload/download: {0}. It will cause runs to fail.", exception.Message), ""));
break;
case 1:
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Error uploading file to cloud storage: {0}. It will cause runs to fail.", exception.Message), ""));
break;
case 2:
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Error downloading file from cloud storage: {0}. It will cause runs to fail.", exception.Message), ""));
break;
}
}
finally
{
// Delete local file
try
{
if (File.Exists(localTempFile))
{
File.Delete(localTempFile);
}
}
catch { }; // Ignore
// Delete cloud file
try
{
if (upload != null)
{
Model.S3FileComment delete = new Model.S3FileComment()
{
BucketName = _awsSettings.S3Bucket,
FileNameWithPath = upload.DestinationFilePath
};
_cloudStorage.DeleteFile(delete);
}
}
catch { }; // Ignore
if (success)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Information, _category, "Cloud test OK", ""));
}
}
return(results);
}
public List <SystemTestResult> Execute(SystemTestCategories systemTestCategory)
{
var results = new List <SystemTestResult>();
try
{
using (var scope = _repositoryFactory.BeginRepositoryScope())
{
var repositoryList = scope.CreateRepositories(
typeof(IBreakRepository),
typeof(ICampaignRepository),
typeof(IClashRepository),
typeof(IProductRepository),
typeof(IProgrammeRepository),
typeof(IProgrammeDictionaryRepository),
typeof(IRatingsScheduleRepository),
typeof(IScheduleRepository),
typeof(ISpotRepository)
);
var breakRepository = repositoryList.Get <IBreakRepository>();
var campaignRepository = repositoryList.Get <ICampaignRepository>();
var clashRepository = repositoryList.Get <IClashRepository>();
var productRepository = repositoryList.Get <IProductRepository>();
var programmeRepository = repositoryList.Get <IProgrammeRepository>();
var programmeDictionaryRepository = repositoryList.Get <IProgrammeDictionaryRepository>();
var ratingsScheduleRepository = repositoryList.Get <IRatingsScheduleRepository>();
var scheduleRepository = repositoryList.Get <IScheduleRepository>();
var spotRepository = repositoryList.Get <ISpotRepository>();
if (breakRepository.CountAll == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "No Break data exists. It will cause runs to fail.", ""));
}
if (campaignRepository.CountAll == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "No Campaign data exists. It will cause runs to fail.", ""));
}
if (clashRepository.CountAll == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "No Clash data exists. It will cause runs to fail.", ""));
}
if (productRepository.CountAll == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "No Product data exists. It will cause runs to fail.", ""));
}
if (programmeRepository.CountAll == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "No Programme data exists. It will cause runs to fail.", ""));
}
if (programmeDictionaryRepository.CountAll == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "No Programme Dictionary data exists. It will cause runs to fail.", ""));
}
if (ratingsScheduleRepository.CountAll == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "No Ratings Schedule data exists. It will cause runs to fail.", ""));
}
if (scheduleRepository.CountAll == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "No Schedule data exists. It will cause runs to fail.", ""));
}
if (spotRepository.CountAll == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "No Spot data exists. It will cause runs to fail.", ""));
}
}
}
catch (System.Exception exception)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Error checking schedule data: {0}", exception.Message), ""));
}
finally
{
if (!results.Where(r => r.ResultType == SystemTestResult.ResultTypes.Error).Any())
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Information, _category, "Schedule data test OK. Please be aware that only basic tests were performed.", ""));
}
}
return(results);
}
public bool CanExecute(SystemTestCategories systemTestCategories)
{
return(systemTestCategories != SystemTestCategories.Deployment); // Not relevant for installation, system can be installed before schedule data uploaded for first time
}
public bool CanExecute(SystemTestCategories systemTestCategories)
{
return(systemTestCategories == SystemTestCategories.Deployment); // Only relevant for installation test
}
public List <SystemTestResult> Execute(SystemTestCategories systemTestCategory)
{
var results = new List <SystemTestResult>();
bool runCompleted = false;
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, "Executing test run"));
try
{
using (var scope = _repositoryFactory.BeginRepositoryScope())
{
var runRepository = scope.CreateRepository <IRunRepository>();
// Load template run
Run templateRun = runRepository.Find(_templateRunId);
if (templateRun == null) // Template run does not exist, create it
{
CreateRunForTemplate(_templateRunId);
templateRun = runRepository.Find(_templateRunId);
}
if (templateRun != null)
{
// Create run from template
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, string.Format("Creating test run from template {0} ({1})", templateRun.Description, templateRun.Id)));
var run = CreateRunFromTemplate(templateRun, runRepository);
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, string.Format("Created test run (Run ID: {0})", run.Id)));
// Validate that run can be started
IEnumerable <SystemMessage> validationMessages = null;
try
{
validationMessages = _runManager.ValidateForStartRun(run);
}
catch (System.Exception exception)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Cannot perform test run because validation failed: {0}", exception.Message), ""));
}
if (!validationMessages.Any())
{
// Start run
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, string.Format("Starting test run {0} ({1})", run.Description, run.Id)));
var runInstances = StartRun(run, runRepository);
bool isRunStarted = runInstances.Any();
try
{
// If run not started (E.g. Provisioning, no free AutoBooks, no working AutoBooks) then try and do something to get them to start. Most likely then
// there's an AutoBook provisioning, in which case we just have to wait, but we may as well restart crashed instances too.
if (!isRunStarted)
{
// Restart non-working AutoBooks
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, "Restarting any non-working AutoBooks"));
int countRestarted = _autoBooks.RestartNonWorking().Count;
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, string.Format("Restarted {0} non-working AutoBooks", countRestarted)));
// Wait for provisioning AutoBooks
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, "Waiting for any AutoBooks that are provisioning"));
int countProvisioned = _autoBooks.WaitForProvisioned().Count;
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, string.Format("Waited for {0} AutoBooks that are provisioning", countProvisioned)));
// Wait for an AutoBook to pick up the run
if (countRestarted > 0 || countProvisioned > 0)
{
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, "Waiting for any AutoBooks to start run"));
DateTime wait = DateTime.UtcNow.AddSeconds(30);
do
{
System.Threading.Thread.Sleep(200);
} while (DateTime.UtcNow < wait);
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, "Waited for any AutoBooks to start run"));
}
// Refresh run details
using (var runScope = scope.BeginRepositoryScope())
{
runRepository = runScope.CreateRepository <IRunRepository>();
run = runRepository.Find(run.Id);
}
// Check if any scenario is running
isRunStarted = run.Scenarios.Where(s => (s.IsScheduledOrRunning && s.Status != ScenarioStatuses.Scheduled) || (s.IsCompleted)).Any();
}
if (isRunStarted) // Running
{
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, string.Format("Started test run (Scenarios started={0})", runInstances.Count)));
// Wait for run to complete
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, "Waiting for run to complete"));
runCompleted = WaitForRunCompleted(run.Id, TimeSpan.FromMinutes(_scenarioCompleteTimeoutMins));
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, string.Format("Waited for run to complete (Completed={0})", runCompleted)));
// Refresh run details
using (var runScope = scope.BeginRepositoryScope())
{
runRepository = runScope.CreateRepository <IRunRepository>();
run = runRepository.Find(run.Id);
}
// Check run results
ISystemTest runOutputTest = new RunOutputTest(run, _category, _repositoryFactory, _scenarioCompleteTimeoutMins);
results.AddRange(runOutputTest.Execute(systemTestCategory));
}
else // Run not started (E.g. Provisioning AutoBook, no working AutoBooks)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "Unable to start test run", ""));
}
}
finally
{
// Refresh run details
using (var runScope = scope.BeginRepositoryScope())
{
runRepository = runScope.CreateRepository <IRunRepository>();
run = runRepository.Find(run.Id);
}
// Reset scenario status to not in progress otherwise other runs can't start
bool resetRun = false;
foreach (var scenario in run.Scenarios)
{
if (Array.IndexOf(new ScenarioStatuses[] { ScenarioStatuses.Pending, ScenarioStatuses.CompletedError, ScenarioStatuses.CompletedSuccess, ScenarioStatuses.Deleted }, scenario.Status) == -1)
{
resetRun = true;
scenario.ResetToPendingStatus();
}
}
if (resetRun)
{
runRepository.Add(run);
runRepository.SaveChanges();
}
}
}
else // Cannot execute run, validation message
{
foreach (var validationMessage in validationMessages)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Cannot execute test run because validation failed: {0}", validationMessage.Description[Globals.SupportedLanguages[0]]), ""));
}
}
}
}
}
catch (System.Exception exception)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Error executing test run: {0}", exception.Message), ""));
}
if (results.Where(r => r.ResultType == SystemTestResult.ResultTypes.Error).Any())
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "Test run failed", ""));
}
else
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Information, _category, "Test run OK", ""));
}
return(results);
}
public List <SystemTestResult> Execute(SystemTestCategories systemTestCategory)
{
var results = new List <SystemTestResult>();
try
{
using (var scope = _repositoryFactory.BeginRepositoryScope())
{
var repositories = scope.CreateRepositories(
typeof(IPassRepository),
typeof(IRunRepository),
typeof(IScenarioRepository),
typeof(ITenantSettingsRepository)
);
var passRepository = repositories.Get <IPassRepository>();
var runRepository = repositories.Get <IRunRepository>();
var scenarioRepository = repositories.Get <IScenarioRepository>();
var tenantSettingsRepository = repositories.Get <ITenantSettingsRepository>();
var librariedScenarios = scenarioRepository.GetLibraried();
var defaultScenarioId = tenantSettingsRepository.Get().DefaultScenarioId;
// Check default scenario
if (defaultScenarioId == Guid.Empty) // Not set
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "Default scenario is not set", ""));
}
else // Set, check that it exists
{
var defaultScenario = scenarioRepository.Get(defaultScenarioId);
if (defaultScenario == null)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "Default scenario does not exist", ""));
}
}
// Load runs & scenarios
var runs = runRepository.GetAll().OrderBy(r => r.CreatedDateTime).ThenBy(r => r.Description);
var scenarios = scenarioRepository.GetAll().OrderBy(s => s.Id);
// Check runs
foreach (var run in runs)
{
// Check that runs for scenario exist
var scenarioIdsForRun = run.Scenarios.Select(s => s.Id).ToList();
foreach (var scenarioId in scenarioIdsForRun)
{
var scenario = scenarios.FirstOrDefault(s => s.Id == scenarioId);
if (scenario == null)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Run ID {0} has Scenario ID {1} that does not exist", run.Id, scenarioId), ""));
}
if (librariedScenarios.Any(x => x.Id == scenarioId))
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Run ID {0} has Scenario ID {1} that is libraried", run.Id, scenarioId), ""));
}
if (scenarioId == defaultScenarioId)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Run ID {0} has Scenario ID {1} that is the default scenario", run.Id, scenarioId), ""));
}
}
if (run.Scenarios.Select(s => s.Id).Distinct().ToList().Count != run.Scenarios.Count)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Run ID {0} has duplicate scenarios", run.Id), ""));
}
if (run.Scenarios.Count == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Run ID {0} has no scenarios", run.Id), ""));
}
}
// Check each scenario. E.g. Passes exist
Dictionary <int, Guid> firstScenarioByPassId = new Dictionary <int, Guid>();
foreach (var scenario in scenarios)
{
// Get list of Run IDs for this scenario, will be one at most
var runsForScenario = runs.Where(r => r.Scenarios.FindIndex(s => s.Id == scenario.Id) != -1).ToList();
System.Text.StringBuilder runDescriptions = new System.Text.StringBuilder("");
foreach (var run in runsForScenario)
{
if (runDescriptions.Length > 0)
{
runDescriptions.Append(", ");
}
runDescriptions.Append(run.Id.ToString());
}
if (runsForScenario.Count > 1)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Scenario ID {0} is used by multiple runs (Run IDs {1})", scenario.Id, runDescriptions.ToString()), ""));
}
// Check that all passes exist
var passesForScenario = passRepository.FindByIds(scenario.Passes.Select(p => p.Id));
foreach (var passReference in scenario.Passes)
{
// Check that pass isn't referenced by another scenario
if (firstScenarioByPassId.ContainsKey(passReference.Id))
{
if (runDescriptions.Length == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Scenario ID {0} has a Pass ID {1} that is also used by Scenario ID {2}", scenario.Id, passReference.Id, firstScenarioByPassId[passReference.Id]), ""));
}
else
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Scenario ID {0} has a Pass ID {1} that is also used by Scenario ID {2} (Run IDs: {3})", scenario.Id, passReference.Id, firstScenarioByPassId[passReference.Id], runDescriptions.ToString()), ""));
}
}
else
{
firstScenarioByPassId.Add(passReference.Id, scenario.Id);
}
var pass = passesForScenario.FirstOrDefault(p => p.Id == passReference.Id);
if (pass == null)
{
if (runDescriptions.Length == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Scenario ID {0} has a Pass ID {1} that does not exist", scenario.Id, passReference.Id), ""));
}
else
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Scenario ID {0} has a Pass ID {1} that does not exist (Run IDs: {2})", scenario.Id, passReference.Id, runDescriptions.ToString()), ""));
}
}
}
if (scenario.Passes.Select(p => p.Id).Distinct().ToList().Count != scenario.Passes.Count)
{
if (runDescriptions.Length == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Scenario ID {0} has duplicate passes", scenario.Id), ""));
}
else
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Scenario ID {0} has duplicate passes (Run IDs: {1})", scenario.Id, runDescriptions.ToString()), ""));
}
}
if (scenario.Passes.Count == 0)
{
if (runDescriptions.Length == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Scenario ID {0} has no passes", scenario.Id), ""));
}
else
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Scenario ID {0} has no passes (Run IDs: {1})", scenario.Id, runDescriptions.ToString()), ""));
}
}
}
}
}
catch (System.Exception exception)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Error checking Scenario/Pass library: {0}", exception.Message), ""));
}
finally
{
if (!results.Where(r => r.ResultType == SystemTestResult.ResultTypes.Error).Any())
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Information, _category, "Scenario/Pass library OK", ""));
}
}
return(results);
}
public bool CanExecute(SystemTestCategories systemTestCategories) => true;
public bool CanExecute(SystemTestCategories systemTestCategories)
{
return(systemTestCategories != SystemTestCategories.Deployment); // Run history isn't relevant for installation test
}
public List <SystemTestResult> Execute(SystemTestCategories systemTestCategory)
{
var results = new List <SystemTestResult>();
try
{
using (var scope = _repositoryFactory.BeginRepositoryScope())
{
var repositoryList = scope.CreateRepositories(
typeof(IFunctionalAreaRepository),
typeof(IISRSettingsRepository),
typeof(IRSSettingsRepository),
typeof(ILanguageRepository),
typeof(IMetadataRepository),
typeof(ISalesAreaRepository),
typeof(IOutputFileRepository),
typeof(IUniverseRepository),
typeof(IEmailAuditEventSettingsRepository),
typeof(IAutoBookInstanceConfigurationRepository),
typeof(IAutoBookSettingsRepository),
typeof(ISmoothFailureMessageRepository),
typeof(IDemographicRepository),
typeof(IProgrammeClassificationRepository),
typeof(IProgrammeCategoryHierarchyRepository));
var functionalAreaRepository = repositoryList.Get <IFunctionalAreaRepository>();
var languageRepository = repositoryList.Get <ILanguageRepository>();
var metadataRepository = repositoryList.Get <IMetadataRepository>();
var salesAreaRepository = repositoryList.Get <ISalesAreaRepository>();
var outputFileRepository = repositoryList.Get <IOutputFileRepository>();
var isrSettingsRepository = repositoryList.Get <IISRSettingsRepository>();
var rsSettingsRepository = repositoryList.Get <IRSSettingsRepository>();
var universeRepository = repositoryList.Get <IUniverseRepository>();
var autoBookInstanceConfigurationRepository = repositoryList.Get <IAutoBookInstanceConfigurationRepository>();
var autoBookSettingsRepository = repositoryList.Get <IAutoBookSettingsRepository>();
var smoothFailureMessageRepository = repositoryList.Get <ISmoothFailureMessageRepository>();
var demographicRepository = repositoryList.Get <IDemographicRepository>();
var programmeClassificationRepository = repositoryList.Get <IProgrammeClassificationRepository>();
var programmeCategoryRepository = repositoryList.Get <IProgrammeCategoryHierarchyRepository>();
var salesAreas = salesAreaRepository.GetAll().ToList();
/*
* if (demographicRepository.GetAll().ToList().Count == 0)
* {
* results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, "No Demographic data exists. It will cause runs to fail."));
* }
*/
if (demographicRepository.CountAll == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "No Demographic data exists. It will cause runs to fail.", ""));
}
if (metadataRepository.GetByKey(MetaDataKeys.BreakTypes).Count == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "No Break Types data exists. It will cause runs to fail.", ""));
}
if (functionalAreaRepository.GetAll().ToList().Count == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "No Functional Area data exists. It will cause errors displaying Failure data when the run is complete.", ""));
}
if (languageRepository.GetAll().ToList().Count == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "No Language data exists.", ""));
}
if (salesAreaRepository.GetAll().ToList().Count == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "No Sales Area data exists. It will cause runs to fail.", ""));
}
if (outputFileRepository.GetAll().ToList().Count == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "No Output File data exists. It will cause runs to fail. It is needed for processing output files.", ""));
}
var isrSettingsList = isrSettingsRepository.GetAll();
if (isrSettingsList.Count == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "No ISR Settings data exists. It will cause runs to fail if ISR is requested.", ""));
}
else if (isrSettingsList.Count < salesAreas.Count)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "ISR Settings data is missing for some sales areas. It will cause runs to fail if ISR is requested.", ""));
}
var rsSettingsList = rsSettingsRepository.GetAll();
if (rsSettingsList.Count == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "No Right Sizer Settings data exists. It will cause runs to fail if Right Sizer is requested.", ""));
}
else if (rsSettingsList.Count < salesAreas.Count)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "Right Sizer Settings data is missing for some sales areas. It will cause runs to fail if Right Sizer is requested.", ""));
}
if (universeRepository.GetAll().ToList().Count == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "No Universe data exists. It will cause runs to fail.", ""));
}
if (programmeClassificationRepository.CountAll == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "No Programme Classificiation data exists. It will cause runs to fail.", "")); // Not mandatory
}
if (programmeCategoryRepository.GetAll().ToList().Count == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "No Programme Category Hierarchy data exists. It will cause runs to fail.", ""));
}
// Test internal users
//results.AddRange(TestInternalUsers());
}
}
catch (System.Exception exception)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, String.Format("Error checking static data: {0}", exception.Message), ""));
}
finally
{
if (!results.Where(r => r.ResultType == SystemTestResult.ResultTypes.Error).Any())
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Information, _category, "Static data test OK", ""));
}
}
return(results);
}
