public void Execute(
Run run,
RunScenario scenario,
IReadOnlyCollection <AutoBookInstanceConfiguration> autoBookInstanceConfigurationsForRun,
double autoBookRequiredStorageGB,
ConcurrentBag <RunInstance> runInstances,
ConcurrentDictionary <Guid, ScenarioStatuses> newScenarioStatuses,
ConcurrentDictionary <Guid, bool> scenarioSyncStatuses,
bool autoDistributed)
{
AutoBookDomainObject autoBook = null;
IAutoBook autoBookInterface = null;
bool runStarted = false;
RaiseInfo($"Begin Execute for ScenarioID: { scenario.Id}");
try
{
AutoBookInstanceConfiguration runAutoBookInstanceConfiguration = null;
if (autoDistributed)
{
RaiseInfo($"AutoDistributed - RunScenarioTask Execute Starting ScenarioID ={ scenario.Id}, RunID ={ run.Id}");
//create instance for scenario
RunInstance runInstance = _runInstanceCreator.Create(run.Id, scenario.Id);
RaiseInfo($"AutoDistributed - about to enter: {nameof(runInstance.UploadInputFilesAndCreateAutoBookRequest)}");
runInstance.UploadInputFilesAndCreateAutoBookRequest(autoBookInstanceConfigurationsForRun, autoBookRequiredStorageGB);
RaiseInfo($"AutoDistributed - returned from: {nameof(runInstance.UploadInputFilesAndCreateAutoBookRequest)}");
// Flag run as started
runStarted = true;
runInstances.Add(runInstance);
_ = newScenarioStatuses.TryRemove(scenario.Id, out _); // Don't update scenario status at the end
scenarioSyncStatuses[scenario.Id] = false;
RaiseInfo($"AutoDistributed - RunScenarioTask Execute Started ScenarioID ={ scenario.Id}, RunID ={ run.Id}");
}
else
{
try
{
using (MachineLock.Create("xggameplan.AWSAutoBooks.GetFreeAutoBook",
new TimeSpan(0, 10, 0)))
{
foreach (var autoBookInstanceConfiguration in autoBookInstanceConfigurationsForRun)
{
autoBook = _autoBooks.GetFirstAdequateIdleAutoBook(autoBookInstanceConfiguration, autoBookRequiredStorageGB, true);
if (autoBook != null) // Got free AutoBook
{
RaiseInfo($"Got Free AutoBook: {autoBook.Id} ConfigurationId: {autoBook.InstanceConfigurationId}");
runAutoBookInstanceConfiguration = autoBookInstanceConfiguration;
break;
}
}
}
}
catch (MachineLockTimeoutException)
{
RaiseInfo($"MachineLockTimeoutException in xggameplan.AWSAutoBooks.GetFreeAutoBook");
}
// Get autobook interface
autoBookInterface = (autoBook == null) ? null : _autoBooks.GetInterface(autoBook);
// Get free AutoBook instance, will be locked so that it can't be used elsewhere
if (autoBook != null) // Free AutoBook - start run
{
RaiseInfo($"Free Autobook - Starting ScenarioID ={ scenario.Id}, AutoBookID ={autoBook?.Id}, RunID ={ run.Id}, Instance Configuration = { runAutoBookInstanceConfiguration.Description }");
// Start run, exception will cause cleanup below
RunInstance runInstance = _runInstanceCreator.Create(run.Id, scenario.Id);
runInstance.UploadInputFilesStartAutoBookRun(autoBookInterface, autoBook);
// Flag run as started
runStarted = true;
runInstances.Add(runInstance);
_ = newScenarioStatuses.TryRemove(scenario.Id, out _); // Don't update scenario status at the end
scenarioSyncStatuses[scenario.Id] = false;
RaiseInfo($"Started ScenarioID ={ scenario.Id}, AutoBookID ={ autoBook?.Id}, RunID ={ run.Id}, Instance Configuration = { runAutoBookInstanceConfiguration?.Description }");
}
else // No free AutoBook, awaiting for provisioning
{
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForWarningMessage(0, 0,
$"No free AutoBook, awaiting for provisioning, waiting for existing AutoBooks to be Idle (RunID={run.Id}, ScenarioID={scenario.Id})"));
// Update scenario so that it can be retried later when an AutoBook becomes idle
RunManager.UpdateScenarioStatuses(_repositoryFactory, _auditEventRepository, run.Id, new List <Guid> {
scenario.Id
},
new List <ScenarioStatuses> {
ScenarioStatuses.Scheduled
}, new List <DateTime?> {
null
});
_ = newScenarioStatuses.TryRemove(scenario.Id, out _); // Don't update scenario status at the end
scenarioSyncStatuses[scenario.Id] = false;
}
}
}
catch (System.Exception exception)
{
// Log exception but don't throw it. We want to try and start other scenarios
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForException(0, 0, $"Error starting scenario (RunID={run.Id}, ScenarioID={scenario.Id}, AutoBookID={(autoBook == null ? "Unknown" : autoBook.Id)})", exception));
}
finally
{
// If we locked a free AutoBook instance but didn't start the scenario then reset to free, unlocks it.
if (!runStarted && autoBook != null)
{
autoBookInterface.ResetFree();
}
}
}
public List <SystemTestResult> Execute(SystemTestCategories systemTestCategory)
{
var results = new List <SystemTestResult>();
try
{
using (var scope = _repositoryFactory.BeginRepositoryScope())
{
// Get list of AutoBooks
var autoBookRepository = scope.CreateRepository <IAutoBookRepository>();
var autoBooks = autoBookRepository.GetAll();
// Check AutoBook provisioning settings
results.AddRange(ExecuteProvisioningTests(autoBooks.ToList()));
// Warn if no AutoBooks exist
if (autoBookRepository.GetAll().ToList().Count == 0)
{
if (!_autoBooks.Settings.AutoProvisioning)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "No AutoBook data exists and auto-provisioning is disabled. It will not be possible to execute any runs.", ""));
}
}
else
{
// Check connectivity to each AutoBook
int countWorkingAutoBooks = 0;
int countFreeAutoBooks = 0;
foreach (var autoBook in autoBooks)
{
try
{
IAutoBook autoBookInterface = _autoBooks.GetInterface(autoBook);
var autoBookStatus = autoBookInterface.GetStatus();
if (AutoBook.IsWorkingStatus(autoBookStatus))
{
countWorkingAutoBooks++;
if (autoBookStatus == AutoBookStatuses.Idle)
{
countFreeAutoBooks++;
}
}
else if (autoBookStatus == AutoBookStatuses.Fatal_Error)
{
if (!_autoBooks.Settings.AutoProvisioning)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Warning, _category, string.Format("AutoBook {0} has indicated that is has encountered a fatal error. It cannot be used for any runs while it is in this state. Please manually restart it.", autoBook.Id), ""));
}
}
}
catch
{
if (autoBook.Status != AutoBookStatuses.Provisioning) // Ignore error if Provisioning
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Warning, _category, string.Format("Error testing connection to AutoBook {0} ({1})", autoBook.Id, autoBook.Api), ""));
}
}
}
if (countWorkingAutoBooks == 0)
{
if (!_autoBooks.Settings.AutoProvisioning)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "There are no working AutoBooks. It will not be possible to start any runs.", ""));
}
}
else if (countFreeAutoBooks == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Warning, _category, string.Format("All {0} working AutoBooks are currently executing a run. It will not be possible to start any runs until existing runs have completed.", countWorkingAutoBooks), ""));
}
else if (results.Count == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Information, _category, string.Format("AutoBook test OK ({0} working AutoBooks)", countWorkingAutoBooks), ""));
}
}
}
}
catch (System.Exception exception)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Error checking AutoBooks: {0}", exception.Message), ""));
}
return(results);
}