public void Execute(
Run run,
RunScenario scenario,
IReadOnlyCollection <AutoBookInstanceConfiguration> autoBookInstanceConfigurationsForRun,
double autoBookRequiredStorageGB,
ConcurrentBag <RunInstance> runInstances,
ConcurrentDictionary <Guid, ScenarioStatuses> newScenarioStatuses,
ConcurrentDictionary <Guid, bool> scenarioSyncStatuses,
bool autoDistributed)
{
AutoBookDomainObject autoBook = null;
IAutoBook autoBookInterface = null;
bool runStarted = false;
RaiseInfo($"Begin Execute for ScenarioID: { scenario.Id}");
try
{
AutoBookInstanceConfiguration runAutoBookInstanceConfiguration = null;
if (autoDistributed)
{
RaiseInfo($"AutoDistributed - RunScenarioTask Execute Starting ScenarioID ={ scenario.Id}, RunID ={ run.Id}");
//create instance for scenario
RunInstance runInstance = _runInstanceCreator.Create(run.Id, scenario.Id);
RaiseInfo($"AutoDistributed - about to enter: {nameof(runInstance.UploadInputFilesAndCreateAutoBookRequest)}");
runInstance.UploadInputFilesAndCreateAutoBookRequest(autoBookInstanceConfigurationsForRun, autoBookRequiredStorageGB);
RaiseInfo($"AutoDistributed - returned from: {nameof(runInstance.UploadInputFilesAndCreateAutoBookRequest)}");
// Flag run as started
runStarted = true;
runInstances.Add(runInstance);
_ = newScenarioStatuses.TryRemove(scenario.Id, out _); // Don't update scenario status at the end
scenarioSyncStatuses[scenario.Id] = false;
RaiseInfo($"AutoDistributed - RunScenarioTask Execute Started ScenarioID ={ scenario.Id}, RunID ={ run.Id}");
}
else
{
try
{
using (MachineLock.Create("xggameplan.AWSAutoBooks.GetFreeAutoBook",
new TimeSpan(0, 10, 0)))
{
foreach (var autoBookInstanceConfiguration in autoBookInstanceConfigurationsForRun)
{
autoBook = _autoBooks.GetFirstAdequateIdleAutoBook(autoBookInstanceConfiguration, autoBookRequiredStorageGB, true);
if (autoBook != null) // Got free AutoBook
{
RaiseInfo($"Got Free AutoBook: {autoBook.Id} ConfigurationId: {autoBook.InstanceConfigurationId}");
runAutoBookInstanceConfiguration = autoBookInstanceConfiguration;
break;
}
}
}
}
catch (MachineLockTimeoutException)
{
RaiseInfo($"MachineLockTimeoutException in xggameplan.AWSAutoBooks.GetFreeAutoBook");
}
// Get autobook interface
autoBookInterface = (autoBook == null) ? null : _autoBooks.GetInterface(autoBook);
// Get free AutoBook instance, will be locked so that it can't be used elsewhere
if (autoBook != null) // Free AutoBook - start run
{
RaiseInfo($"Free Autobook - Starting ScenarioID ={ scenario.Id}, AutoBookID ={autoBook?.Id}, RunID ={ run.Id}, Instance Configuration = { runAutoBookInstanceConfiguration.Description }");
// Start run, exception will cause cleanup below
RunInstance runInstance = _runInstanceCreator.Create(run.Id, scenario.Id);
runInstance.UploadInputFilesStartAutoBookRun(autoBookInterface, autoBook);
// Flag run as started
runStarted = true;
runInstances.Add(runInstance);
_ = newScenarioStatuses.TryRemove(scenario.Id, out _); // Don't update scenario status at the end
scenarioSyncStatuses[scenario.Id] = false;
RaiseInfo($"Started ScenarioID ={ scenario.Id}, AutoBookID ={ autoBook?.Id}, RunID ={ run.Id}, Instance Configuration = { runAutoBookInstanceConfiguration?.Description }");
}
else // No free AutoBook, awaiting for provisioning
{
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForWarningMessage(0, 0,
$"No free AutoBook, awaiting for provisioning, waiting for existing AutoBooks to be Idle (RunID={run.Id}, ScenarioID={scenario.Id})"));
// Update scenario so that it can be retried later when an AutoBook becomes idle
RunManager.UpdateScenarioStatuses(_repositoryFactory, _auditEventRepository, run.Id, new List <Guid> {
scenario.Id
},
new List <ScenarioStatuses> {
ScenarioStatuses.Scheduled
}, new List <DateTime?> {
null
});
_ = newScenarioStatuses.TryRemove(scenario.Id, out _); // Don't update scenario status at the end
scenarioSyncStatuses[scenario.Id] = false;
}
}
}
catch (System.Exception exception)
{
// Log exception but don't throw it. We want to try and start other scenarios
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForException(0, 0, $"Error starting scenario (RunID={run.Id}, ScenarioID={scenario.Id}, AutoBookID={(autoBook == null ? "Unknown" : autoBook.Id)})", exception));
}
finally
{
// If we locked a free AutoBook instance but didn't start the scenario then reset to free, unlocks it.
if (!runStarted && autoBook != null)
{
autoBookInterface.ResetFree();
}
}
}
public void CreateAndDelete(Run run, AutoBookInstanceConfiguration autoBookInstanceConfiguration)
{
IEnumerable <AutoBook> allWorkingAutoBooks = _autoBooks.WorkingAutoBooks;
var allAutoBookInstanceConfigurations = _autoBookInstanceConfigurationRepository.GetAll();
bool singleInstance = allAutoBookInstanceConfigurations.Count == 1;
RaiseInfo($"Enter ManageAutoBooksCreateAndDelete for runid: {run.Id}, AutoBooks.Instances: {_autoBooks.Instances.ToString()}");
RaiseInfo($"Scenario count: {run.Scenarios.Count.ToString()}, max autobooks allowed: {_autoBooks.Settings.MaxInstances.ToString()}");
RaiseInfo(singleInstance == true ?
$"Single instance type available: {autoBookInstanceConfiguration.Id.ToString()}" :
$"Varying instance types available, autobook type required: {autoBookInstanceConfiguration.Id.ToString()}");
if (!singleInstance && allWorkingAutoBooks.Any())
{
var otherRunsAreActive = new Lazy <bool>(() => OtherActiveRunsWithWorkingScenarios(_runRepository, run.Id));
var autobooksOfCorrectType = allWorkingAutoBooks.Where(w => w.InstanceConfigurationId >= autoBookInstanceConfiguration.Id).ToList();
var autobooksOfTooSmallType = allWorkingAutoBooks.Where(w => w.InstanceConfigurationId < autoBookInstanceConfiguration.Id).ToList();
RaiseInfo($"Varying Instance Types & Some working autobooks - Scenarios count: {run.Scenarios.Count} AutoBooks correct type: {autobooksOfCorrectType.Count}, AutoBooks too small type: {autobooksOfTooSmallType.Count}");
if (run.Scenarios.Count > autobooksOfCorrectType.Count)
{
foreach (RunScenario currentScenario in run.Scenarios)
{
if (run.Scenarios.Count > autobooksOfCorrectType.Count)
{
RaiseInfo($"Need autobooks: {run.Scenarios.Count}, Existing autobooks of required type: {autobooksOfCorrectType.Count}");
if (((autobooksOfCorrectType.Count + autobooksOfTooSmallType.Count) < _autoBooks.Settings.MaxInstances) || _autoBooks.Settings.MaxInstances == 0)
{
//create autobook
var newAutobook = CreateAutoBookOfType(autoBookInstanceConfiguration.Id);
autobooksOfCorrectType.Add(newAutobook);
RaiseInfo($"Within the max allowed so created one of suitable size: {newAutobook.Id.ToString()} autobooktype.Id: {autoBookInstanceConfiguration.Id.ToString()}");
}
else if (autobooksOfTooSmallType.Any())
{
if (!otherRunsAreActive.Value)
{
RaiseInfo($"Max reached - checking for autobooks to delete, number of autobooks of too small type: {autobooksOfTooSmallType.Count.ToString()}, ScenarioID = {currentScenario.Id.ToString()}");
var autoBook = GetAutoBookAbleToDelete(autobooksOfTooSmallType);
if (autoBook != null)
{
//delete autobook
var delid = autoBook.Id.ToString();
DeleteAutoBook(autoBook);
_ = autobooksOfTooSmallType.Remove(autoBook);
//create autobook
var newAutobook = CreateAutoBookOfType(autoBookInstanceConfiguration.Id);
autobooksOfCorrectType.Add(newAutobook);
RaiseInfo($"Max reached - deleted smaller autobook: {delid}, created larger autobook: {newAutobook.Id.ToString()}, autobooktype.Id: {autoBookInstanceConfiguration.Id.ToString()}, ScenarioID = {currentScenario.Id.ToString()}");
}
else
{
RaiseInfo("Max reached but couldn't delete an autobook (none with suitable status of Idle or Fatal Error)");
}
}
else
{
RaiseInfo("Other Active Runs, can't delete an autobook");
}
}
}
}
}
}
else
{
RaiseInfo($"Single Instance Type or No working autobooks - scenario count: {run.Scenarios.Count.ToString()}, max allowed: {_autoBooks.Settings.MaxInstances.ToString()}, so creating autobooks of type: {autoBookInstanceConfiguration.Id.ToString()}");
CreateAnAutoBookForEachScenario(run, autoBookInstanceConfiguration.Id, allWorkingAutoBooks.Count());
}
RaiseInfo($"Exit ManageAutoBooksCreateAndDelete for runid: {run.Id.ToString()}, AutoBooks.Instances: {_autoBooks.Instances.ToString()}");
}
