public void Generate(AutoBookDomainObject autoBook, Guid scenarioId)
{
//string snapshotFile = string.Format(@"{0}\{1}.snapshot.zip", System.Web.Hosting.HostingEnvironment.MapPath("/Output"), _scenarioId);
string snapshotFile = string.Format(@"{0}\Output\{1}.snapshot.zip", _rootFolder, scenarioId);
System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(snapshotFile));
if (System.IO.File.Exists(snapshotFile))
{
System.IO.File.Delete(snapshotFile);
}
IAutoBook autoBookInterface = _autoBooks.GetInterface(autoBook);
GetAutoBookSnapshotModel autoBookSnapshot = autoBookInterface.GetSnapshot(scenarioId);
System.IO.File.WriteAllBytes(snapshotFile, autoBookSnapshot.Data);
}
public void Execute(
Run run,
RunScenario scenario,
IReadOnlyCollection <AutoBookInstanceConfiguration> autoBookInstanceConfigurationsForRun,
double autoBookRequiredStorageGB,
ConcurrentBag <RunInstance> runInstances,
ConcurrentDictionary <Guid, ScenarioStatuses> newScenarioStatuses,
ConcurrentDictionary <Guid, bool> scenarioSyncStatuses,
bool autoDistributed)
{
AutoBookDomainObject autoBook = null;
IAutoBook autoBookInterface = null;
bool runStarted = false;
RaiseInfo($"Begin Execute for ScenarioID: { scenario.Id}");
try
{
AutoBookInstanceConfiguration runAutoBookInstanceConfiguration = null;
if (autoDistributed)
{
RaiseInfo($"AutoDistributed - RunScenarioTask Execute Starting ScenarioID ={ scenario.Id}, RunID ={ run.Id}");
//create instance for scenario
RunInstance runInstance = _runInstanceCreator.Create(run.Id, scenario.Id);
RaiseInfo($"AutoDistributed - about to enter: {nameof(runInstance.UploadInputFilesAndCreateAutoBookRequest)}");
runInstance.UploadInputFilesAndCreateAutoBookRequest(autoBookInstanceConfigurationsForRun, autoBookRequiredStorageGB);
RaiseInfo($"AutoDistributed - returned from: {nameof(runInstance.UploadInputFilesAndCreateAutoBookRequest)}");
// Flag run as started
runStarted = true;
runInstances.Add(runInstance);
_ = newScenarioStatuses.TryRemove(scenario.Id, out _); // Don't update scenario status at the end
scenarioSyncStatuses[scenario.Id] = false;
RaiseInfo($"AutoDistributed - RunScenarioTask Execute Started ScenarioID ={ scenario.Id}, RunID ={ run.Id}");
}
else
{
try
{
using (MachineLock.Create("xggameplan.AWSAutoBooks.GetFreeAutoBook",
new TimeSpan(0, 10, 0)))
{
foreach (var autoBookInstanceConfiguration in autoBookInstanceConfigurationsForRun)
{
autoBook = _autoBooks.GetFirstAdequateIdleAutoBook(autoBookInstanceConfiguration, autoBookRequiredStorageGB, true);
if (autoBook != null) // Got free AutoBook
{
RaiseInfo($"Got Free AutoBook: {autoBook.Id} ConfigurationId: {autoBook.InstanceConfigurationId}");
runAutoBookInstanceConfiguration = autoBookInstanceConfiguration;
break;
}
}
}
}
catch (MachineLockTimeoutException)
{
RaiseInfo($"MachineLockTimeoutException in xggameplan.AWSAutoBooks.GetFreeAutoBook");
}
// Get autobook interface
autoBookInterface = (autoBook == null) ? null : _autoBooks.GetInterface(autoBook);
// Get free AutoBook instance, will be locked so that it can't be used elsewhere
if (autoBook != null) // Free AutoBook - start run
{
RaiseInfo($"Free Autobook - Starting ScenarioID ={ scenario.Id}, AutoBookID ={autoBook?.Id}, RunID ={ run.Id}, Instance Configuration = { runAutoBookInstanceConfiguration.Description }");
// Start run, exception will cause cleanup below
RunInstance runInstance = _runInstanceCreator.Create(run.Id, scenario.Id);
runInstance.UploadInputFilesStartAutoBookRun(autoBookInterface, autoBook);
// Flag run as started
runStarted = true;
runInstances.Add(runInstance);
_ = newScenarioStatuses.TryRemove(scenario.Id, out _); // Don't update scenario status at the end
scenarioSyncStatuses[scenario.Id] = false;
RaiseInfo($"Started ScenarioID ={ scenario.Id}, AutoBookID ={ autoBook?.Id}, RunID ={ run.Id}, Instance Configuration = { runAutoBookInstanceConfiguration?.Description }");
}
else // No free AutoBook, awaiting for provisioning
{
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForWarningMessage(0, 0,
$"No free AutoBook, awaiting for provisioning, waiting for existing AutoBooks to be Idle (RunID={run.Id}, ScenarioID={scenario.Id})"));
// Update scenario so that it can be retried later when an AutoBook becomes idle
RunManager.UpdateScenarioStatuses(_repositoryFactory, _auditEventRepository, run.Id, new List <Guid> {
scenario.Id
},
new List <ScenarioStatuses> {
ScenarioStatuses.Scheduled
}, new List <DateTime?> {
null
});
_ = newScenarioStatuses.TryRemove(scenario.Id, out _); // Don't update scenario status at the end
scenarioSyncStatuses[scenario.Id] = false;
}
}
}
catch (System.Exception exception)
{
// Log exception but don't throw it. We want to try and start other scenarios
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForException(0, 0, $"Error starting scenario (RunID={run.Id}, ScenarioID={scenario.Id}, AutoBookID={(autoBook == null ? "Unknown" : autoBook.Id)})", exception));
}
finally
{
// If we locked a free AutoBook instance but didn't start the scenario then reset to free, unlocks it.
if (!runStarted && autoBook != null)
{
autoBookInterface.ResetFree();
}
}
}
public List <SystemTestResult> Execute(SystemTestCategories systemTestCategory)
{
var results = new List <SystemTestResult>();
try
{
using (var scope = _repositoryFactory.BeginRepositoryScope())
{
// Get list of AutoBooks
var autoBookRepository = scope.CreateRepository <IAutoBookRepository>();
var autoBooks = autoBookRepository.GetAll();
// Check AutoBook provisioning settings
results.AddRange(ExecuteProvisioningTests(autoBooks.ToList()));
// Warn if no AutoBooks exist
if (autoBookRepository.GetAll().ToList().Count == 0)
{
if (!_autoBooks.Settings.AutoProvisioning)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "No AutoBook data exists and auto-provisioning is disabled. It will not be possible to execute any runs.", ""));
}
}
else
{
// Check connectivity to each AutoBook
int countWorkingAutoBooks = 0;
int countFreeAutoBooks = 0;
foreach (var autoBook in autoBooks)
{
try
{
IAutoBook autoBookInterface = _autoBooks.GetInterface(autoBook);
var autoBookStatus = autoBookInterface.GetStatus();
if (AutoBook.IsWorkingStatus(autoBookStatus))
{
countWorkingAutoBooks++;
if (autoBookStatus == AutoBookStatuses.Idle)
{
countFreeAutoBooks++;
}
}
else if (autoBookStatus == AutoBookStatuses.Fatal_Error)
{
if (!_autoBooks.Settings.AutoProvisioning)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Warning, _category, string.Format("AutoBook {0} has indicated that is has encountered a fatal error. It cannot be used for any runs while it is in this state. Please manually restart it.", autoBook.Id), ""));
}
}
}
catch
{
if (autoBook.Status != AutoBookStatuses.Provisioning) // Ignore error if Provisioning
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Warning, _category, string.Format("Error testing connection to AutoBook {0} ({1})", autoBook.Id, autoBook.Api), ""));
}
}
}
if (countWorkingAutoBooks == 0)
{
if (!_autoBooks.Settings.AutoProvisioning)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "There are no working AutoBooks. It will not be possible to start any runs.", ""));
}
}
else if (countFreeAutoBooks == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Warning, _category, string.Format("All {0} working AutoBooks are currently executing a run. It will not be possible to start any runs until existing runs have completed.", countWorkingAutoBooks), ""));
}
else if (results.Count == 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Information, _category, string.Format("AutoBook test OK ({0} working AutoBooks)", countWorkingAutoBooks), ""));
}
}
}
}
catch (System.Exception exception)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, string.Format("Error checking AutoBooks: {0}", exception.Message), ""));
}
return(results);
}
/// <summary>
/// Executes provisioning tests
/// </summary>
/// <returns></returns>
private List <SystemTestResult> ExecuteProvisioningTests(List <AutoBook> autoBooks)
{
List <SystemTestResult> results = new List <SystemTestResult>();
using (var scope = _repositoryFactory.BeginRepositoryScope())
{
var autoBookInstanceConfigurationRepository = scope.CreateRepository <IAutoBookInstanceConfigurationRepository>();
if (!_autoBooks.Settings.AutoProvisioning) // Auto-provisioning enabled
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Warning, _category, "Automatic AutoBook provisioning is disabled. AutoBooks must be managed manually. " +
"Crashed AutoBooks will not be automatically restarted.", ""));
}
if (String.IsNullOrEmpty(_autoBooks.Settings.ApplicationVersion))
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "The AutoBook version is not set in the configuration. It is not possible to create AutoBook instances.", ""));
}
if (String.IsNullOrEmpty(_autoBooks.Settings.ProvisioningAPIURL))
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "The Provisioning API URL is not set in the configuration. It is not possible to create AutoBook instances.", ""));
}
if (_autoBooks.Settings.AutoProvisioning)
{
var autoBookInstanceConfigurations = autoBookInstanceConfigurationRepository.GetAll();
if (!autoBookInstanceConfigurations.Any())
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "There are no AutoBook instance configurations. It is not possible to create AutoBook instances.", ""));
}
if (_autoBooks.Settings.MinInstances > 0)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Warning, _category, string.Format("The minimum number of AutoBook instances is set to {0} but, for cost reasons, it is recommended to be zero so that " +
"the instances are deleted when they are no longer needed.", _autoBooks.Settings.MinInstances), ""));
}
}
// Warn if AutoBooks are running the wrong versions
if (!_autoBooks.Settings.Locked && !String.IsNullOrEmpty(_autoBooks.Settings.ApplicationVersion)) // Do nothing if provisioning in progress
{
foreach (var autoBook in autoBooks)
{
IAutoBook autoBookInterface = _autoBooks.GetInterface(autoBook);
try
{
GetAutoBookVersionModel getAutoBookVersionModel = autoBookInterface.GetVersion();
if (getAutoBookVersionModel.Version.ToUpper() != _autoBooks.Settings.ApplicationVersion.ToUpper())
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Warning, _category, string.Format("AutoBook {0} is running version {1} but the settings indicate that it should be running {2}. Please check that auto-provisioning is working.", autoBook.Id, getAutoBookVersionModel.Version, _autoBooks.Settings.ApplicationVersion), ""));
}
}
catch { }; // Ignore error
}
}
if (!_autoBooks.Settings.Locked) // Don't check if provisioning in progress
{
// Warn if insufficient AutoBooks
if (_autoBooks.Settings.MinInstances > 0 && autoBooks.Count < _autoBooks.Settings.MinInstances)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Warning, _category, string.Format("The settings indicate a minimum of {0} AutoBooks but currently there are only {1} AutoBooks.", _autoBooks.Settings.MinInstances, autoBooks.Count), ""));
}
// Warn if too many AutoBooks
if (_autoBooks.Settings.MaxInstances > 0 && autoBooks.Count > _autoBooks.Settings.MaxInstances)
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Warning, _category, string.Format("The settings indicate a maximum of {0} AutoBooks but currently there are {1} AutoBooks.", _autoBooks.Settings.MinInstances, autoBooks.Count), ""));
}
}
// Test AutoBooks API. If we can't contact it then we can't provision AutoBooks
try
{
bool testProvisioningResult = _autoBooks.TestProvisioning();
}
catch
{
results.Add(new SystemTestResult(SystemTestResult.ResultTypes.Error, _category, "There was an error testing the AutoBooks API. It will prevent AutoBooks from being provisioned, unprovisioned or restarted.", ""));
}
}
return(results);
}
/// <summary>
/// Starts run. Uploads input data and then instructs AutoBook instance to start
/// </summary>
/// <param name="autoBookInterface"></param>
/// <param name="autoBook"></param>
public void UploadInputFilesStartAutoBookRun(IAutoBook autoBookInterface, AutoBook autoBook)
{
using var scope = _repositoryFactory.BeginRepositoryScope();
var runRepository = scope.CreateRepository <IRunRepository>();
var autoBookRepository = scope.CreateRepository <IAutoBookRepository>();
bool startedRun = false;
try
{
// Update scenario status to Starting, Scenario.StartedDateTime has already been set
RunManager.UpdateScenarioStatuses(_repositoryFactory, _auditEventRepository, RunId,
new List <Guid>()
{
ScenarioId
}, new List <ScenarioStatuses>()
{
ScenarioStatuses.Starting
});
// Get run
var run = runRepository.Find(RunId);
// Record which run/scenario is being processed
//IAutoBookRepository autoBookRepository = (IAutoBookRepository)repositories[typeof(IAutoBookRepository)];
//AutoBook localAutoBook = autoBookRepository.Find(autoBook.Id);
lock (autoBook)
{
autoBook.Status = AutoBookStatuses.In_Progress;
autoBook.LastRunStarted = DateTime.UtcNow;
autoBook.Locked = true;
autoBook.Task = new AutoBookTask()
{
RunId = RunId,
ScenarioId = ScenarioId
};
}
autoBookRepository.Update(autoBook);
autoBookRepository.SaveChanges(); // Force save
// Upload input data
_autoBookInputHandler.Handle(run, ScenarioId);
// Instruct AutoBook to start processing
bool loggedNotifyFinished = false;
try
{
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineStart(0, 0,
PipelineEventIDs.STARTED_NOTIFYING_AUTOBOOK_API, RunId, ScenarioId, autoBook.Id, null));
GetAutoBookStatusModel autoBookStatusModel = autoBookInterface.StartAutoBookRun(RunId, ScenarioId);
_pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun,
PipelineEventIDs.STARTED_NOTIFYING_AUTOBOOK_API, RunId, ScenarioId, null));
if (autoBookStatusModel.Status == AutoBookStatuses.In_Progress) // Task_Error or Fatal_Error
{
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineEnd(0, 0,
PipelineEventIDs.FINISHED_NOTIFYING_AUTOBOOK_API, RunId, ScenarioId, autoBook.Id,
null, null, null));
_pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun,
PipelineEventIDs.FINISHED_NOTIFYING_AUTOBOOK_API, RunId, ScenarioId, null));
loggedNotifyFinished = true;
}
else
{
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineEnd(0, 0,
PipelineEventIDs.FINISHED_NOTIFYING_AUTOBOOK_API, RunId, ScenarioId, autoBook.Id,
null,
String.Format(
"AutoBook API unexpectedly returned status {0} when instructing it to start run",
autoBookStatusModel.ToString()), null));
_pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun,
PipelineEventIDs.FINISHED_NOTIFYING_AUTOBOOK_API, RunId, ScenarioId,
$"AutoBook API unexpectedly returned status {autoBookStatusModel.ToString()} " +
"when instructing it to start run"));
loggedNotifyFinished = true;
throw new Exception(String.Format(
"AutoBook returned status {0} when starting run (AutoBookID={1})",
autoBookStatusModel.Status, autoBook.Id));
}
}
catch (System.Exception exception)
{
if (!loggedNotifyFinished)
{
_auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineEnd(0, 0,
PipelineEventIDs.FINISHED_NOTIFYING_AUTOBOOK_API, RunId, ScenarioId, autoBook.Id,
null, exception.Message, exception));
_pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun,
PipelineEventIDs.FINISHED_NOTIFYING_AUTOBOOK_API, RunId, ScenarioId, exception.Message));
}
throw;
}
finally
{
_pipelineAuditEventRepository.SaveChanges();
}
// Flag as InProgress
RunManager.UpdateScenarioStatuses(_repositoryFactory, _auditEventRepository, RunId,
new List <Guid>()
{
ScenarioId
}, new List <ScenarioStatuses>()
{
ScenarioStatuses.InProgress
});
startedRun = true;
}
catch
{
throw;
}
finally
{
// Clean up failure
if (!startedRun)
{
// Reset to free, unlocks AutoBook instance so that it can be re-used
autoBookInterface.ResetFree();
autoBookRepository.SaveChanges();
_pipelineAuditEventRepository.SaveChanges();
}
}
}
