private void UploadRunData(Run run, Guid scenarioId) { string runFilePath = null; string scenarioFilePath = null; bool uploadRunData = false; bool uploadScenarioData = false; string runFile = $"{run.Id}.zip"; string runFileNameWithPath = $@"input/{runFile}"; string scenarioFile = $"{scenarioId}.zip"; string scenarioFileNameWithPath = $@"input/{scenarioFile}"; bool loggedStarted = false; try { // Ensure that only one instance attempts to upload <RunId>.zip. We can't just check FileExists because it will take time for the file to // be uploaded and appear. using (MachineLock.Create($"xggameplan.AWSInputHandler.UploadRunData.Run Id: {run.Id}", new TimeSpan(2, 0, 0))) { if (!_cloudStorage.FileExists(new S3FileComment() { BucketName = _awsSettings.S3Bucket, FileNameWithPath = runFileNameWithPath })) { uploadRunData = true; if (!loggedStarted) { _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineStart( 0, 0, PipelineEventIDs.STARTED_GENERATING_INPUT_FILES, run.Id, scenarioId, null, null)); _pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun, PipelineEventIDs.STARTED_GENERATING_INPUT_FILES, run.Id, scenarioId, null)); loggedStarted = true; } runFilePath = _optimiserInputFiles.PopulateRunData(run); } } // Upload scenario data if (!_cloudStorage.FileExists(new S3FileComment() { BucketName = _awsSettings.S3Bucket, FileNameWithPath = scenarioFileNameWithPath })) { uploadScenarioData = true; if (!loggedStarted) { _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineStart(0, 0, PipelineEventIDs.STARTED_GENERATING_INPUT_FILES, run.Id, scenarioId, null, null)); _pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun, PipelineEventIDs.STARTED_GENERATING_INPUT_FILES, run.Id, scenarioId, null)); loggedStarted = true; } scenarioFilePath = _optimiserInputFiles.PopulateScenarioData(run, scenarioId); } if (loggedStarted) { _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineEnd(0, 0, PipelineEventIDs.FINISHED_GENERATING_INPUT_FILES, run.Id, scenarioId, null, null, null, null)); _pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun, PipelineEventIDs.FINISHED_GENERATING_INPUT_FILES, run.Id, scenarioId, null)); } } catch (System.Exception exception) { if (loggedStarted) { _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineEnd(0, 0, PipelineEventIDs.FINISHED_GENERATING_INPUT_FILES, run.Id, scenarioId, null, null, exception.Message, exception)); _pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun, PipelineEventIDs.FINISHED_GENERATING_INPUT_FILES, run.Id, scenarioId, exception.Message)); } throw; } finally { _pipelineAuditEventRepository.SaveChanges(); } // Because we do zipping as we create each input file... _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineStart(0, 0, PipelineEventIDs.STARTED_ZIPPING_INPUT_FILES, run.Id, scenarioId, null, null)); _pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun, PipelineEventIDs.STARTED_ZIPPING_INPUT_FILES, run.Id, scenarioId, null)); _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineEnd(0, 0, PipelineEventIDs.FINISHED_ZIPPING_INPUT_FILES, run.Id, scenarioId, null, null, null, null)); _pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun, PipelineEventIDs.FINISHED_ZIPPING_INPUT_FILES, run.Id, scenarioId, null)); _pipelineAuditEventRepository.SaveChanges(); try { using (MachineLock.Create( $"xggameplan.AWSInputHandler.UploadRunData.Run Id: {run.Id}", new TimeSpan(2, 0, 0))) { _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineStart(0, 0, PipelineEventIDs.STARTED_UPLOADING_INPUT_ZIP_ARCHIVE_TO_CLOUD_STORAGE, run.Id, scenarioId, null, null)); _pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun, PipelineEventIDs.STARTED_UPLOADING_INPUT_ZIP_ARCHIVE_TO_CLOUD_STORAGE, run.Id, scenarioId, null)); //Upload run data if (uploadRunData && !String.IsNullOrWhiteSpace(runFilePath) && File.Exists(runFilePath) && !_cloudStorage.FileExists(new S3FileComment() { BucketName = _awsSettings.S3Bucket, FileNameWithPath = runFileNameWithPath })) { _cloudStorage.Upload(new S3UploadComment() { BucketName = _awsSettings.S3Bucket, DestinationFilePath = runFileNameWithPath, SourceFilePath = runFilePath }); uploadRunData = false; if (File.Exists(runFilePath)) { File.Delete(runFilePath); } } // Upload scenario data if (uploadScenarioData && !String.IsNullOrWhiteSpace(scenarioFilePath) && File.Exists(scenarioFilePath)) { _cloudStorage.Upload(new S3UploadComment() { BucketName = _awsSettings.S3Bucket, DestinationFilePath = scenarioFileNameWithPath, SourceFilePath = scenarioFilePath }); if (File.Exists(scenarioFilePath)) { File.Delete(scenarioFilePath); } } _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineEnd(0, 0, PipelineEventIDs.FINISHED_UPLOADING_INPUT_ZIP_ARCHIVE_TO_CLOUD_STORAGE, run.Id, scenarioId, null, null, null, null)); _pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun, PipelineEventIDs.FINISHED_UPLOADING_INPUT_ZIP_ARCHIVE_TO_CLOUD_STORAGE, run.Id, scenarioId, null)); } } catch (System.Exception exception) { _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineEnd(0, 0, PipelineEventIDs.FINISHED_UPLOADING_INPUT_ZIP_ARCHIVE_TO_CLOUD_STORAGE, run.Id, scenarioId, null, null, exception.Message, exception)); _pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun, PipelineEventIDs.FINISHED_UPLOADING_INPUT_ZIP_ARCHIVE_TO_CLOUD_STORAGE, run.Id, scenarioId, exception.Message)); throw; } finally { _pipelineAuditEventRepository.SaveChanges(); } }
/// <summary> /// Handles run completed, all scenarios /// </summary> /// <param name="run"></param> /// <param name="success"></param> public void Notify(Run run, bool success) { TenantSettings tenantSettings = _tenantSettingsRepository.Get(); bool notificationsConfigured = false; _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForRunCompleted(0, 0, run.Id, success ? "Completed successfully" : "Completed with errors")); // Get run event settings if (tenantSettings.RunEventSettings != null) { RunEventSettings runEventSettings = tenantSettings.RunEventSettings.Where(item => item.EventType == RunEvents.RunCompleted).FirstOrDefault(); if (runEventSettings != null) { var exceptions = new List <Exception>(); try { // Check HTTP notification INotification <HTTPNotificationSettings> httpNotification = _notifications?.GetNotification <HTTPNotificationSettings>(); if (runEventSettings.HTTP != null && httpNotification != null && runEventSettings.HTTP.Enabled) { notificationsConfigured = true; try { _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineStart(0, 0, PipelineEventIDs.STARTED_NOTIFYING_MULE_SOFT_API, run.Id, null, null, string.Format("Generating HTTP notification for run completed (RunID={0}, Success={1})", run.Id, success))); _pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun, PipelineEventIDs.STARTED_NOTIFYING_MULE_SOFT_API, run.Id, Guid.Empty, null)); httpNotification.RunCompleted(run, success, runEventSettings.HTTP); _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineEnd(0, 0, PipelineEventIDs.FINISHED_NOTIFYING_MULE_SOFT_API, run.Id, null, null, string.Format("Generated HTTP notification for run completed (RunID={0}, Success={1})", run.Id, success), null, null)); _pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun, PipelineEventIDs.FINISHED_NOTIFYING_MULE_SOFT_API, run.Id, Guid.Empty, null)); } catch (System.Exception exception) { _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineEnd(0, 0, PipelineEventIDs.FINISHED_NOTIFYING_MULE_SOFT_API, run.Id, null, null, string.Format("Failed to generate HTTP notification for run completed (RunID={0}, Success={1})", run.Id, success), exception.Message, exception)); _pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun, PipelineEventIDs.FINISHED_NOTIFYING_MULE_SOFT_API, run.Id, Guid.Empty, exception.Message)); throw; } finally { _pipelineAuditEventRepository.SaveChanges(); } } } catch (Exception exception) { exceptions.Add(exception); } if (exceptions.Count == 1) { throw exceptions[0]; } else if (exceptions.Count > 0) { throw new AggregateException(exceptions); } } } // Log warning if notifications not enabled so that there's no dispute over why they weren't generated if (!notificationsConfigured) { _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForWarningMessage(0, 0, string.Format("Not generating notification for run completed because notifications are not configured or enabled (RunID={0}, Success={1})", run.Id, success))); } }
/// <summary> /// Processes files. For each file we pass it to a file processor which /// performs some action on it. /// </summary> /// <param name="runId">RunID</param> /// <param name="scenarioId">ScenarioId</param> /// <param name="scenarioResult"></param> private void ProcessOutputFiles(Guid runId, Guid scenarioId, ScenarioResult scenarioResult) { _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineStart(0, 0, PipelineEventIDs.STARTED_EXPORTING_DATA_TO_DATABASE, runId, scenarioId, null, null)); _pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun, PipelineEventIDs.STARTED_EXPORTING_DATA_TO_DATABASE, runId, scenarioId, null)); string scenarioLocalFolder = GetScenarioLocalFolder(scenarioId); var dataProcessingExceptions = new List <Exception>(); using (var kpiCalculationScope = _kpiCalculationScopeFactory.CreateCalculationScope(runId, scenarioId)) { var kpiCalculationContext = kpiCalculationScope.Resolve <IKPICalculationContext>(); var kpiCalculationManager = kpiCalculationScope.Resolve <IKPICalculationManager>(); foreach (var dataProcessingAction in _dataProcessingActions) { try { dataProcessingAction(kpiCalculationScope, _featureManager, _auditEventRepository, scenarioLocalFolder); } catch (Exception e) { _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForException(0, 0, $"Error processing: {dataProcessingAction.Method.GetGenericArguments().First().Name}. ScenarioId: {scenarioId}", e)); dataProcessingExceptions.Add(e); } } if (_includeScenarioPerformanceMeasurementKPIs) { kpiCalculationContext.SetDefaultKpiDemos(); } _scenarioCampaignMetricsProcessor.ProcessScenarioCampaignMetrics(runId, scenarioId, kpiCalculationContext.Recommendations); scenarioResult.Metrics = kpiCalculationManager.CalculateKPIs(runId, scenarioId); scenarioResult.AnalysisGroupMetrics = kpiCalculationManager.CalculateAnalysisGroupKPIs(runId); _scenarioResultRepository.Add(scenarioResult); _scenarioResultRepository.SaveChanges(); } // Exceptions handling if (dataProcessingExceptions.Count == 1) { _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineEnd(0, 0, PipelineEventIDs.FINISHED_EXPORTING_DATA_TO_DATABASE, runId, scenarioId, null, null, dataProcessingExceptions[0].Message, dataProcessingExceptions[0])); throw dataProcessingExceptions[0]; } else if (dataProcessingExceptions.Count > 1) { _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineEnd(0, 0, PipelineEventIDs.FINISHED_EXPORTING_DATA_TO_DATABASE, runId, scenarioId, null, null, dataProcessingExceptions[0].Message, dataProcessingExceptions[0])); throw new AggregateException(dataProcessingExceptions); } _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineEnd(0, 0, PipelineEventIDs.FINISHED_EXPORTING_DATA_TO_DATABASE, runId, scenarioId, null, "Import has been finished", null, null)); _pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun, PipelineEventIDs.FINISHED_EXPORTING_DATA_TO_DATABASE, runId, scenarioId, null)); _pipelineAuditEventRepository.SaveChanges(); }
/// <summary> /// Downloads output files /// </summary> /// <param name="runId"></param> /// <param name="scenarioId"></param> /// <param name="remoteFolder"></param> /// <param name="localFolder"></param> private void DownloadOutputFiles(Guid runId, Guid scenarioId, string localFolder) { // For some reason the output goes to \Output\[ScenarioId]\output string zipFileName = $"{scenarioId}.zip"; string localZipFile = Path.Combine(localFolder + @"\output", zipFileName); try { _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineStart(0, 0, PipelineEventIDs.STARTED_DOWNLOADING_ZIP_ARCHIVE_FROM_CLOUD_STORAGE, runId, scenarioId, null, null)); _pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun, PipelineEventIDs.STARTED_DOWNLOADING_ZIP_ARCHIVE_FROM_CLOUD_STORAGE, runId, scenarioId, null)); // For some reason the output goes to \Output\[ScenarioId]\output if (!Directory.Exists(Path.GetDirectoryName(localZipFile))) { _ = Directory.CreateDirectory(Path.GetDirectoryName(localZipFile)); } if (File.Exists(localZipFile)) { File.Delete(localZipFile); } var filename = $@"output/{zipFileName}"; _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, $"Downloading scenario output from cloud provider ( Bucket={_awsSettings.S3Bucket},filename={filename} Local Folder={localFolder})")); _ = _cloudStorage.Download(new S3DownloadComment { BucketName = _awsSettings.S3Bucket, FileName = filename, LocalFileFolder = localFolder }); _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineEnd(0, 0, PipelineEventIDs.FINISHED_DOWNLOADING_ZIP_ARCHIVE_FROM_CLOUD_STORAGE, runId, scenarioId, null, null, null, null)); _pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun, PipelineEventIDs.FINISHED_DOWNLOADING_ZIP_ARCHIVE_FROM_CLOUD_STORAGE, runId, scenarioId, null)); } catch (Exception exception) { _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineEnd(0, 0, PipelineEventIDs.FINISHED_DOWNLOADING_ZIP_ARCHIVE_FROM_CLOUD_STORAGE, runId, scenarioId, null, null, exception.Message, exception)); _pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun, PipelineEventIDs.FINISHED_DOWNLOADING_ZIP_ARCHIVE_FROM_CLOUD_STORAGE, runId, scenarioId, exception.Message)); throw; } finally { _pipelineAuditEventRepository.SaveChanges(); } try { // Clear files in destination folder before we unzip ClearFolder(localFolder); ZipConversion.DeserializeFromZip(localZipFile, localFolder); } catch (Exception exception) { _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForException(0, 0, String.Format("Error unzipping {0}", localZipFile), exception)); throw; } }
/// <summary> /// Starts run. Uploads input data and then instructs AutoBook instance to start /// </summary> /// <param name="autoBookInterface"></param> /// <param name="autoBook"></param> public void UploadInputFilesStartAutoBookRun(IAutoBook autoBookInterface, AutoBook autoBook) { using var scope = _repositoryFactory.BeginRepositoryScope(); var runRepository = scope.CreateRepository <IRunRepository>(); var autoBookRepository = scope.CreateRepository <IAutoBookRepository>(); bool startedRun = false; try { // Update scenario status to Starting, Scenario.StartedDateTime has already been set RunManager.UpdateScenarioStatuses(_repositoryFactory, _auditEventRepository, RunId, new List <Guid>() { ScenarioId }, new List <ScenarioStatuses>() { ScenarioStatuses.Starting }); // Get run var run = runRepository.Find(RunId); // Record which run/scenario is being processed //IAutoBookRepository autoBookRepository = (IAutoBookRepository)repositories[typeof(IAutoBookRepository)]; //AutoBook localAutoBook = autoBookRepository.Find(autoBook.Id); lock (autoBook) { autoBook.Status = AutoBookStatuses.In_Progress; autoBook.LastRunStarted = DateTime.UtcNow; autoBook.Locked = true; autoBook.Task = new AutoBookTask() { RunId = RunId, ScenarioId = ScenarioId }; } autoBookRepository.Update(autoBook); autoBookRepository.SaveChanges(); // Force save // Upload input data _autoBookInputHandler.Handle(run, ScenarioId); // Instruct AutoBook to start processing bool loggedNotifyFinished = false; try { _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineStart(0, 0, PipelineEventIDs.STARTED_NOTIFYING_AUTOBOOK_API, RunId, ScenarioId, autoBook.Id, null)); GetAutoBookStatusModel autoBookStatusModel = autoBookInterface.StartAutoBookRun(RunId, ScenarioId); _pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun, PipelineEventIDs.STARTED_NOTIFYING_AUTOBOOK_API, RunId, ScenarioId, null)); if (autoBookStatusModel.Status == AutoBookStatuses.In_Progress) // Task_Error or Fatal_Error { _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineEnd(0, 0, PipelineEventIDs.FINISHED_NOTIFYING_AUTOBOOK_API, RunId, ScenarioId, autoBook.Id, null, null, null)); _pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun, PipelineEventIDs.FINISHED_NOTIFYING_AUTOBOOK_API, RunId, ScenarioId, null)); loggedNotifyFinished = true; } else { _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineEnd(0, 0, PipelineEventIDs.FINISHED_NOTIFYING_AUTOBOOK_API, RunId, ScenarioId, autoBook.Id, null, String.Format( "AutoBook API unexpectedly returned status {0} when instructing it to start run", autoBookStatusModel.ToString()), null)); _pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun, PipelineEventIDs.FINISHED_NOTIFYING_AUTOBOOK_API, RunId, ScenarioId, $"AutoBook API unexpectedly returned status {autoBookStatusModel.ToString()} " + "when instructing it to start run")); loggedNotifyFinished = true; throw new Exception(String.Format( "AutoBook returned status {0} when starting run (AutoBookID={1})", autoBookStatusModel.Status, autoBook.Id)); } } catch (System.Exception exception) { if (!loggedNotifyFinished) { _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineEnd(0, 0, PipelineEventIDs.FINISHED_NOTIFYING_AUTOBOOK_API, RunId, ScenarioId, autoBook.Id, null, exception.Message, exception)); _pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun, PipelineEventIDs.FINISHED_NOTIFYING_AUTOBOOK_API, RunId, ScenarioId, exception.Message)); } throw; } finally { _pipelineAuditEventRepository.SaveChanges(); } // Flag as InProgress RunManager.UpdateScenarioStatuses(_repositoryFactory, _auditEventRepository, RunId, new List <Guid>() { ScenarioId }, new List <ScenarioStatuses>() { ScenarioStatuses.InProgress }); startedRun = true; } catch { throw; } finally { // Clean up failure if (!startedRun) { // Reset to free, unlocks AutoBook instance so that it can be re-used autoBookInterface.ResetFree(); autoBookRepository.SaveChanges(); _pipelineAuditEventRepository.SaveChanges(); } } }
/// <summary> /// Uploads input data and then send API post for AutoBookRequest /// </summary> /// <param name="autoBookInterface"></param> /// <param name="autoBook"></param> public void UploadInputFilesAndCreateAutoBookRequest(IReadOnlyCollection <AutoBookInstanceConfiguration> autoBookInstanceConfigurations, double storageGB) { _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, $"AutoDistributed - {nameof(UploadInputFilesAndCreateAutoBookRequest)}")); using var scope = _repositoryFactory.BeginRepositoryScope(); var runRepository = scope.CreateRepository <IRunRepository>(); bool startedRun = false; try { // Update scenario status to Starting, Scenario.StartedDateTime has already been set RunManager.UpdateScenarioStatuses(_repositoryFactory, _auditEventRepository, RunId, new List <Guid>() { ScenarioId }, new List <ScenarioStatuses>() { ScenarioStatuses.Starting }); var run = runRepository.Find(RunId); // Get run _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, $"AutoDistributed - about to enter input handler")); _autoBookInputHandler.Handle(run, ScenarioId); // Upload input data _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, $"AutoDistributed - uploaded files for run: {RunId.ToString()} scenario: {ScenarioId.ToString()}")); // Create AutoBookRequest bool loggedNotifyFinished = false; try { _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineStart(0, 0, PipelineEventIDs.STARTED_NOTIFYING_AUTOBOOK_API, RunId, ScenarioId, null, null)); _pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun, PipelineEventIDs.STARTED_NOTIFYING_AUTOBOOK_API, RunId, ScenarioId, null)); _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, $"AutoDistributed - Create AutBookRequest for run: {run.Id} scenario: {ScenarioId}")); var result = CreateAutoBookRequest(autoBookInstanceConfigurations); _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForInformationMessage(0, 0, $"AutoDistributed - Created AutBookRequest for run: {run.Id} scenario: {ScenarioId}, result: {result}")); _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineEnd(0, 0, PipelineEventIDs.FINISHED_NOTIFYING_AUTOBOOK_API, RunId, ScenarioId, null, null, null, null)); _pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun, PipelineEventIDs.FINISHED_NOTIFYING_AUTOBOOK_API, RunId, ScenarioId, null)); loggedNotifyFinished = true; } catch (System.Exception exception) { if (!loggedNotifyFinished) { _auditEventRepository.Insert(AuditEventFactory.CreateAuditEventForGameplanPipelineEnd(0, 0, PipelineEventIDs.FINISHED_NOTIFYING_AUTOBOOK_API, RunId, ScenarioId, null, null, exception.Message, exception)); _pipelineAuditEventRepository.Add(PipelineEventHelper.CreatePipelineAuditEvent(AuditEventTypes.GamePlanRun, PipelineEventIDs.FINISHED_NOTIFYING_AUTOBOOK_API, RunId, ScenarioId, exception.Message)); } throw; } finally { _pipelineAuditEventRepository.SaveChanges(); } // Flag as InProgress RunManager.UpdateScenarioStatuses(_repositoryFactory, _auditEventRepository, RunId, new List <Guid>() { ScenarioId }, new List <ScenarioStatuses>() { ScenarioStatuses.InProgress }); startedRun = true; } catch { throw; } finally { // Clean up failure if (!startedRun) { _pipelineAuditEventRepository.SaveChanges(); } } }