private async Task PopulateWorkerQueueRecordsAsync(String delimitedListOfResourceGroupIds) { IParser parser = null; FileStream downloadedFile = null; ImportManagerQueueRecord managerQueueRecord = null; RDO importJob; var violations = new List <ImportJobError>(); try { DataTable next = await RetrieveNextAsync(delimitedListOfResourceGroupIds); if (TableIsNotEmpty(next)) { managerQueueRecord = new ImportManagerQueueRecord(next.Rows[0]); SetJobProperties(managerQueueRecord); await UpdateJobStatusAsync(managerQueueRecord.WorkspaceArtifactId, managerQueueRecord.JobId, Constant.Status.Job.IN_PROGRESS_MANAGER); RaiseAndLogDebugMessage($"Initializing Job {managerQueueRecord.JobId}. [Table = {QueueTable}, ID = {managerQueueRecord.RecordId}, Workspace Artifact ID = {managerQueueRecord.WorkspaceArtifactId}]"); importJob = await GetImportJob(managerQueueRecord.WorkspaceArtifactId, managerQueueRecord.JobId); var jobFileField = importJob[Constant.Guids.Field.ImportUtilityJob.ImportFile]; RaiseAndLogDebugMessage($"Downloading File {jobFileField.ValueAsFixedLengthText}. [Table = {QueueTable}, ID = {managerQueueRecord.RecordId}, Workspace Artifact ID = {managerQueueRecord.WorkspaceArtifactId}]"); await DownloadFile(managerQueueRecord.WorkspaceArtifactId, managerQueueRecord.JobId, jobFileField.ArtifactID, LocalTempFilePath); RaiseAndLogDebugMessage($"Opening File {LocalTempFilePath}. [Table = {QueueTable}, ID = {managerQueueRecord.RecordId}, Workspace Artifact ID = {managerQueueRecord.WorkspaceArtifactId}]"); downloadedFile = await OpenFileAsync(LocalTempFilePath); RaiseAndLogDebugMessage($"Creating Delimited File Parser. [Table = {QueueTable}, ID = {managerQueueRecord.RecordId}, Workspace Artifact ID = {managerQueueRecord.WorkspaceArtifactId}]"); parser = await CreateParserAsync(downloadedFile, violations); RaiseAndLogDebugMessage($"Creating Import Object. [Table = {QueueTable}, ID = {managerQueueRecord.RecordId}, Workspace Artifact ID = {managerQueueRecord.WorkspaceArtifactId}]"); //var migrationObject = await Utility.GetImportObjectSelection(managerQueueRecord.ObjectType, violations); var migrationObject = await GetMigrationObject(managerQueueRecord.ObjectType, violations); RaiseAndLogDebugMessage($"Validating load file columns. [Table = {QueueTable}, ID = {managerQueueRecord.RecordId}, Workspace Artifact ID = {managerQueueRecord.WorkspaceArtifactId}]"); await ValidateColumnsAsync(migrationObject, parser, violations); await ValidateFileDoesNotContainExtraColumnsAsync(migrationObject, parser, managerQueueRecord.ObjectType, violations); RaiseAndLogDebugMessage($"Validating load file minimum data requirement. [Table = {QueueTable}, ID = {managerQueueRecord.RecordId}, Workspace Artifact ID = {managerQueueRecord.WorkspaceArtifactId}]"); await ValidateFileHasMetaData(parser, violations); if (violations.Any()) { RaiseAndLogDebugMessage($"Violations found, recording them in the Job's error log. [Table = {QueueTable}, ID = {managerQueueRecord.RecordId}, Workspace Artifact ID = {managerQueueRecord.WorkspaceArtifactId}]"); await _artifactQueryHelper.CreateImportJobErrorRecordsAsync(_apiOptions, managerQueueRecord.WorkspaceArtifactId, managerQueueRecord.JobId, _rsapiRepositoryGroup.RdoRepository, violations, managerQueueRecord.ObjectType); await UpdateJobStatusAsync(managerQueueRecord.WorkspaceArtifactId, managerQueueRecord.JobId, Constant.Status.Job.ERROR); await ClearQueueRecords(managerQueueRecord.WorkspaceArtifactId, managerQueueRecord.JobId); await SendEmail(managerQueueRecord, Constant.Status.Job.ERROR); } else { try { RaiseAndLogDebugMessage($"Populating Worker Queue Table. [Table = {QueueTable}, ID = {managerQueueRecord.RecordId}, Workspace Artifact ID = {managerQueueRecord.WorkspaceArtifactId}]"); var numberProcessed = await PopulateWorkerQueueAsync(managerQueueRecord, parser, migrationObject, violations); await UpdateJobStatisticsAsync(managerQueueRecord, numberProcessed); await _artifactQueryHelper.CreateImportJobErrorRecordsAsync(_apiOptions, managerQueueRecord.WorkspaceArtifactId, managerQueueRecord.JobId, _rsapiRepositoryGroup.RdoRepository, violations, managerQueueRecord.ObjectType); await SqlQueryHelper.UpdateQueueStatusAsync(AgentHelper.GetDBContext(-1), managerQueueRecord.WorkspaceArtifactId, managerQueueRecord.JobId, Constant.Tables.ImportManagerQueue, Constant.Sql.ColumnsNames.ImportManagerQueue.QueueStatus, Constant.Status.Queue.WAITING_FOR_WORKERS_TO_FINISH); await UpdateJobStatusAsync(managerQueueRecord.WorkspaceArtifactId, managerQueueRecord.JobId, Constant.Status.Job.COMPLETED_MANAGER); } catch (Exception ex) { var error = new ImportJobError() { Message = ex.Message, Type = Constant.ImportUtilityJob.ErrorType.FileLevel, Details = ex.ToString(), LineNumber = null }; await _artifactQueryHelper.CreateImportJobErrorRecordsAsync(_apiOptions, managerQueueRecord.WorkspaceArtifactId, managerQueueRecord.JobId, _rsapiRepositoryGroup.RdoRepository, new[] { error }, managerQueueRecord.ObjectType); await UpdateJobStatusAsync(managerQueueRecord.WorkspaceArtifactId, managerQueueRecord.JobId, Constant.Status.Job.ERROR); await ClearQueueRecords(managerQueueRecord.WorkspaceArtifactId, managerQueueRecord.JobId); await SendEmail(managerQueueRecord, Constant.Status.Job.ERROR); } } } else { RaiseAndLogDebugMessage("No records in the queue for this resource pool."); } } catch (Exception ex) { if (managerQueueRecord != null) { var error = new ImportJobError() { Message = ex.Message, Type = Constant.ImportUtilityJob.ErrorType.JobLevel, LineNumber = null, Details = ex.ToString() }; await _artifactQueryHelper.CreateImportJobErrorRecordsAsync(_apiOptions, managerQueueRecord.WorkspaceArtifactId, managerQueueRecord.JobId, _rsapiRepositoryGroup.RdoRepository, new[] { error }, managerQueueRecord.ObjectType); await UpdateJobStatusAsync(managerQueueRecord.WorkspaceArtifactId, managerQueueRecord.JobId, Constant.Status.Job.RETRY); } throw; } finally { if (parser != null) { parser.Dispose(); } if (downloadedFile != null) { downloadedFile.Close(); } } }
public override async Task ExecuteAsync() { if (await IsOffHoursAsync(ProcessedOnDateTime)) { //Check for jobs which stopped unexpectedly on this agent thread RaiseAndLogDebugMessage($"Resetting records which failed. [Table = {QueueTable}]"); await ResetUnfinishedJobsAsync(AgentHelper.GetDBContext(-1)); //Retrieve the next record to work on RaiseAndLogDebugMessage($"Retrieving next record(s) in the queue. [Table = {QueueTable}]"); var delimitedListOfResourceGroupIds = GetCommaDelimitedListOfResourceIds(AgentResourceGroupIds); if (delimitedListOfResourceGroupIds != String.Empty) { DataTable batch = await RetrieveBatchAsync(delimitedListOfResourceGroupIds); if (TableIsNotEmpty(batch)) { ImportWorkerQueueRecord record = null; var errors = new List <ImportJobError>(); foreach (DataRow row in batch.Rows) { try { record = new ImportWorkerQueueRecord(row); SetJobProperties(record); RaiseAndLogDebugMessage($"Retrieved record(s) in the queue. [Table = {QueueTable}, ID = {TableRowId}, Workspace Artifact ID = {WorkspaceArtifactId}]"); //Update Status var status = await GetImportJobStatus(record.WorkspaceArtifactID, record.JobID); if (status == Constant.Status.Job.COMPLETED_MANAGER) { await UpdateJobStatus(record.WorkspaceArtifactID, record.JobID, Constant.Status.Job.IN_PROGRESS_WORKER); } //Process the record(s) RaiseAndLogDebugMessage($"Processing record(s). [Table = {QueueTable}, ID = {TableRowId}, Workspace Artifact ID = {WorkspaceArtifactId}]"); await ProcessRecordsAsync(AgentHelper, AgentHelper.GetDBContext(-1), _repositoryGroup, SqlQueryHelper, record, errors); RaiseAndLogDebugMessage($"Processed record(s). [Table = {QueueTable}, ID = {TableRowId}, Workspace Artifact ID = {WorkspaceArtifactId}]"); } catch (Exception ex) { if (record != null) { var exceptionError = new ImportJobError() { Message = String.Format(Constant.ErrorMessages.ImportWorkerLineNumberError, record.ImportRowID, ex.Message), Type = Constant.ImportUtilityJob.ErrorType.DataLevel, LineNumber = record.ImportRowID }; errors.Add(exceptionError); } else { throw; } } } if (record != null) { if (errors.Any()) { RaiseAndLogDebugMessage($"Recording Error(s). [Table = {QueueTable}, ID = {TableRowId}, Workspace Artifact ID = {WorkspaceArtifactId}]"); await _artifactQueryHelper.CreateImportJobErrorRecordsAsync(_apiOptions, record.WorkspaceArtifactID, record.JobID, _repositoryGroup.RdoRepository, errors, record.ObjectType); } RaiseAndLogDebugMessage($"Deleting Batch. [Table = {QueueTable}, ID = {TableRowId}, Workspace Artifact ID = {WorkspaceArtifactId}]"); await SqlQueryHelper.DeleteRecordFromQueueAsync(AgentHelper.GetDBContext(-1), AgentId, record.WorkspaceArtifactID, record.JobID, Constant.Tables.ImportWorkerQueue); } } else { RaiseAndLogDebugMessage("No records in the queue for this resource pool."); } } else { RaiseAndLogDebugMessage("This agent server is not part of any resource pools. Agent execution skipped."); } } else { RaiseAndLogDebugMessage($"Current time is not between {OffHoursStartTime} and {OffHoursEndTime}. Agent execution skipped."); } }