private static async Task <string> ConstructDetailsColumn(ImportWorkerQueueRecord importWorkerQueueRecord, int markupSetArtifactId, int redactionId, string fileGuid, string valueType)
{
var retVal = await Task.Run(() =>
{
var sb = new StringBuilder();
sb.Append($@"<auditElement>");
sb.Append($@"<imageMarkup id=""{redactionId}"" pageNumber=""{importWorkerQueueRecord.FileOrder + 1}"" markupSetArtifactID=""{markupSetArtifactId}"" />");
sb.Append($@"<field name=""ID""><{valueType}>{redactionId}</{valueType}></field>");
sb.Append($@"<field name=""FileGuid""><{valueType}>{fileGuid}</{valueType}></field>");
sb.Append($@"<field name=""X""><{valueType}>{importWorkerQueueRecord.X}</{valueType}></field>");
sb.Append($@"<field name=""Y""><{valueType}>{importWorkerQueueRecord.Y}</{valueType}></field>");
sb.Append($@"<field name=""Width""><{valueType}>{importWorkerQueueRecord.Width}</{valueType}></field>");
sb.Append($@"<field name=""Height""><{valueType}>{importWorkerQueueRecord.Height}</{valueType}></field>");
sb.Append($@"<field name=""MarkupSetArtifactID""><{valueType}>{markupSetArtifactId}</{valueType}></field>");
sb.Append($@"<field name=""FillA""><{valueType}>{importWorkerQueueRecord.FillA}</{valueType}></field>");
sb.Append($@"<field name=""FillR""><{valueType}>{importWorkerQueueRecord.FillR}</{valueType}></field>");
sb.Append($@"<field name=""FillG""><{valueType}>{importWorkerQueueRecord.FillG}</{valueType}></field>");
sb.Append($@"<field name=""FillB""><{valueType}>{importWorkerQueueRecord.FillB}</{valueType}></field>");
sb.Append($@"<field name=""BorderSize""><{valueType}>{importWorkerQueueRecord.BorderSize}</{valueType}></field>");
sb.Append($@"<field name=""BorderA""><{valueType}>{importWorkerQueueRecord.BorderA}</{valueType}></field>");
sb.Append($@"<field name=""BorderR""><{valueType}>{importWorkerQueueRecord.BorderR}</{valueType}></field>");
sb.Append($@"<field name=""BorderG""><{valueType}>{importWorkerQueueRecord.BorderG}</{valueType}></field>");
sb.Append($@"<field name=""BorderB""><{valueType}>{importWorkerQueueRecord.BorderB}</{valueType}></field>");
sb.Append($@"<field name=""BorderStyle""><{valueType}>{importWorkerQueueRecord.BorderStyle}</{valueType}></field>");
sb.Append($@"<field name=""FontName""><{valueType}>{importWorkerQueueRecord.FontName}</{valueType}></field>");
sb.Append($@"<field name=""FontA""><{valueType}>{importWorkerQueueRecord.FontA}</{valueType}></field>");
sb.Append($@"<field name=""FontR""><{valueType}>{importWorkerQueueRecord.FontR}</{valueType}></field>");
sb.Append($@"<field name=""FontG""><{valueType}>{importWorkerQueueRecord.FontG}</{valueType}></field>");
sb.Append($@"<field name=""FontB""><{valueType}>{importWorkerQueueRecord.FontB}</{valueType}></field>");
sb.Append($@"<field name=""FontSize""><{valueType}>{importWorkerQueueRecord.FontSize}</{valueType}></field>");
sb.Append($@"<field name=""FontStyle""><{valueType}>{importWorkerQueueRecord.FontStyle}</{valueType}></field>");
sb.Append($@"<field name=""Text""><{valueType}>{importWorkerQueueRecord.Text}</{valueType}></field>");
sb.Append($@"<field name=""ZOrder""><{valueType}>{importWorkerQueueRecord.ZOrder}</{valueType}></field>");
sb.Append($@"<field name=""DrawCrossLines""><{valueType}>{importWorkerQueueRecord.DrawCrossLines}</{valueType}></field>");
sb.Append($@"<field name=""MarkupSubType""><{valueType}>{importWorkerQueueRecord.MarkupSubType}</{valueType}></field>");
sb.Append($@"<field name=""MarkupType""><{valueType}>{importWorkerQueueRecord.MarkupType}</{valueType}></field>");
sb.Append($@"<field name=""X_d""><{valueType}>{importWorkerQueueRecord.Xd}</{valueType}></field>");
sb.Append($@"<field name=""Y_d""><{valueType}>{importWorkerQueueRecord.Yd}</{valueType}></field>");
sb.Append($@"<field name=""Width_d""><{valueType}>{importWorkerQueueRecord.WidthD}</{valueType}></field>");
sb.Append($@"<field name=""Height_d""><{valueType}>{importWorkerQueueRecord.HeightD}</{valueType}></field>");
sb.Append($@"</auditElement>");
return(sb.ToString());
});
return(retVal);
}
private async Task ProcessSingleImportJobAsync(DataRow dataRow)
{
_duplicateRedactionInserted = false;
try
{
//convert datarow to object
_importWorkerQueueRecord = new ImportWorkerQueueRecord(dataRow);
//set _imporJobArtifactId
_importJobArtifactId = _importWorkerQueueRecord.ImportJobArtifactId;
//create error context
_errorContext = $"[RecordId = {_importWorkerQueueRecord.Id}, WorkspaceArtifactId = {WorkspaceArtifactId}, ImportJobArtifactId = {_importWorkerQueueRecord.ImportJobArtifactId}]";
//retrieve import job
var markupUtilityImportJob = await RetrieveImportJobAsync(_importJobArtifactId);
//construct redaction data for history record
_redactionData = await ConstructRedactionDataAsync();
//get document artifact id
var documentArtifactId = await GetDocumentArtifactIdAsync();
//get fileGuid for the document and fileOrder
var fileGuid = await GetFileGuidForDocumentFileOrderAsync(documentArtifactId);
//check if markup set exists
await ArtifactQueries.VerifyIfMarkupSetExistsAsync(AgentHelper.GetServicesManager(), _executionIdentity, WorkspaceArtifactId, markupUtilityImportJob.MarkupSetArtifactId);
//insert redaction into redaction table
await InsertMarkupIntoRedactionTableAsync(fileGuid, markupUtilityImportJob, documentArtifactId);
}
catch (Exception ex)
{
RaiseMessage($"An exception occured when processing import job. {_errorContext}. [Error Message = {ex.Message}]");
//create Markup Utility history record as completed with errors
await CreateFailureHistoryRecordAsync(ex);
//update error redaction count
_errorRedactionCount++;
//log error
await LogErrorAsync(ex);
}
}
public void Constructor_ReceivesTable_Initializes()
{
// Arrange
var table = GetTable();
// Act
var record = new ImportWorkerQueueRecord(table.Rows[0]);
// Assert
Assert.AreEqual(RecordID, record.RecordID);
Assert.AreEqual(WorkspaceArtifactID, record.WorkspaceArtifactID);
Assert.AreEqual(AgentID, record.AgentID);
Assert.AreEqual(JobID, record.JobID);
Assert.AreEqual(ObjectType, record.ObjectType);
Assert.AreEqual(JobType, record.JobType);
Assert.AreEqual(MetaData, record.MetaData);
Assert.AreEqual(ImportRowID, record.ImportRowID);
Assert.AreEqual(QueueStatus, record.QueueStatus);
Assert.AreEqual(ResourceGroupID, record.ResourceGroupID);
Assert.AreEqual(Priority, record.Priority);
Assert.AreEqual(TimeStampUTC.Day, record.TimeStampUTC.Day);
}
public override async Task ExecuteAsync()
{
try
{
//Check for jobs which stopped unexpectedly on this agent thread
RaiseMessage($"Resetting records which failed. [Table = {QueueTable}]");
await ResetUnfishedJobsAsync(AgentHelper.GetDBContext(-1));
//Retrieve the next record to work on
RaiseMessage($"Retrieving next record(s) in the queue. [Table = {QueueTable}]");
string commaDelimitedListOfResourceIds = GetCommaDelimitedListOfResourceIds(AgentResourceGroupIds);
if (commaDelimitedListOfResourceIds != string.Empty)
{
DataTable next = await RetrieveNextAsync(commaDelimitedListOfResourceIds);
if (TableIsNotEmpty(next))
{
try
{
ImportWorkerQueueRecord firstImportWorkerQueueRecord = new ImportWorkerQueueRecord(next.Rows[0]);
// Sets the workspaceArtifactID and RecordID so the agent will have access to them in case of an exception
WorkspaceArtifactId = firstImportWorkerQueueRecord.WorkspaceArtifactId;
RecordId = firstImportWorkerQueueRecord.Id;
int importJobArtifactId = firstImportWorkerQueueRecord.ImportJobArtifactId;
RaiseMessage($"Retrieved record(s) in the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}, ImportJobArtifactId = {importJobArtifactId}]");
//set _imporJobArtifactId
_importJobArtifactId = firstImportWorkerQueueRecord.ImportJobArtifactId;
//create error context
_errorContext = $"[WorkspaceArtifactId = {WorkspaceArtifactId}, ImportJobArtifactId = {_importJobArtifactId}]";
//Process the record(s)
string importJobType = firstImportWorkerQueueRecord.JobType;
await ProcessRecordsAsync(next, importJobType);
}
catch (Exception ex)
{
throw new MarkupUtilityException("Error encountered while Executing Import Worker.", ex);
}
}
else
{
RaiseMessage(Constant.AgentRaiseMessages.NO_RECORDS_IN_QUEUE_FOR_THIS_RESOURCE_POOL);
}
}
else
{
RaiseMessage(Constant.AgentRaiseMessages.AGENT_SERVER_NOT_PART_OF_ANY_RESOURCE_POOL);
}
}
catch (Exception ex)
{
string innerMostExceptionMessage = await ConstructDetailsExceptionMessageAsync(ex);
//update import job status field to complete with errors and details field to inner most exception message
await UpdateImportJobStatusAndDetailsFieldAsync(Constant.Status.Job.ERROR, innerMostExceptionMessage);
//log error
await LogErrorAsync(ex);
}
finally
{
//drop batch table
await FinishAsync();
}
}
public override async Task ExecuteAsync()
{
if (await IsOffHoursAsync(ProcessedOnDateTime))
{
//Check for jobs which stopped unexpectedly on this agent thread
RaiseAndLogDebugMessage($"Resetting records which failed. [Table = {QueueTable}]");
await ResetUnfinishedJobsAsync(AgentHelper.GetDBContext(-1));
//Retrieve the next record to work on
RaiseAndLogDebugMessage($"Retrieving next record(s) in the queue. [Table = {QueueTable}]");
var delimitedListOfResourceGroupIds = GetCommaDelimitedListOfResourceIds(AgentResourceGroupIds);
if (delimitedListOfResourceGroupIds != String.Empty)
{
DataTable batch = await RetrieveBatchAsync(delimitedListOfResourceGroupIds);
if (TableIsNotEmpty(batch))
{
ImportWorkerQueueRecord record = null;
var errors = new List <ImportJobError>();
foreach (DataRow row in batch.Rows)
{
try
{
record = new ImportWorkerQueueRecord(row);
SetJobProperties(record);
RaiseAndLogDebugMessage($"Retrieved record(s) in the queue. [Table = {QueueTable}, ID = {TableRowId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
//Update Status
var status = await GetImportJobStatus(record.WorkspaceArtifactID, record.JobID);
if (status == Constant.Status.Job.COMPLETED_MANAGER)
{
await UpdateJobStatus(record.WorkspaceArtifactID, record.JobID, Constant.Status.Job.IN_PROGRESS_WORKER);
}
//Process the record(s)
RaiseAndLogDebugMessage($"Processing record(s). [Table = {QueueTable}, ID = {TableRowId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
await ProcessRecordsAsync(AgentHelper, AgentHelper.GetDBContext(-1), _repositoryGroup, SqlQueryHelper, record, errors);
RaiseAndLogDebugMessage($"Processed record(s). [Table = {QueueTable}, ID = {TableRowId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
}
catch (Exception ex)
{
if (record != null)
{
var exceptionError = new ImportJobError()
{
Message = String.Format(Constant.ErrorMessages.ImportWorkerLineNumberError, record.ImportRowID, ex.Message), Type = Constant.ImportUtilityJob.ErrorType.DataLevel, LineNumber = record.ImportRowID
};
errors.Add(exceptionError);
}
else
{
throw;
}
}
}
if (record != null)
{
if (errors.Any())
{
RaiseAndLogDebugMessage($"Recording Error(s). [Table = {QueueTable}, ID = {TableRowId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
await _artifactQueryHelper.CreateImportJobErrorRecordsAsync(_apiOptions, record.WorkspaceArtifactID, record.JobID, _repositoryGroup.RdoRepository, errors, record.ObjectType);
}
RaiseAndLogDebugMessage($"Deleting Batch. [Table = {QueueTable}, ID = {TableRowId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
await SqlQueryHelper.DeleteRecordFromQueueAsync(AgentHelper.GetDBContext(-1), AgentId, record.WorkspaceArtifactID, record.JobID, Constant.Tables.ImportWorkerQueue);
}
}
else
{
RaiseAndLogDebugMessage("No records in the queue for this resource pool.");
}
}
else
{
RaiseAndLogDebugMessage("This agent server is not part of any resource pools. Agent execution skipped.");
}
}
else
{
RaiseAndLogDebugMessage($"Current time is not between {OffHoursStartTime} and {OffHoursEndTime}. Agent execution skipped.");
}
}
public async Task ProcessRecordsAsync(IHelper helper, IDBContext eddsDbContext, IRsapiRepositoryGroup repositoryGroup, ISqlQueryHelper sqlQueryHelper, ImportWorkerQueueRecord record, List <ImportJobError> errors)
{
var currentRecordErrors = new List <ImportJobError>();
IAdminObject migrationObject = null;
try
{
migrationObject = await _serializationHelper.DeserializeToAdminObjectAsync(record.MetaData);
}
catch (Exception ex)
{
var serializationError = new ImportJobError()
{
Message = Constant.ErrorMessages.GeneralDeSerializationError, Details = ex.ToString(), Type = Constant.ImportUtilityJob.ErrorType.DataLevel, LineNumber = record.ImportRowID
};
currentRecordErrors.Add(serializationError);
}
if (migrationObject != null && !currentRecordErrors.Any())
{
var violations = new List <String>();
violations.AddRange(await migrationObject.ValidateAsync(_apiOptions, repositoryGroup, _artifactQueryHelper, eddsDbContext, sqlQueryHelper));
if (record.JobType == Constant.ImportUtilityJob.JobType.VALIDATE_SUBMIT && !violations.Any())
{
violations.AddRange(await migrationObject.ImportAsync(_apiOptions, repositoryGroup, _artifactQueryHelper, helper, eddsDbContext, sqlQueryHelper));
}
currentRecordErrors.AddRange(
violations.Select(x => new ImportJobError()
{
Message = x, Type = Constant.ImportUtilityJob.ErrorType.DataLevel, LineNumber = record.ImportRowID
})
);
}
errors.AddRange(currentRecordErrors);
}
private void SetJobProperties(ImportWorkerQueueRecord importWorkerQueueRecord)
{
WorkspaceArtifactId = importWorkerQueueRecord.WorkspaceArtifactID;
TableRowId = importWorkerQueueRecord.ImportRowID;
}
public async Task CreateRedactionAuditRecordAsync(IDBContext workspaceDbContext, int auditActionId, int artifactId, int userId, ImportWorkerQueueRecord importWorkerQueueRecord, int markupSetArtifactId, int redactionId, string fileGuid)
{
if (auditActionId < 0)
{
throw new MarkupUtilityException($"{auditActionId} cannot be negative.");
}
if (artifactId < 0)
{
throw new MarkupUtilityException($"{artifactId} cannot be negative.");
}
if (userId < 0)
{
throw new MarkupUtilityException($"{userId} cannot be negative.");
}
if (importWorkerQueueRecord == null)
{
throw new ArgumentNullException(nameof(importWorkerQueueRecord));
}
if (markupSetArtifactId < 0)
{
throw new MarkupUtilityException($"{markupSetArtifactId} cannot be negative.");
}
if (redactionId < 0)
{
throw new MarkupUtilityException($"{redactionId} cannot be negative.");
}
if (fileGuid == null)
{
throw new ArgumentNullException(nameof(fileGuid));
}
string errorContext = $"{Constant.ErrorMessages.CREATE_REDACTION_AUDIT_RECORD_ERROR} [AuditActionId = {auditActionId}, ArtifactId = {artifactId}, UserId = {userId}, MarkupSetArtifactId = {markupSetArtifactId}, RedactionId = {redactionId}, FileGuid = {fileGuid}, RedactionData = {importWorkerQueueRecord.ToStringRedactionData()}]";
try
{
var requestOrigination = await ConstructRequestOrigination();
var recordOrigination = await ConstructRecordOrigination();
var valueType = auditActionId == Constant.AuditRecord.AuditAction.REDACTION_CREATED ? Constant.AuditRecord.ValueType.NEW_VALUE : Constant.AuditRecord.ValueType.OLD_VALUE;
var details = await ConstructDetailsColumn(importWorkerQueueRecord, markupSetArtifactId, redactionId, fileGuid, valueType);
var redactionAuditRecord = new RedactionAuditRecord(
artifactId,
auditActionId,
details,
userId,
DateTime.UtcNow,
requestOrigination,
recordOrigination,
null,
null
);
await _query.CreateAuditRecordAsync(workspaceDbContext, redactionAuditRecord);
}
catch (Exception ex)
{
throw new MarkupUtilityException(errorContext, ex);
}
}
