private async Task ProcessImportJobAsync(DataTable dataTable)
{
//reset redaction counts
_importedRedactionCount = 0;
_skippedRedactionCount = 0;
_errorRedactionCount = 0;
//update status field of the import job to worker in progress and details field to empty string
await UpdateImportJobStatusAndDetailsFieldAsync(Constant.Status.Job.IN_PROGRESS_WORKER, string.Empty);
//retrieve import job
MarkupUtilityImportJob markupUtilityImportJob = await RetrieveImportJobAsync(_importJobArtifactId);
//set default markup set related field values
await QueryMarkupSetRelatedFieldValuesAsync(markupUtilityImportJob.MarkupSetArtifactId);
for (int i = 0; i < dataTable.Rows.Count; i++)
{
RaiseMessage($"Processing record - {i + 1}");
DataRow currentDataRow = dataTable.Rows[i];
await ProcessSingleImportJobAsync(currentDataRow);
RaiseMessage($"Processed record - {i + 1}");
}
//update all redaction count fields on import job
await UpdateAllRedactionCountFieldValuesAsync();
//Remove the record from the import worker queue
await RemoveRecordsFromImportWorkerQueueTableAsync();
//update import job status and details field if no records exist in import worker queue for the workspace and import job
await UpdateImportJobStatusToCompleteIfNoRecordsExistsInImportWorkerQueueForJobAsync();
}
private async Task ValidateImportJobAsync(ImportManagerQueueRecord importManagerQueueRecord)
{
_errorContext = $"[WorkspaceArtifactId = {WorkspaceArtifactId}, ImportJobArtifactId = {importManagerQueueRecord.ImportJobArtifactId}]";
RaiseMessage($"Validating import job. {_errorContext}");
//Set temporary file path to download file
var tempFileName = $"MarkupUtilitiesImportFile_{Guid.NewGuid()}.csv";
var temporaryDirectory = GetTemporaryDirectory();
var tempFileLocation = Path.Combine(temporaryDirectory, tempFileName);
try
{
//retrieve import job
_markupUtilityImportJob = await RetrieveImportJobAsync(importManagerQueueRecord);
//update status field of the import job to validating and details field to empty string
await UpdateImportJobStatusAndDetailsFieldAsync(Constant.Status.Job.VALIDATING, string.Empty);
//read contents of the import job file
var fileContentsStream = await ReadImportJobFileContentsAsync(tempFileLocation);
//validate contents of the import job file
await ValidateImportJobFileContentsAsync(fileContentsStream);
//update status field of the import job to validating and details field to empty string
await UpdateImportJobStatusAndDetailsFieldAsync(Constant.Status.Job.VALIDATED, string.Empty);
}
catch (Exception ex)
{
RaiseMessage($"An exception occured when validating import job. {_errorContext}. [Error Message = {ex.Message}]");
var innerMostExceptionMessage = await ConstructDetailsExceptionMessageAsync(ex);
//update status field of the import job to validation fail and details field to exception message
await UpdateImportJobStatusAndDetailsFieldAsync(Constant.Status.Job.VALIDATION_FAILED, innerMostExceptionMessage);
//log error
await LogErrorAsync(ex);
}
finally
{
//Delete temp file
if (File.Exists(tempFileLocation))
{
File.Delete(tempFileLocation);
}
if (Directory.Exists(temporaryDirectory))
{
Directory.Delete(temporaryDirectory);
}
}
RaiseMessage($"Validated import job. {_errorContext}");
}
private void MockRetrieveImportJob(MarkupUtilityImportJob markupUtilityImportJob)
{
MockArtifactQueries
.Setup(x => x.RetrieveImportJobAsync(
It.IsAny <IServicesMgr>(),
It.IsAny <ExecutionIdentity>(),
It.IsAny <int>(),
It.IsAny <int>()))
.Returns(Task.FromResult(markupUtilityImportJob))
.Verifiable();
}
private async Task CreateAuditRecordAsync(string fileGuid, MarkupUtilityImportJob markupUtilityImportJob, int documentArtifactId, int redactionId)
{
RaiseMessage($"creating audit record. {_errorContext}");
await AuditRecordHelper.CreateRedactionAuditRecordAsync(
AgentHelper.GetDBContext(WorkspaceArtifactId),
Constant.AuditRecord.AuditAction.REDACTION_CREATED,
documentArtifactId,
markupUtilityImportJob.CreatedBy,
_importWorkerQueueRecord,
markupUtilityImportJob.MarkupSetArtifactId,
redactionId,
fileGuid);
}
private async Task InsertMarkupIntoRedactionTableAsync(string fileGuid, MarkupUtilityImportJob markupUtilityImportJob, int documentArtifactId)
{
//verify if redaction insert has to skipped for duplicate redaction
var skipInsertRedactionIntoRedactionTable = await CheckToSkipInsertRedactionIntoRedactionTableAsync(fileGuid, markupUtilityImportJob);
if (!skipInsertRedactionIntoRedactionTable)
{
RaiseMessage($"Inserting redaction into redaction table. {_errorContext}");
//insert redaction into Redaction table
var redactionId = await QueryHelper.InsertRowIntoRedactionTableAsync(
AgentHelper.GetDBContext(WorkspaceArtifactId),
fileGuid,
markupUtilityImportJob.MarkupSetArtifactId,
_importWorkerQueueRecord);
if (redactionId > 0) //redaction was created
{
RaiseMessage($"Redaction inserted into redaction table. {_errorContext}");
//update Has Redactions/Has Highlights choice values for the document
await UpdateMarkupSetMultipleChoiceFieldValueAsync(documentArtifactId, _importWorkerQueueRecord.MarkupType);
//create audit record
await CreateAuditRecordAsync(fileGuid, markupUtilityImportJob, documentArtifactId, redactionId);
//create Markup Utility history record as completed
await CreateSuccessHistoryRecordAsync(redactionId);
//update imported redaction count
_importedRedactionCount++;
}
else //redaction creation failed
{
RaiseMessage($"An error occured when inserting redaction into redaction table. {_errorContext}");
throw new MarkupUtilityException(Constant.ErrorMessages.INSERT_REDACTION_INTO_REDACTION_TABLE_ERROR);
}
}
else
{
RaiseMessage($"Duplicate redaction found. skipping redaction insert into redaction table. {_errorContext}");
//create Markup Utility history record as skipped
await CreateSkippedHistoryRecordAsync();
//update skipped redaction count
_skippedRedactionCount++;
}
}
private static MarkupUtilityImportJob GetMarkupUtilityImportJob(string jobType)
{
var retVal = new MarkupUtilityImportJob(
123,
string.Empty,
123,
Constant.MarkupSubTypeCategory.SupportedMarkupUtilityTypes,
false,
123,
string.Empty,
string.Empty,
123,
123,
jobType,
123);
return(retVal);
}
private async Task <bool> CheckToSkipInsertRedactionIntoRedactionTableAsync(string fileGuid, MarkupUtilityImportJob markupUtilityImportJob)
{
RaiseMessage($"Checking for duplicate redaction. {_errorContext}");
var retVal = true;
//check for duplicate redaction
var hasRedactions = await QueryHelper.DoesRedactionExistAsync(AgentHelper.GetDBContext(WorkspaceArtifactId), fileGuid, markupUtilityImportJob.MarkupSetArtifactId, _importWorkerQueueRecord);
if (hasRedactions)
{
_duplicateRedactionInserted = true;
}
if (!_importWorkerQueueRecord.SkipDuplicateRedactions || (_importWorkerQueueRecord.SkipDuplicateRedactions && !hasRedactions))
{
retVal = false;
}
return(retVal);
}
private async Task ImportJobAsync(ImportManagerQueueRecord importManagerQueueRecord)
{
_errorContext = $"[WorkspaceArtifactId = {WorkspaceArtifactId}, ImportJobArtifactId = {importManagerQueueRecord.ImportJobArtifactId}]";
RaiseMessage($"Processing import job. {_errorContext}");
//Set temporary file path to download file
var tempFileName = $"MarkupUtilitiesImportFile_{Guid.NewGuid()}.csv";
var temporaryDirectory = GetTemporaryDirectory();
var tempFileLocation = Path.Combine(temporaryDirectory, tempFileName);
try
{
//Create import manager holding table
await CreateImportManagerHoldingTableAsync();
//retrieve import job
_markupUtilityImportJob = await RetrieveImportJobAsync(importManagerQueueRecord);
//update status field of the import job to manager in progress and details field to empty string
await UpdateImportJobStatusAndDetailsFieldAsync(Constant.Status.Job.IN_PROGRESS_MANAGER, string.Empty);
//read contents of the import job file
var fileContentsStream = await ReadImportJobFileContentsAsync(tempFileLocation);
//parse import job file for contents
await ParseImportJobFileContentsAsync(fileContentsStream);
//bulk copy data from import manager holding table to import worker queue table
await BulkCopyDataFromImportManagerHoldingTableIntoImportWorkerQueueTableAsync();
//update status field of the import job to manager complete and details field to empty string
await UpdateImportJobStatusAndDetailsFieldAsync(Constant.Status.Job.COMPLETED_MANAGER, string.Empty);
}
catch (Exception ex)
{
RaiseMessage($"An exception occured when processing import job. {_errorContext}. [Error Message = {ex.Message}]");
var innerMostExceptionMessage = await ConstructDetailsExceptionMessageAsync(ex);
//update status field of the import job to parsing fail and details field to exception message
await UpdateImportJobStatusAndDetailsFieldAsync(Constant.Status.Job.ERROR, innerMostExceptionMessage);
//log error
await LogErrorAsync(ex);
}
finally
{
//drop import manager holding table
await DropImportManagerHoldingTableAsync();
//Delete temp file
if (File.Exists(tempFileLocation))
{
File.Delete(tempFileLocation);
}
if (Directory.Exists(temporaryDirectory))
{
Directory.Delete(temporaryDirectory);
}
}
RaiseMessage($"Processed import job. {_errorContext}");
}
