private async Task CreateAuditRecordAsync(DataRow datarow, MarkupUtilityReproduceJob job, int documentArtifactId)
{
RaiseMessage($"creating audit record. {_errorContext}");
await _auditRecordHelper.CreateRedactionAuditRecordAsync(
AgentHelper.GetDBContext(WorkspaceArtifactId),
Constant.AuditRecord.AuditAction.REDACTION_CREATED,
documentArtifactId,
job.CreatedBy,
(string)datarow["FileGuid"],
(int)datarow["ID"],
(int)datarow["MarkupSetArtifactID"],
(int)datarow["Order"] + 1);
}
public void TestMarkupUtilityReproduceJob()
{
var markupUtilityReproduceJob = new MarkupUtilityReproduceJob(1, "2", 3, 4, 5, "6", "7", 8, 1, 1, 1);
Assert.AreEqual(1, markupUtilityReproduceJob.ArtifactId);
Assert.AreEqual("2", markupUtilityReproduceJob.Name);
Assert.AreEqual(3, markupUtilityReproduceJob.SavedSearchArtifactId);
Assert.AreEqual(4, markupUtilityReproduceJob.SourceMarkupSetArtifactId);
Assert.AreEqual(5, markupUtilityReproduceJob.DestinationMarkupSetArtifactId);
Assert.AreEqual("6", markupUtilityReproduceJob.Status);
Assert.AreEqual("7", markupUtilityReproduceJob.Details);
Assert.AreEqual(8, markupUtilityReproduceJob.CreatedBy);
}
private async Task ProcessRedactionsAsync(ReproduceWorkerQueueRecord reproduceWorkerQueueRecord, MarkupUtilityReproduceJob markupUtilityReproduceJob)
{
try
{
//Bulk insert the redactions for the document batch
DataTable dataTable;
var relationalGroup = markupUtilityReproduceJob.RelationalField > 0;
if (relationalGroup)
{
dataTable = await ReproduceAcrossRelationalGroup(reproduceWorkerQueueRecord, markupUtilityReproduceJob);
}
else
{
dataTable = await ReproduceAcrossDocumentSet(reproduceWorkerQueueRecord, markupUtilityReproduceJob);
}
//Get the ids from the result and add to audit and history tables. Ideally, this should be in the same db transaction as above....
await CreateHistoryAndAuditRecords(reproduceWorkerQueueRecord, dataTable, markupUtilityReproduceJob, relationalGroup);
await Finish(reproduceWorkerQueueRecord, Constant.Status.Job.COMPLETED);
}
catch (Exception ex)
{
//Set the status of the Export Job to Error
await UpdateStatusFieldAsync(reproduceWorkerQueueRecord.ReproduceJobArtifactId, Constant.Status.Job.ERROR);
await UpdateDetailsFieldAsync(reproduceWorkerQueueRecord.ReproduceJobArtifactId, await ConstructDetailsExceptionMessageAsync(ex));
RaiseMessage($"Logging error.");
await LogErrorAsync(ex);
}
}
private async Task CreateHistoryAndAuditRecords(ReproduceWorkerQueueRecord reproduceWorkerQueueRecord, DataTable dataTable, MarkupUtilityReproduceJob markupUtilityReproduceJob, bool relationalGroup)
{
var details = Constant.Status.History.Details.REPRODUCED_REDACTION_IN_DOCUMENT_SET;
if (relationalGroup)
{
details = Constant.Status.History.Details.REPRODUCED_REDACTION_IN_RELATIONAL_GROUP;
}
foreach (DataRow row in dataTable.Rows)
{
var redactionId = (int)row["ID"];
var table = await RetrieveRedactionInfoAsync(reproduceWorkerQueueRecord, redactionId);
var dataRow = table.Rows[0];
var identifier = (string)dataRow["Identifier"];
var pageNumber = (int)dataRow["Order"] + 1;
var markupUtilityHistoryRecordAsync = _artifactQueries.CreateMarkupUtilityHistoryRecordAsync(
AgentHelper.GetServicesManager(),
ExecutionIdentity.CurrentUser,
WorkspaceArtifactId,
-1,
identifier,
pageNumber,
Constant.ImportJobType.REPRODUCE,
null,
Constant.Status.History.COMPLETED,
details,
ToStringRedactionData(dataRow),
redactionId,
reproduceWorkerQueueRecord.ReproduceJobArtifactId);
var auditRecordAsync = CreateAuditRecordAsync(dataRow, markupUtilityReproduceJob, (int)dataRow["DocumentArtifactID"]);
await Task.WhenAll(markupUtilityHistoryRecordAsync, auditRecordAsync);
}
}
private async Task <DataTable> ReproduceAcrossRelationalGroup(ReproduceWorkerQueueRecord reproduceWorkerQueueRecord, MarkupUtilityReproduceJob markupUtilityReproduceJob)
{
var dataTable = await BulkInsertRedactionRecordsForRelationalGroup(reproduceWorkerQueueRecord);
if (!string.IsNullOrEmpty(reproduceWorkerQueueRecord.HasAutoRedactionsColumn))
{
await BulkUpdateHasAutoRedactionsFieldForRelationalGroup(reproduceWorkerQueueRecord);
}
await UpdateHasRedactionsOrHighlightsAsync(reproduceWorkerQueueRecord, markupUtilityReproduceJob.DestinationMarkupSetArtifactId, true);
return(dataTable);
}
private async Task <bool> AddDocumentsToSavedSearchHoldingTableAsync(int workspaceArtifactId, MarkupUtilityReproduceJob reproduceJob)
{
List <SqlBulkCopyColumnMapping> columnMappingsHoldingTable;
//Create column mappings for holding table
if (reproduceJob.RelationalFieldColumnName != null)
{
var sqlBulkCopyColumnMappings = new List <SqlBulkCopyColumnMapping>();
sqlBulkCopyColumnMappings.Add(new SqlBulkCopyColumnMapping("DocumentArtifactID", "DocumentArtifactID"));
sqlBulkCopyColumnMappings.Add(new SqlBulkCopyColumnMapping(reproduceJob.RelationalFieldColumnName, "RelationalGroup"));
columnMappingsHoldingTable = sqlBulkCopyColumnMappings;
}
else
{
columnMappingsHoldingTable =
await GenerateColumnMappingsHoldingTableAsync(new List <string> {
"DocumentArtifactID"
});
}
return(await _artifactQueries.AddDocumentsToHoldingTableAsync(AgentHelper.GetServicesManager(), AgentHelper.GetDBContext(workspaceArtifactId), _utilityQueryHelper, ExecutionIdentity.CurrentUser, WorkspaceArtifactId, reproduceJob.SavedSearchArtifactId, SavedSearchHoldingTable, columnMappingsHoldingTable));
}
private async Task <int> ReproduceAcrossRelationalGroup(ReproduceManagerQueueRecord reproduceManagerQueueRecord, MarkupUtilityReproduceJob reproduceJob, int codeTypeId, int markupSetRedactionCodeArtifactId, int markupSetAnnotationCodeArtifactId)
{
var dataTable = await RetrieveRelationalGroupsAsync(reproduceManagerQueueRecord.WorkspaceArtifactId);
//create a new record for each relational group
var insertTasks = new List <Task>();
foreach (DataRow row in dataTable.Rows)
{
var relationalGroup = (string)row["RelationalGroup"];
insertTasks.Add(InsertRecordIntoToReproduceWorkerQueueAsync(
reproduceManagerQueueRecord.WorkspaceArtifactId,
-1,
-1,
-1,
reproduceJob.ArtifactId,
reproduceManagerQueueRecord.ResourceGroupId, codeTypeId, markupSetRedactionCodeArtifactId,
markupSetAnnotationCodeArtifactId, reproduceJob.RelationalFieldColumnName,
reproduceJob.HasAutoRedactionsFieldColumnName, relationalGroup));
}
await Task.WhenAll(insertTasks.ToArray());
return(insertTasks.Count);
}
private async Task <int> ReproduceAcrossDocumentSet(ReproduceManagerQueueRecord reproduceManagerQueueRecord, MarkupUtilityReproduceJob reproduceJob, int min, int max, int codeTypeId, int markupSetRedactionCodeArtifactId, int markupSetAnnotationCodeArtifactId)
{
//create Worker records splitting the records
var insertTasks = new List <Task>();
for (var i = min; i <= max; i = i + Constant.Sizes.ReproduceJobInsertBatchSize)
{
insertTasks.Add(InsertRecordIntoToReproduceWorkerQueueAsync(
reproduceManagerQueueRecord.WorkspaceArtifactId,
i,
i + Constant.Sizes.ReproduceJobInsertBatchSize - 1,
reproduceJob.DestinationMarkupSetArtifactId,
reproduceJob.ArtifactId,
reproduceManagerQueueRecord.ResourceGroupId, codeTypeId, markupSetRedactionCodeArtifactId,
markupSetAnnotationCodeArtifactId, null, reproduceJob.HasAutoRedactionsFieldColumnName, null));
}
await Task.WhenAll(insertTasks.ToArray());
return(insertTasks.Count);
}
public async Task ProcessRecordsAsync(ReproduceManagerQueueRecord reproduceManagerQueueRecord, MarkupUtilityReproduceJob reproduceJob)
{
RaiseMessage($"Processing record(s). [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
try
{
//Update status of the Job to Manager In Progress
await UpdateStatusFieldAsync(reproduceManagerQueueRecord.ReproduceJobArtifactId, Constant.Status.Job.IN_PROGRESS_MANAGER);
//Flush the Details of the Job
await UpdateDetailsFieldAsync(reproduceManagerQueueRecord.ReproduceJobArtifactId, string.Empty);
var includeRelationalGroup = reproduceJob.RelationalField > 0;
if (reproduceJob.HasAutoRedactionsField > 0)
{
var columnName = await RetrieveDocumentColumnAsync(reproduceManagerQueueRecord.WorkspaceArtifactId, reproduceJob.HasAutoRedactionsField);
reproduceJob.HasAutoRedactionsFieldColumnName = columnName;
}
//Create holding tables
if (includeRelationalGroup)
{
await CreateSavedSearchHoldingTableAsync(reproduceManagerQueueRecord.WorkspaceArtifactId, true);
var relationalColumnName = await RetrieveDocumentColumnAsync(reproduceManagerQueueRecord.WorkspaceArtifactId, reproduceJob.RelationalField);
reproduceJob.RelationalFieldColumnName = relationalColumnName;
}
else
{
await CreateSavedSearchHoldingTableAsync(reproduceManagerQueueRecord.WorkspaceArtifactId, false);
}
await CreateRedactionsHoldingTableAsync(reproduceManagerQueueRecord.WorkspaceArtifactId, includeRelationalGroup);
//Add Documents from Saved Search in the Export Worker Queue
var isSuccessful = await AddDocumentsToSavedSearchHoldingTableAsync(reproduceManagerQueueRecord.WorkspaceArtifactId, reproduceJob);
var jobIsComplete = false;
var jobMessage = "";
if (isSuccessful)
{
var dataTable = await RetrieveMinMaxIdAsync(reproduceManagerQueueRecord.WorkspaceArtifactId);
var min = (int)dataTable.Rows[0][0];
var max = (int)dataTable.Rows[0][1];
var tasks = new List <Task <int> >();
for (var i = min; i <= max; i = i + Constant.Sizes.ReproduceJobBatchSize)
{
//sql between has inclusive end
tasks.Add(AddDocumentsToRedactionsHoldingTableAsync(reproduceManagerQueueRecord.WorkspaceArtifactId, reproduceJob.SourceMarkupSetArtifactId, reproduceJob.DestinationMarkupSetArtifactId, i, i + Constant.Sizes.ReproduceJobBatchSize - 1, includeRelationalGroup));
}
var results = await Task.WhenAll(tasks.ToArray());
var existRedactions = Array.Exists(results, r => r > 0);
if (existRedactions)
{
var result = await RetrieveZCodesAsync(reproduceManagerQueueRecord.WorkspaceArtifactId, reproduceJob.DestinationMarkupSetArtifactId);
var codeTypeId = (int)result.Rows[0]["CodeTypeID"];
var markupSetRedactionCodeArtifactId = (int)result.Rows[0]["RedactionCodeArtifactID"];
var markupSetAnnotationCodeArtifactId = (int)result.Rows[0]["AnnotationCodeArtifactID"];
int count;
if (reproduceJob.RelationalField > 0)
{
count = await ReproduceAcrossRelationalGroup(reproduceManagerQueueRecord, reproduceJob, codeTypeId, markupSetRedactionCodeArtifactId, markupSetAnnotationCodeArtifactId);
}
else
{
count = await ReproduceAcrossDocumentSet(reproduceManagerQueueRecord, reproduceJob, min, max, codeTypeId, markupSetRedactionCodeArtifactId, markupSetAnnotationCodeArtifactId);
}
if (count == 0)
{
jobIsComplete = true;
jobMessage = "No redactions found for selected criteria.";
}
}
else
{
jobIsComplete = true;
jobMessage = "No redactions found for selected criteria.";
}
}
else
{
jobIsComplete = true;
jobMessage = "No Documents found in the selected Saved Search.";
}
if (jobIsComplete)
{
var tasks = new List <Task>
{
UpdateStatusFieldAsync(reproduceManagerQueueRecord.ReproduceJobArtifactId, Constant.Status.Job.COMPLETED), UpdateDetailsFieldAsync(reproduceManagerQueueRecord.ReproduceJobArtifactId, jobMessage), QueryHelper.DropTableAsync(AgentHelper.GetDBContext(reproduceManagerQueueRecord.WorkspaceArtifactId), RedactionsHoldingTable), QueryHelper.DropTableAsync(AgentHelper.GetDBContext(reproduceManagerQueueRecord.WorkspaceArtifactId), SavedSearchHoldingTable)
};
//Update status of the Export Job to Completed
//Update Details of the Export Job indicating no Documents in Saved Search
await Task.WhenAll(tasks);
}
else
{
//Update status of the Export Job to Completed - Manager
await UpdateStatusFieldAsync(reproduceManagerQueueRecord.ReproduceJobArtifactId, Constant.Status.Job.COMPLETED_MANAGER);
//Flush the Details of the Export Job
await UpdateDetailsFieldAsync(reproduceManagerQueueRecord.ReproduceJobArtifactId, string.Empty);
}
//delete import job from queue
RaiseMessage($"Removing record(s) from the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
await FinishAsync();
await DeleteSavedSearchHoldingTableAsync();
await DeleteRedactionsHoldingTableAsync();
RaiseMessage($"Removed record(s) from the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
RaiseMessage($"Processed record(s). [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
}
catch (Exception ex)
{
//Update status of the Export Job to Error
await UpdateStatusFieldAsync(reproduceManagerQueueRecord.ReproduceJobArtifactId, Constant.Status.Job.ERROR);
//Update Details of the Export Job
await UpdateDetailsFieldAsync(reproduceManagerQueueRecord.ReproduceJobArtifactId, ex.ToString());
//log error
await LogErrorAsync(ex);
//delete import job from queue
RaiseMessage($"Removing record(s) from the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
await FinishAsync();
await DeleteSavedSearchHoldingTableAsync();
await DeleteRedactionsHoldingTableAsync();
RaiseMessage($"Removed record(s) from the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
RaiseMessage($"Processed record(s). [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
}
}
