private async Task CreateAuditRecordAsync(DataRow datarow, MarkupUtilityReproduceJob job, int documentArtifactId) { RaiseMessage($"creating audit record. {_errorContext}"); await _auditRecordHelper.CreateRedactionAuditRecordAsync( AgentHelper.GetDBContext(WorkspaceArtifactId), Constant.AuditRecord.AuditAction.REDACTION_CREATED, documentArtifactId, job.CreatedBy, (string)datarow["FileGuid"], (int)datarow["ID"], (int)datarow["MarkupSetArtifactID"], (int)datarow["Order"] + 1); }
public void TestMarkupUtilityReproduceJob() { var markupUtilityReproduceJob = new MarkupUtilityReproduceJob(1, "2", 3, 4, 5, "6", "7", 8, 1, 1, 1); Assert.AreEqual(1, markupUtilityReproduceJob.ArtifactId); Assert.AreEqual("2", markupUtilityReproduceJob.Name); Assert.AreEqual(3, markupUtilityReproduceJob.SavedSearchArtifactId); Assert.AreEqual(4, markupUtilityReproduceJob.SourceMarkupSetArtifactId); Assert.AreEqual(5, markupUtilityReproduceJob.DestinationMarkupSetArtifactId); Assert.AreEqual("6", markupUtilityReproduceJob.Status); Assert.AreEqual("7", markupUtilityReproduceJob.Details); Assert.AreEqual(8, markupUtilityReproduceJob.CreatedBy); }
private async Task ProcessRedactionsAsync(ReproduceWorkerQueueRecord reproduceWorkerQueueRecord, MarkupUtilityReproduceJob markupUtilityReproduceJob) { try { //Bulk insert the redactions for the document batch DataTable dataTable; var relationalGroup = markupUtilityReproduceJob.RelationalField > 0; if (relationalGroup) { dataTable = await ReproduceAcrossRelationalGroup(reproduceWorkerQueueRecord, markupUtilityReproduceJob); } else { dataTable = await ReproduceAcrossDocumentSet(reproduceWorkerQueueRecord, markupUtilityReproduceJob); } //Get the ids from the result and add to audit and history tables. Ideally, this should be in the same db transaction as above.... await CreateHistoryAndAuditRecords(reproduceWorkerQueueRecord, dataTable, markupUtilityReproduceJob, relationalGroup); await Finish(reproduceWorkerQueueRecord, Constant.Status.Job.COMPLETED); } catch (Exception ex) { //Set the status of the Export Job to Error await UpdateStatusFieldAsync(reproduceWorkerQueueRecord.ReproduceJobArtifactId, Constant.Status.Job.ERROR); await UpdateDetailsFieldAsync(reproduceWorkerQueueRecord.ReproduceJobArtifactId, await ConstructDetailsExceptionMessageAsync(ex)); RaiseMessage($"Logging error."); await LogErrorAsync(ex); } }
private async Task CreateHistoryAndAuditRecords(ReproduceWorkerQueueRecord reproduceWorkerQueueRecord, DataTable dataTable, MarkupUtilityReproduceJob markupUtilityReproduceJob, bool relationalGroup) { var details = Constant.Status.History.Details.REPRODUCED_REDACTION_IN_DOCUMENT_SET; if (relationalGroup) { details = Constant.Status.History.Details.REPRODUCED_REDACTION_IN_RELATIONAL_GROUP; } foreach (DataRow row in dataTable.Rows) { var redactionId = (int)row["ID"]; var table = await RetrieveRedactionInfoAsync(reproduceWorkerQueueRecord, redactionId); var dataRow = table.Rows[0]; var identifier = (string)dataRow["Identifier"]; var pageNumber = (int)dataRow["Order"] + 1; var markupUtilityHistoryRecordAsync = _artifactQueries.CreateMarkupUtilityHistoryRecordAsync( AgentHelper.GetServicesManager(), ExecutionIdentity.CurrentUser, WorkspaceArtifactId, -1, identifier, pageNumber, Constant.ImportJobType.REPRODUCE, null, Constant.Status.History.COMPLETED, details, ToStringRedactionData(dataRow), redactionId, reproduceWorkerQueueRecord.ReproduceJobArtifactId); var auditRecordAsync = CreateAuditRecordAsync(dataRow, markupUtilityReproduceJob, (int)dataRow["DocumentArtifactID"]); await Task.WhenAll(markupUtilityHistoryRecordAsync, auditRecordAsync); } }
private async Task <DataTable> ReproduceAcrossRelationalGroup(ReproduceWorkerQueueRecord reproduceWorkerQueueRecord, MarkupUtilityReproduceJob markupUtilityReproduceJob) { var dataTable = await BulkInsertRedactionRecordsForRelationalGroup(reproduceWorkerQueueRecord); if (!string.IsNullOrEmpty(reproduceWorkerQueueRecord.HasAutoRedactionsColumn)) { await BulkUpdateHasAutoRedactionsFieldForRelationalGroup(reproduceWorkerQueueRecord); } await UpdateHasRedactionsOrHighlightsAsync(reproduceWorkerQueueRecord, markupUtilityReproduceJob.DestinationMarkupSetArtifactId, true); return(dataTable); }
private async Task <bool> AddDocumentsToSavedSearchHoldingTableAsync(int workspaceArtifactId, MarkupUtilityReproduceJob reproduceJob) { List <SqlBulkCopyColumnMapping> columnMappingsHoldingTable; //Create column mappings for holding table if (reproduceJob.RelationalFieldColumnName != null) { var sqlBulkCopyColumnMappings = new List <SqlBulkCopyColumnMapping>(); sqlBulkCopyColumnMappings.Add(new SqlBulkCopyColumnMapping("DocumentArtifactID", "DocumentArtifactID")); sqlBulkCopyColumnMappings.Add(new SqlBulkCopyColumnMapping(reproduceJob.RelationalFieldColumnName, "RelationalGroup")); columnMappingsHoldingTable = sqlBulkCopyColumnMappings; } else { columnMappingsHoldingTable = await GenerateColumnMappingsHoldingTableAsync(new List <string> { "DocumentArtifactID" }); } return(await _artifactQueries.AddDocumentsToHoldingTableAsync(AgentHelper.GetServicesManager(), AgentHelper.GetDBContext(workspaceArtifactId), _utilityQueryHelper, ExecutionIdentity.CurrentUser, WorkspaceArtifactId, reproduceJob.SavedSearchArtifactId, SavedSearchHoldingTable, columnMappingsHoldingTable)); }
private async Task <int> ReproduceAcrossRelationalGroup(ReproduceManagerQueueRecord reproduceManagerQueueRecord, MarkupUtilityReproduceJob reproduceJob, int codeTypeId, int markupSetRedactionCodeArtifactId, int markupSetAnnotationCodeArtifactId) { var dataTable = await RetrieveRelationalGroupsAsync(reproduceManagerQueueRecord.WorkspaceArtifactId); //create a new record for each relational group var insertTasks = new List <Task>(); foreach (DataRow row in dataTable.Rows) { var relationalGroup = (string)row["RelationalGroup"]; insertTasks.Add(InsertRecordIntoToReproduceWorkerQueueAsync( reproduceManagerQueueRecord.WorkspaceArtifactId, -1, -1, -1, reproduceJob.ArtifactId, reproduceManagerQueueRecord.ResourceGroupId, codeTypeId, markupSetRedactionCodeArtifactId, markupSetAnnotationCodeArtifactId, reproduceJob.RelationalFieldColumnName, reproduceJob.HasAutoRedactionsFieldColumnName, relationalGroup)); } await Task.WhenAll(insertTasks.ToArray()); return(insertTasks.Count); }
private async Task <int> ReproduceAcrossDocumentSet(ReproduceManagerQueueRecord reproduceManagerQueueRecord, MarkupUtilityReproduceJob reproduceJob, int min, int max, int codeTypeId, int markupSetRedactionCodeArtifactId, int markupSetAnnotationCodeArtifactId) { //create Worker records splitting the records var insertTasks = new List <Task>(); for (var i = min; i <= max; i = i + Constant.Sizes.ReproduceJobInsertBatchSize) { insertTasks.Add(InsertRecordIntoToReproduceWorkerQueueAsync( reproduceManagerQueueRecord.WorkspaceArtifactId, i, i + Constant.Sizes.ReproduceJobInsertBatchSize - 1, reproduceJob.DestinationMarkupSetArtifactId, reproduceJob.ArtifactId, reproduceManagerQueueRecord.ResourceGroupId, codeTypeId, markupSetRedactionCodeArtifactId, markupSetAnnotationCodeArtifactId, null, reproduceJob.HasAutoRedactionsFieldColumnName, null)); } await Task.WhenAll(insertTasks.ToArray()); return(insertTasks.Count); }
public async Task ProcessRecordsAsync(ReproduceManagerQueueRecord reproduceManagerQueueRecord, MarkupUtilityReproduceJob reproduceJob) { RaiseMessage($"Processing record(s). [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]"); try { //Update status of the Job to Manager In Progress await UpdateStatusFieldAsync(reproduceManagerQueueRecord.ReproduceJobArtifactId, Constant.Status.Job.IN_PROGRESS_MANAGER); //Flush the Details of the Job await UpdateDetailsFieldAsync(reproduceManagerQueueRecord.ReproduceJobArtifactId, string.Empty); var includeRelationalGroup = reproduceJob.RelationalField > 0; if (reproduceJob.HasAutoRedactionsField > 0) { var columnName = await RetrieveDocumentColumnAsync(reproduceManagerQueueRecord.WorkspaceArtifactId, reproduceJob.HasAutoRedactionsField); reproduceJob.HasAutoRedactionsFieldColumnName = columnName; } //Create holding tables if (includeRelationalGroup) { await CreateSavedSearchHoldingTableAsync(reproduceManagerQueueRecord.WorkspaceArtifactId, true); var relationalColumnName = await RetrieveDocumentColumnAsync(reproduceManagerQueueRecord.WorkspaceArtifactId, reproduceJob.RelationalField); reproduceJob.RelationalFieldColumnName = relationalColumnName; } else { await CreateSavedSearchHoldingTableAsync(reproduceManagerQueueRecord.WorkspaceArtifactId, false); } await CreateRedactionsHoldingTableAsync(reproduceManagerQueueRecord.WorkspaceArtifactId, includeRelationalGroup); //Add Documents from Saved Search in the Export Worker Queue var isSuccessful = await AddDocumentsToSavedSearchHoldingTableAsync(reproduceManagerQueueRecord.WorkspaceArtifactId, reproduceJob); var jobIsComplete = false; var jobMessage = ""; if (isSuccessful) { var dataTable = await RetrieveMinMaxIdAsync(reproduceManagerQueueRecord.WorkspaceArtifactId); var min = (int)dataTable.Rows[0][0]; var max = (int)dataTable.Rows[0][1]; var tasks = new List <Task <int> >(); for (var i = min; i <= max; i = i + Constant.Sizes.ReproduceJobBatchSize) { //sql between has inclusive end tasks.Add(AddDocumentsToRedactionsHoldingTableAsync(reproduceManagerQueueRecord.WorkspaceArtifactId, reproduceJob.SourceMarkupSetArtifactId, reproduceJob.DestinationMarkupSetArtifactId, i, i + Constant.Sizes.ReproduceJobBatchSize - 1, includeRelationalGroup)); } var results = await Task.WhenAll(tasks.ToArray()); var existRedactions = Array.Exists(results, r => r > 0); if (existRedactions) { var result = await RetrieveZCodesAsync(reproduceManagerQueueRecord.WorkspaceArtifactId, reproduceJob.DestinationMarkupSetArtifactId); var codeTypeId = (int)result.Rows[0]["CodeTypeID"]; var markupSetRedactionCodeArtifactId = (int)result.Rows[0]["RedactionCodeArtifactID"]; var markupSetAnnotationCodeArtifactId = (int)result.Rows[0]["AnnotationCodeArtifactID"]; int count; if (reproduceJob.RelationalField > 0) { count = await ReproduceAcrossRelationalGroup(reproduceManagerQueueRecord, reproduceJob, codeTypeId, markupSetRedactionCodeArtifactId, markupSetAnnotationCodeArtifactId); } else { count = await ReproduceAcrossDocumentSet(reproduceManagerQueueRecord, reproduceJob, min, max, codeTypeId, markupSetRedactionCodeArtifactId, markupSetAnnotationCodeArtifactId); } if (count == 0) { jobIsComplete = true; jobMessage = "No redactions found for selected criteria."; } } else { jobIsComplete = true; jobMessage = "No redactions found for selected criteria."; } } else { jobIsComplete = true; jobMessage = "No Documents found in the selected Saved Search."; } if (jobIsComplete) { var tasks = new List <Task> { UpdateStatusFieldAsync(reproduceManagerQueueRecord.ReproduceJobArtifactId, Constant.Status.Job.COMPLETED), UpdateDetailsFieldAsync(reproduceManagerQueueRecord.ReproduceJobArtifactId, jobMessage), QueryHelper.DropTableAsync(AgentHelper.GetDBContext(reproduceManagerQueueRecord.WorkspaceArtifactId), RedactionsHoldingTable), QueryHelper.DropTableAsync(AgentHelper.GetDBContext(reproduceManagerQueueRecord.WorkspaceArtifactId), SavedSearchHoldingTable) }; //Update status of the Export Job to Completed //Update Details of the Export Job indicating no Documents in Saved Search await Task.WhenAll(tasks); } else { //Update status of the Export Job to Completed - Manager await UpdateStatusFieldAsync(reproduceManagerQueueRecord.ReproduceJobArtifactId, Constant.Status.Job.COMPLETED_MANAGER); //Flush the Details of the Export Job await UpdateDetailsFieldAsync(reproduceManagerQueueRecord.ReproduceJobArtifactId, string.Empty); } //delete import job from queue RaiseMessage($"Removing record(s) from the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]"); await FinishAsync(); await DeleteSavedSearchHoldingTableAsync(); await DeleteRedactionsHoldingTableAsync(); RaiseMessage($"Removed record(s) from the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]"); RaiseMessage($"Processed record(s). [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]"); } catch (Exception ex) { //Update status of the Export Job to Error await UpdateStatusFieldAsync(reproduceManagerQueueRecord.ReproduceJobArtifactId, Constant.Status.Job.ERROR); //Update Details of the Export Job await UpdateDetailsFieldAsync(reproduceManagerQueueRecord.ReproduceJobArtifactId, ex.ToString()); //log error await LogErrorAsync(ex); //delete import job from queue RaiseMessage($"Removing record(s) from the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]"); await FinishAsync(); await DeleteSavedSearchHoldingTableAsync(); await DeleteRedactionsHoldingTableAsync(); RaiseMessage($"Removed record(s) from the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]"); RaiseMessage($"Processed record(s). [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]"); } }