Ejemplo n.º 1
0
        private async Task ProcessRedactionsAsync(ReproduceWorkerQueueRecord reproduceWorkerQueueRecord, MarkupUtilityReproduceJob markupUtilityReproduceJob)
        {
            try
            {
                //Bulk insert the redactions for the document batch
                DataTable dataTable;
                var       relationalGroup = markupUtilityReproduceJob.RelationalField > 0;
                if (relationalGroup)
                {
                    dataTable = await ReproduceAcrossRelationalGroup(reproduceWorkerQueueRecord, markupUtilityReproduceJob);
                }
                else
                {
                    dataTable = await ReproduceAcrossDocumentSet(reproduceWorkerQueueRecord, markupUtilityReproduceJob);
                }

                //Get the ids from the result and add to audit and history tables. Ideally, this should be in the same db transaction as above....
                await CreateHistoryAndAuditRecords(reproduceWorkerQueueRecord, dataTable, markupUtilityReproduceJob, relationalGroup);
                await Finish(reproduceWorkerQueueRecord, Constant.Status.Job.COMPLETED);
            }
            catch (Exception ex)
            {
                //Set the status of the Export Job to Error
                await UpdateStatusFieldAsync(reproduceWorkerQueueRecord.ReproduceJobArtifactId, Constant.Status.Job.ERROR);
                await UpdateDetailsFieldAsync(reproduceWorkerQueueRecord.ReproduceJobArtifactId, await ConstructDetailsExceptionMessageAsync(ex));

                RaiseMessage($"Logging error.");
                await LogErrorAsync(ex);
            }
        }
Ejemplo n.º 2
0
 private async Task <MarkupUtilityReproduceJob> RetrieveReproduceJobAsync(ReproduceWorkerQueueRecord reproduceManagerQueueRecord)
 {
     return(await _artifactQueries.RetrieveReproduceJobAsync(
                AgentHelper.GetServicesManager(),
                ExecutionIdentity.CurrentUser,
                WorkspaceArtifactId,
                reproduceManagerQueueRecord.ReproduceJobArtifactId));
 }
Ejemplo n.º 3
0
        public override async Task ExecuteAsync()
        {
            try
            {
                //Check for jobs which stopped unexpectedly on this agent thread
                RaiseMessage($"Resetting records which failed. [Table = {QueueTable}]");
                await ResetUnfishedJobsAsync(AgentHelper.GetDBContext(-1));

                //Retrieve the next record to work on
                RaiseMessage($"Retrieving next record(s) in the queue. [Table = {QueueTable}]");
                var delimitedListOfResourceGroupIds = GetCommaDelimitedListOfResourceIds(AgentResourceGroupIds);

                if (delimitedListOfResourceGroupIds != string.Empty)
                {
                    var next = await RetrieveNextAsync(delimitedListOfResourceGroupIds);

                    if (TableIsNotEmpty(next))
                    {
                        var workerQueueRecord = new ReproduceWorkerQueueRecord(next.Rows[0]);
                        WorkspaceArtifactId = workerQueueRecord.WorkspaceArtifactId;

                        //Process document redactions
                        var markupUtilityReproduceJob = await RetrieveReproduceJobAsync(workerQueueRecord);

                        //someone cancelled the job or managed to delete it
                        if (markupUtilityReproduceJob == null)
                        {
                            await Finish(workerQueueRecord, null);
                        }
                        else if (markupUtilityReproduceJob.Status == Constant.Status.Job.CANCELREQUESTED || markupUtilityReproduceJob.Status == Constant.Status.Job.CANCELLED)
                        {
                            //you can end up in state Cancel Requestd if you cancel just after the ManagerQueue job completed, in which case update the the status to Cancelled
                            await Finish(workerQueueRecord, Constant.Status.Job.CANCELLED);
                        }
                        else
                        {
                            await UpdateStatusFieldAsync(workerQueueRecord.ReproduceJobArtifactId, Constant.Status.Job.IN_PROGRESS_WORKER);
                            await ProcessRedactionsAsync(workerQueueRecord, markupUtilityReproduceJob);
                        }
                        RaiseMessage($"Retrieved record(s) in the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
                    }
                    else
                    {
                        RaiseMessage("No records in the queue for this resource pool.");
                    }
                }
                else
                {
                    RaiseMessage(Constant.AgentRaiseMessages.AGENT_SERVER_NOT_PART_OF_ANY_RESOURCE_POOL);
                }
            }
            catch (Exception ex)
            {
                RaiseMessage($"Logging error.");
                await LogErrorAsync(ex);
            }
        }
Ejemplo n.º 4
0
        private async Task <IEnumerable <int> > UpdateHasRedactionsOrHighlightsAsync(ReproduceWorkerQueueRecord reproduceWorkerQueueRecord, int destinationMarkupSet, bool relationalGroup)
        {
            RaiseMessage($"Updating documents with Has Redactions/Highligths for worker record id {reproduceWorkerQueueRecord.RecordId}. {_errorContext}");

            Task <int> redactions;
            Task <int> highlights;

            if (relationalGroup)
            {
                redactions = QueryHelper.UpdateHasRedactionsOrHighlightsAsync(
                    AgentHelper.GetDBContext(reproduceWorkerQueueRecord.WorkspaceArtifactId),
                    reproduceWorkerQueueRecord.RedactionCodeTypeId,
                    reproduceWorkerQueueRecord.MarkupSetRedactionCodeArtifactId, Constant.MarkupType.Redaction.VALUE,
                    reproduceWorkerQueueRecord.SavedSearchHoldingTable,
                    reproduceWorkerQueueRecord.RedactionsHoldingTable,
                    destinationMarkupSet, reproduceWorkerQueueRecord.RelationalGroup);

                highlights = QueryHelper.UpdateHasRedactionsOrHighlightsAsync(
                    AgentHelper.GetDBContext(reproduceWorkerQueueRecord.WorkspaceArtifactId),
                    reproduceWorkerQueueRecord.RedactionCodeTypeId,
                    reproduceWorkerQueueRecord.MarkupSetAnnotationCodeArtifactId,
                    Constant.MarkupType.Highlight.VALUE,
                    reproduceWorkerQueueRecord.SavedSearchHoldingTable,
                    reproduceWorkerQueueRecord.RedactionsHoldingTable,
                    destinationMarkupSet, reproduceWorkerQueueRecord.RelationalGroup);
            }
            else
            {
                redactions = QueryHelper.UpdateHasRedactionsOrHighlightsAsync(
                    AgentHelper.GetDBContext(reproduceWorkerQueueRecord.WorkspaceArtifactId),
                    reproduceWorkerQueueRecord.RedactionCodeTypeId,
                    reproduceWorkerQueueRecord.MarkupSetRedactionCodeArtifactId, Constant.MarkupType.Redaction.VALUE,
                    reproduceWorkerQueueRecord.SavedSearchHoldingTable,
                    reproduceWorkerQueueRecord.RedactionsHoldingTable,
                    destinationMarkupSet, reproduceWorkerQueueRecord.DocumentIdStart,
                    reproduceWorkerQueueRecord.DocumentIdEnd);

                highlights = QueryHelper.UpdateHasRedactionsOrHighlightsAsync(
                    AgentHelper.GetDBContext(reproduceWorkerQueueRecord.WorkspaceArtifactId),
                    reproduceWorkerQueueRecord.RedactionCodeTypeId,
                    reproduceWorkerQueueRecord.MarkupSetAnnotationCodeArtifactId,
                    Constant.MarkupType.Highlight.VALUE,
                    reproduceWorkerQueueRecord.SavedSearchHoldingTable,
                    reproduceWorkerQueueRecord.RedactionsHoldingTable,
                    destinationMarkupSet, reproduceWorkerQueueRecord.DocumentIdStart,
                    reproduceWorkerQueueRecord.DocumentIdEnd);
            }

            var ints = await Task.WhenAll(redactions, highlights);

            RaiseMessage($"Updated {string.Join("/", ints)} Redaction/Highlight records with Has Redactions/Highligths for worker record id {reproduceWorkerQueueRecord.RecordId}. {_errorContext}");
            return(ints);
        }
Ejemplo n.º 5
0
        private async Task <DataTable> ReproduceAcrossRelationalGroup(ReproduceWorkerQueueRecord reproduceWorkerQueueRecord, MarkupUtilityReproduceJob markupUtilityReproduceJob)
        {
            var dataTable = await BulkInsertRedactionRecordsForRelationalGroup(reproduceWorkerQueueRecord);

            if (!string.IsNullOrEmpty(reproduceWorkerQueueRecord.HasAutoRedactionsColumn))
            {
                await BulkUpdateHasAutoRedactionsFieldForRelationalGroup(reproduceWorkerQueueRecord);
            }

            await UpdateHasRedactionsOrHighlightsAsync(reproduceWorkerQueueRecord, markupUtilityReproduceJob.DestinationMarkupSetArtifactId, true);

            return(dataTable);
        }
Ejemplo n.º 6
0
        private async Task CleanUpIfLast(ReproduceWorkerQueueRecord reproduceWorkerQueueRecord, string jobStatus)
        {
            _errorContext = $"An error occurred while creating the export file. [WorkspaceArtifactId = {WorkspaceArtifactId}, ReproduceJobArtifactId = {reproduceWorkerQueueRecord.ReproduceJobArtifactId}]";

            var queueCount = await QueryHelper.GetJobWorkerRecordCountAsync(AgentHelper.GetDBContext(-1), WorkspaceArtifactId, reproduceWorkerQueueRecord.ReproduceJobArtifactId, Constant.Tables.ReproduceWorkerQueue, "ReproduceJobArtifactID");

            if (queueCount == 0)
            {
                var tasks = new List <Task>();
                if (jobStatus != null)
                {
                    tasks.Add(UpdateStatusFieldAsync(reproduceWorkerQueueRecord.ReproduceJobArtifactId, jobStatus));
                }
                tasks.Add(QueryHelper.DropTableAsync(AgentHelper.GetDBContext(reproduceWorkerQueueRecord.WorkspaceArtifactId), reproduceWorkerQueueRecord.RedactionsHoldingTable));
                tasks.Add(QueryHelper.DropTableAsync(AgentHelper.GetDBContext(reproduceWorkerQueueRecord.WorkspaceArtifactId), reproduceWorkerQueueRecord.SavedSearchHoldingTable));
                await Task.WhenAll(tasks);
            }
        }
Ejemplo n.º 7
0
        public void ReproduceWorkerQueueRecordInitTest()
        {
            var table = new DataTable();

            table.Columns.Add("WorkspaceArtifactID", typeof(int));
            table.Columns.Add("ID", typeof(int));
            table.Columns.Add("DocumentIDStart", typeof(int));
            table.Columns.Add("DocumentIDEnd", typeof(int));
            table.Columns.Add("SavedSearchHoldingTable", typeof(string));
            table.Columns.Add("RedactionsHoldingTable", typeof(string));
            table.Columns.Add("RedactionCodeTypeID", typeof(int));
            table.Columns.Add("QueueStatus", typeof(int));
            table.Columns.Add("MarkupSetRedactionCodeArtifactID", typeof(int));
            table.Columns.Add("MarkupSetAnnotationCodeArtifactID", typeof(int));
            table.Columns.Add("ReproduceJobArtifactID", typeof(int));
            table.Columns.Add("ResourceGroupID", typeof(int));
            table.Columns.Add("RelationalGroupColumn", typeof(string));
            table.Columns.Add("HasAutoRedactionsColumn", typeof(string));
            table.Columns.Add("RelationalGroup", typeof(string));
            table.Rows.Add(1, 2, 3, 4, "5", "6", 7, 8, 9, 10, 11, 12, "13", "14", "15");

            var reproduceManagerQueueRecord = new ReproduceWorkerQueueRecord(table.Rows[0]);

            Assert.AreEqual(1, reproduceManagerQueueRecord.WorkspaceArtifactId);
            Assert.AreEqual(2, reproduceManagerQueueRecord.RecordId);
            Assert.AreEqual(3, reproduceManagerQueueRecord.DocumentIdStart);
            Assert.AreEqual(4, reproduceManagerQueueRecord.DocumentIdEnd);
            Assert.AreEqual("5", reproduceManagerQueueRecord.SavedSearchHoldingTable);
            Assert.AreEqual("6", reproduceManagerQueueRecord.RedactionsHoldingTable);
            Assert.AreEqual(7, reproduceManagerQueueRecord.RedactionCodeTypeId);
            Assert.AreEqual(8, reproduceManagerQueueRecord.QueueStatus);
            Assert.AreEqual(9, reproduceManagerQueueRecord.MarkupSetRedactionCodeArtifactId);
            Assert.AreEqual(10, reproduceManagerQueueRecord.MarkupSetAnnotationCodeArtifactId);
            Assert.AreEqual(11, reproduceManagerQueueRecord.ReproduceJobArtifactId);
            Assert.AreEqual(12, reproduceManagerQueueRecord.ResourceGroupId);

            Assert.Throws <ArgumentNullException>(delegate { new ReproduceWorkerQueueRecord(null); });
        }
Ejemplo n.º 8
0
        private async Task CreateHistoryAndAuditRecords(ReproduceWorkerQueueRecord reproduceWorkerQueueRecord, DataTable dataTable, MarkupUtilityReproduceJob markupUtilityReproduceJob, bool relationalGroup)
        {
            var details = Constant.Status.History.Details.REPRODUCED_REDACTION_IN_DOCUMENT_SET;

            if (relationalGroup)
            {
                details = Constant.Status.History.Details.REPRODUCED_REDACTION_IN_RELATIONAL_GROUP;
            }

            foreach (DataRow row in dataTable.Rows)
            {
                var redactionId = (int)row["ID"];
                var table       = await RetrieveRedactionInfoAsync(reproduceWorkerQueueRecord, redactionId);

                var dataRow    = table.Rows[0];
                var identifier = (string)dataRow["Identifier"];
                var pageNumber = (int)dataRow["Order"] + 1;
                var markupUtilityHistoryRecordAsync = _artifactQueries.CreateMarkupUtilityHistoryRecordAsync(
                    AgentHelper.GetServicesManager(),
                    ExecutionIdentity.CurrentUser,
                    WorkspaceArtifactId,
                    -1,
                    identifier,
                    pageNumber,
                    Constant.ImportJobType.REPRODUCE,
                    null,
                    Constant.Status.History.COMPLETED,
                    details,
                    ToStringRedactionData(dataRow),
                    redactionId,
                    reproduceWorkerQueueRecord.ReproduceJobArtifactId);

                var auditRecordAsync = CreateAuditRecordAsync(dataRow, markupUtilityReproduceJob, (int)dataRow["DocumentArtifactID"]);

                await Task.WhenAll(markupUtilityHistoryRecordAsync, auditRecordAsync);
            }
        }
Ejemplo n.º 9
0
 private async Task <DataTable> RetrieveRedactionInfoAsync(ReproduceWorkerQueueRecord reproduceWorkerQueueRecord, int redactionId)
 {
     return(await QueryHelper.RetrieveRedactionInfoAsync(AgentHelper.GetDBContext(reproduceWorkerQueueRecord.WorkspaceArtifactId), redactionId));
 }
Ejemplo n.º 10
0
 private async Task <DataTable> BulkUpdateHasAutoRedactionsForDocumentRange(ReproduceWorkerQueueRecord reproduceWorkerQueueRecord)
 {
     RaiseMessage($"Bulk update has auto redactions fields for worker record id {reproduceWorkerQueueRecord.RecordId}. {_errorContext}");
     return(await QueryHelper.BulkUpdateHasAutoRedactionsForDocumentRange(AgentHelper.GetDBContext(reproduceWorkerQueueRecord.WorkspaceArtifactId), reproduceWorkerQueueRecord.SavedSearchHoldingTable, reproduceWorkerQueueRecord.RedactionsHoldingTable, reproduceWorkerQueueRecord.DocumentIdStart, reproduceWorkerQueueRecord.DocumentIdEnd, reproduceWorkerQueueRecord.HasAutoRedactionsColumn));
 }
Ejemplo n.º 11
0
 private async Task <DataTable> BulkInsertRedactionRecordsForDocumentRange(ReproduceWorkerQueueRecord reproduceWorkerQueueRecord)
 {
     RaiseMessage($"Bulk inserting redaction records for worker record id {reproduceWorkerQueueRecord.RecordId}. {_errorContext}");
     return(await QueryHelper.BulkInsertRedactionRecordsForDocumentRange(AgentHelper.GetDBContext(reproduceWorkerQueueRecord.WorkspaceArtifactId), reproduceWorkerQueueRecord.SavedSearchHoldingTable, reproduceWorkerQueueRecord.RedactionsHoldingTable, reproduceWorkerQueueRecord.DocumentIdStart, reproduceWorkerQueueRecord.DocumentIdEnd));
 }
Ejemplo n.º 12
0
 private async Task <DataTable> BulkUpdateHasAutoRedactionsFieldForRelationalGroup(ReproduceWorkerQueueRecord reproduceWorkerQueueRecord)
 {
     RaiseMessage($"Bulk updating document hasAutoRedactions field for worker record id {reproduceWorkerQueueRecord.RecordId}. {_errorContext}");
     return(await QueryHelper.BulkUpdateHasAutoRedactionsFieldForRelationalGroup(AgentHelper.GetDBContext(reproduceWorkerQueueRecord.WorkspaceArtifactId), reproduceWorkerQueueRecord.SavedSearchHoldingTable, reproduceWorkerQueueRecord.RedactionsHoldingTable, reproduceWorkerQueueRecord.RelationalGroup, reproduceWorkerQueueRecord.HasAutoRedactionsColumn));
 }
Ejemplo n.º 13
0
 private async Task Finish(ReproduceWorkerQueueRecord reproduceWorkerQueueRecord, string jobStatus)
 {
     await DeleteRecordFromReproduceWorkerQueueAsync(reproduceWorkerQueueRecord.RecordId);
     await CleanUpIfLast(reproduceWorkerQueueRecord, jobStatus);
 }