Пример #1
0
        public override async Task ExecuteAsync()
        {
            try
            {
                //Check for jobs which stopped unexpectedly on this agent thread
                RaiseMessage($"Resetting records which failed. [Table = {QueueTable}]");
                await ResetUnfishedJobsAsync(AgentHelper.GetDBContext(-1));

                //Retrieve the next record to work on
                RaiseMessage($"Retrieving next record(s) in the queue. [Table = {QueueTable}]");
                var delimitedListOfResourceGroupIds = GetCommaDelimitedListOfResourceIds(AgentResourceGroupIds);

                if (delimitedListOfResourceGroupIds != string.Empty)
                {
                    var next = await RetrieveNextAsync(delimitedListOfResourceGroupIds);

                    if (TableIsNotEmpty(next))
                    {
                        var firstExportWorkerQueueRecord = new ExportWorkerQueueRecord(next.Rows[0]);
                        WorkspaceArtifactId = firstExportWorkerQueueRecord.WorkspaceArtifactId;

                        //Retrieve export job
                        _markupUtilityExportJob = await _artifactQueries.RetrieveExportJobAsync(_serviceMgr, ExecutionIdentity.CurrentUser, WorkspaceArtifactId, firstExportWorkerQueueRecord.ExportJobArtifactId);

                        //Create Export Worker Job holding table
                        await CreateExportWorkerHoldingTableAsync();

                        //Set the status of the Export Job to In Progess - Export Worker
                        await UpdateStatusFieldAsync(firstExportWorkerQueueRecord.ExportJobArtifactId, Constant.Status.Job.IN_PROGRESS_WORKER);

                        //Process document redactions
                        await ProcessDocumentRedactionsAsync(next, firstExportWorkerQueueRecord.WorkspaceArtifactId, firstExportWorkerQueueRecord.ExportJobArtifactId);

                        RaiseMessage($"Retrieved record(s) in the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
                    }
                    else
                    {
                        RaiseMessage("No records in the queue for this resource pool.");
                    }
                }
                else
                {
                    RaiseMessage(Constant.AgentRaiseMessages.AGENT_SERVER_NOT_PART_OF_ANY_RESOURCE_POOL);
                }
            }
            catch (Exception ex)
            {
                RaiseMessage($"Logging error.");
                await LogErrorAsync(ex);
            }
            finally
            {
                //Remove the Export Worker Holding table
                await FinishHoldingTableAsync();

                //Remove records from Batch table and Delete the Export Worker Batch table
                await FinishAsync();
            }
        }
Пример #2
0
        public void Constructor_ReceivesTable_Initializes()
        {
            // Arrange
            var table = GetTable();

            // Act
            var record = new ExportWorkerQueueRecord(table.Rows[0]);

            // Assert
            Assert.AreEqual(2345678, record.WorkspaceArtifactId);
            Assert.AreEqual(1, record.QueueRowId);
            Assert.AreEqual(3456789, record.ArtifactId);
            Assert.AreEqual(2, record.Priority);
        }
Пример #3
0
        public async Task ProcessUsers(Int32 workspaceArtifactID, Int32 jobArtifactID, DataTable batchDataTable)
        {
            try
            {
                RaiseAndLogDebugMessage("Processing all users in export worker batch.");

                foreach (DataRow currentDataRow in batchDataTable.AsEnumerable())
                {
                    ExportWorkerQueueRecord exportWorkerQueueRecord = new ExportWorkerQueueRecord(currentDataRow);
                    await ProcessSingleUserAsync(workspaceArtifactID, jobArtifactID, exportWorkerQueueRecord);
                }

                RaiseAndLogDebugMessage("Processed all users in export worker batch.");
            }
            catch (Exception ex)
            {
                throw new AdminMigrationUtilityException(Constant.ErrorMessages.ProcessAllUsersInExportWorkerBatchError, ex);
            }
        }
Пример #4
0
        public async Task ProcessDocumentRedactionsSingleAsync(ExportWorkerQueueRecord exportWorkerQueueRecord)
        {
            _errorContext = $"[WorkspaceArtifactId = {WorkspaceArtifactId}, ExportJobArtifactId = {exportWorkerQueueRecord.ExportJobArtifactId}]";
            RaiseMessage($"Processing record(s). [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}, DocumentArtifactId = {exportWorkerQueueRecord.DocumentArtifactId}, MarkupSetArtifactId = {exportWorkerQueueRecord.MarkupSetArtifactId}, MarkupSubTypes = {exportWorkerQueueRecord.MarkupSubType}]");

            //Check if Document exists
            await _artifactQueries.VerifyIfDocumentExistsAsync(_serviceMgr, ExecutionIdentity.CurrentUser, WorkspaceArtifactId, exportWorkerQueueRecord.DocumentArtifactId);

            //Retrieve the Column Name of the Document Identifier Field
            var documentIdentifierColumnName = await QueryHelper.GetDocumentIdentifierFieldColumnNameAsync(AgentHelper.GetDBContext(WorkspaceArtifactId));

            //Retrieve Redactions for the selected Documents from the Workspace, store in DataTable
            var dtDocumentRedactions = await QueryHelper.RetrieveRedactionsForDocumentAsync(AgentHelper.GetDBContext(exportWorkerQueueRecord.WorkspaceArtifactId), exportWorkerQueueRecord.WorkspaceArtifactId, exportWorkerQueueRecord.ExportJobArtifactId, exportWorkerQueueRecord.DocumentArtifactId, exportWorkerQueueRecord.MarkupSetArtifactId, documentIdentifierColumnName, exportWorkerQueueRecord.MarkupSubType);

            //Write records to holding table in EDDS
            await AddRedactionsFromHoldingToResultsTableAsync(dtDocumentRedactions, ExportWorkerHoldingTable);

            RaiseMessage($"Processed record(s). [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}, DocumentArtifactId = {exportWorkerQueueRecord.DocumentArtifactId}, MarkupSetArtifactId = {exportWorkerQueueRecord.MarkupSetArtifactId}, MarkupSubTypes = {exportWorkerQueueRecord.MarkupSubType}]");
        }
Пример #5
0
        public void Constructor_ReceivesTable_Initializes()
        {
            // Arrange
            var table = GetTable();

            // Act
            var record = new ExportWorkerQueueRecord(table.Rows[0]);

            // Assert
            Assert.AreEqual(2345678, record.WorkspaceArtifactId);
            Assert.AreEqual(1, record.RecordId);
            Assert.AreEqual(3456789, record.ExportJobArtifactId);
            Assert.AreEqual(3, record.DocumentArtifactId);
            Assert.AreEqual(4, record.MarkupSetArtifactId);
            Assert.AreEqual(5, record.QueueStatus);
            Assert.AreEqual(6, record.AgentId);
            Assert.AreEqual("7", record.MarkupSubType);
            Assert.AreEqual(8, record.ResourceGroupId);
        }
Пример #6
0
        private async Task ProcessDocumentRedactionsAsync(DataTable dtExportJobWorkerQueueRecords, int workspaceArtifactId, int exportJobArtifactId)
        {
            try
            {
                foreach (DataRow row in dtExportJobWorkerQueueRecords.Rows)
                {
                    var exportWorkerQueueRecord = new ExportWorkerQueueRecord(row);

                    //Sets the workspaceArtifactID and RecordID so the agent will have access to them in case of an exception
                    WorkspaceArtifactId = exportWorkerQueueRecord.WorkspaceArtifactId;
                    RecordId            = exportWorkerQueueRecord.RecordId;
                    _errorContext       = $"[WorkspaceArtifactId = {WorkspaceArtifactId}, Id = {RecordId}]";

                    //Retrieve the Document redactions
                    await ProcessDocumentRedactionsSingleAsync(exportWorkerQueueRecord);
                }

                //Bulk copy from Holding table to Export Results table
                await AddRecordsToExportResultsAsync(ExportWorkerHoldingTable);

                //Remove the Export Worker Holding table
                await FinishHoldingTableAsync();

                //Remove records from the Export Worker Batch table
                await DeleteRecordsFromExportWorkerBatchTableAsync();

                //Check if no more records exist in the Export Worker queue, if not, proceed with creating CSV for attachment to Export Job
                await VerifyIfExportWorkerQueueContainsRecordsForJobAsync(workspaceArtifactId, exportJobArtifactId);
            }
            catch (Exception ex)
            {
                //Set the status of the Export Job to Error
                await UpdateStatusFieldAsync(_markupUtilityExportJob.ArtifactId, Constant.Status.Job.ERROR);
                await UpdateDetailsFieldAsync(_markupUtilityExportJob.ArtifactId, await ConstructDetailsExceptionMessageAsync(ex));

                RaiseMessage($"Logging error.");
                await LogErrorAsync(ex);

                //Remove the Export Worker Holding table
                await FinishHoldingTableAsync();
            }
        }
Пример #7
0
        public async Task <String> RetrieveExportJobStatusAsync(ExportWorkerQueueRecord exportWorkerQueueRecord)
        {
            try
            {
                RaiseAndLogDebugMessage("Retrieving the status of export job.");

                String exportJobStatus = await ArtifactQueries.RetrieveRdoTextFieldValueAsync(
                    rsapiApiOptions : RsapiApiOptions,
                    workspaceArtifactId : exportWorkerQueueRecord.WorkspaceArtifactId,
                    rdoRepository : RsapiRepositoryGroup.RdoRepository,
                    rdoArtifactId : exportWorkerQueueRecord.ExportJobArtifactId,
                    textFieldGuid : Constant.Guids.Field.ExportUtilityJob.Status);

                RaiseAndLogDebugMessage("Retrieved the status of export job.");

                return(exportJobStatus);
            }
            catch (Exception ex)
            {
                throw new AdminMigrationUtilityException(Constant.ErrorMessages.RetrieveExportJobStatusError, ex);
            }
        }
Пример #8
0
        public async Task InsertUserDataIntoExportWorkerResultsTableAsync(ExportWorkerQueueRecord exportWorkerQueueRecord, UserAdminObject userAdminObject)
        {
            try
            {
                //deserialize user admin object
                String metadata = await SerializationHelper.SerializeAdminObjectAsync(userAdminObject);

                //insert into export worker results table
                await SqlQueryHelper.InsertRowIntoExportWorkerResultsTableAsync(
                    eddsDbContext : AgentHelper.GetDBContext(-1),
                    exportWorkerResultTableName : exportWorkerQueueRecord.ResultTableName,
                    workspaceArtifactId : WorkspaceArtifactId,
                    exportJobArtifactId : exportWorkerQueueRecord.ExportJobArtifactId,
                    objectType : exportWorkerQueueRecord.ObjectType,
                    workspaceResourceGroupArtifactId : exportWorkerQueueRecord.WorkspaceResourceGroupArtifactId,
                    queueStatus : Constant.Status.Queue.NOT_STARTED,
                    metadata : metadata,
                    timeStampUtc : DateTime.UtcNow);
            }
            catch (Exception ex)
            {
                throw new AdminMigrationUtilityException(Constant.ErrorMessages.InsertUserDataIntoExportWorkerResultsTableError, ex);
            }
        }
Пример #9
0
        public override async Task ExecuteAsync()
        {
            try
            {
                Boolean isOffHours = await IsOffHoursAsync(ProcessedOnDateTime);

                if (!isOffHours)
                {
                    RaiseAndLogDebugMessage($"Current time is not between {OffHoursStartTime} and {OffHoursEndTime}. Agent execution skipped.");
                }
                else
                {
                    //Check for jobs which stopped unexpectedly on this agent thread and reset them
                    await ResetUnfinishedExportWorkerJobsAsync();

                    //Retrieve the next record to work on
                    String commaDelimitedListOfResourceIds = GetCommaDelimitedListOfResourceIds(AgentResourceGroupIds);
                    if (commaDelimitedListOfResourceIds == String.Empty)
                    {
                        RaiseAndLogDebugMessage("This agent server is not part of any resource pools.  Agent execution skipped.");
                    }
                    else
                    {
                        DataTable batchDataTable = await RetrieveNextBatchOfRecordsInExportWorkerQueueTableAsync(commaDelimitedListOfResourceIds);

                        if (IsTableEmpty(batchDataTable))
                        {
                            RaiseAndLogDebugMessage("No records in the export worker queue table for this resource pool.");
                        }
                        else
                        {
                            // Use first record to check the status of the Job RDO.  All records are from the same job
                            var workerQueueRecord = new ExportWorkerQueueRecord(batchDataTable.Rows[0]);

                            SetJobProperties(workerQueueRecord);

                            var    objectType      = (Constant.Enums.SupportedObjects)Enum.Parse(typeof(Constant.Enums.SupportedObjects), workerQueueRecord.ObjectType);
                            String exportJobStatus = await RetrieveExportJobStatusAsync(workerQueueRecord);

                            if (exportJobStatus != Constant.Status.Job.IN_PROGRESS_WORKER)
                            {
                                //update export job status to in progress
                                await UpdateExportJobStatus(workerQueueRecord.WorkspaceArtifactId, workerQueueRecord.ExportJobArtifactId, Constant.Status.Job.IN_PROGRESS_WORKER);
                            }

                            //Create results table name if it does not already exists
                            await CreateExportWorkerResultsTableIfItDoesNotAlreadyExistsAsync(workerQueueRecord);

                            await ProcessBatchRecordsAsync(batchDataTable, workerQueueRecord.WorkspaceArtifactId, workerQueueRecord.ExportJobArtifactId, objectType);
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                throw new AdminMigrationUtilityException(Constant.ErrorMessages.ExportWorkerAgentError, ex);
            }
            finally
            {
                //drop batch table
                await DropExportWorkerBatchTableAsync();
            }
        }
Пример #10
0
        public async Task ProcessSingleUserAsync(Int32 workspaceArtifactID, Int32 jobArtifactID, ExportWorkerQueueRecord exportWorkerQueueRecord)
        {
            try
            {
                RaiseAndLogDebugMessage($"Processing a single user in export worker batch. [{nameof(exportWorkerQueueRecord.ArtifactId)}] = {exportWorkerQueueRecord.ArtifactId}");

                kCura.Relativity.Client.DTOs.User userRDO = await ArtifactQueries.RetrieveUserAsync(RsapiApiOptions, RsapiRepositoryGroup.UserRepository, exportWorkerQueueRecord.ArtifactId);

                //query user Auth data
                LoginProfile userLoginProfile = await AuthenticationHelper.RetrieveExistingUserLoginProfileAsync(userRDO.ArtifactID);

                //query keywords and notes for user
                KeywordsNotesModel userKeywordsNotesModel = await SqlQueryHelper.RetrieveKeywordsAndNotesForUserAsync(
                    eddsDbContext : AgentHelper.GetDBContext(-1),
                    userArtifactId : userRDO.ArtifactID);

                //query groups user is part of
                IEnumerable <String> userGroupNameList = await QueryGroupsNamesUserIsPartOfAsync(userRDO);

                UserAdminObject userAdminObject = await ConvertUserResultToUserAdminObjectAsync(userRDO);

                //add keywords and notes, auth and groups data to UserAdminObject
                await AddKeywordsNotesAuthInfoAndGroupsToUserAdminObjectAsync(userAdminObject, userKeywordsNotesModel, userGroupNameList, userLoginProfile);

                //insert user data info into the export worker results table
                await InsertUserDataIntoExportWorkerResultsTableAsync(exportWorkerQueueRecord, userAdminObject);

                RaiseAndLogDebugMessage($"Processed a single user in export worker batch. [{nameof(exportWorkerQueueRecord.ArtifactId)}] = {exportWorkerQueueRecord.ArtifactId}");
            }
            catch (Exception ex)
            {
                //create export job error record
                String details = ExceptionMessageFormatter.GetExceptionMessageIncludingAllInnerExceptions(ex);
                await CreateExportJobErrorRecordAsync(workspaceArtifactID, jobArtifactID, exportWorkerQueueRecord.ObjectType, Constant.Status.JobErrors.ERROR, details);
            }
        }
Пример #11
0
        public async Task CreateExportWorkerResultsTableIfItDoesNotAlreadyExistsAsync(ExportWorkerQueueRecord workerQueueRecord)
        {
            try
            {
                RaiseAndLogDebugMessage("Creating the export worker results table if it doesn't already exists.");

                await SqlQueryHelper.CreateExportWorkerResultsTableIfItDoesNotAlreadyExistsAsync(
                    eddsDbContext : AgentHelper.GetDBContext(-1),
                    tableName : workerQueueRecord.ResultTableName);

                RaiseAndLogDebugMessage("Created the export worker results table if it doesn't already exists.");
            }
            catch (Exception ex)
            {
                throw new AdminMigrationUtilityException(Constant.ErrorMessages.CreateExportWorkerResultsTableIfItDoesNotAlreadyExistsError, ex);
            }
        }
Пример #12
0
 public void SetJobProperties(ExportWorkerQueueRecord exportWorkerQueueRecord)
 {
     WorkspaceArtifactId = exportWorkerQueueRecord.WorkspaceArtifactId;
     TableRowId          = exportWorkerQueueRecord.QueueRowId;
 }