public async Task InsertUserArtifactIDBatchIntoExportWorkerQueueTableAsync(ExportManagerQueueRecord exportManagerQueueRecord, String exportResultsTableName, QueryResultSet <kCura.Relativity.Client.DTOs.User> users) { var currentTime = DateTime.UtcNow; var dt = new DataTable(); dt.Columns.Add(Constant.Sql.ColumnsNames.ExportWorkerQueue.WorkspaceArtifactId, typeof(Int32)); dt.Columns.Add(Constant.Sql.ColumnsNames.ExportWorkerQueue.ExportJobArtifactId, typeof(Int32)); dt.Columns.Add(Constant.Sql.ColumnsNames.ExportWorkerQueue.ObjectType, typeof(String)); dt.Columns.Add(Constant.Sql.ColumnsNames.ExportWorkerQueue.Priority, typeof(Int32)); dt.Columns.Add(Constant.Sql.ColumnsNames.ExportWorkerQueue.WorkspaceResourceGroupArtifactId, typeof(Int32)); dt.Columns.Add(Constant.Sql.ColumnsNames.ExportWorkerQueue.QueueStatus, typeof(Int32)); dt.Columns.Add(Constant.Sql.ColumnsNames.ExportWorkerQueue.ArtifactId, typeof(Int32)); dt.Columns.Add(Constant.Sql.ColumnsNames.ExportWorkerQueue.ResultsTableName, typeof(String)); dt.Columns.Add(Constant.Sql.ColumnsNames.ExportWorkerQueue.MetaData, typeof(String)); dt.Columns.Add(Constant.Sql.ColumnsNames.ExportWorkerQueue.TimeStampUtc, typeof(DateTime)); try { foreach (var user in users.Results) { dt.Rows.Add(exportManagerQueueRecord.WorkspaceArtifactId, exportManagerQueueRecord.ExportJobArtifactId, exportManagerQueueRecord.ObjectType, exportManagerQueueRecord.Priority, exportManagerQueueRecord.WorkspaceResourceGroupArtifactId, Constant.Status.Queue.NOT_STARTED, user.Artifact.ArtifactID, exportResultsTableName, String.Empty, currentTime); } await SqlQueryHelper.BulkInsertIntoTableAsync(AgentHelper.GetDBContext(-1), dt, Constant.Tables.ExportWorkerQueue, Constant.BatchSizes.ExportManagerIntoWorkerQueue); } catch (Exception ex) { throw new AdminMigrationUtilityException(Constant.ErrorMessages.UnableToInsertUserArtifactIDsIntoWorkerQueue, ex); } }
public async Task AttachFileToExportJobAsync(ExportManagerQueueRecord exportManagerQueueRecord, String fileLocation) { try { Int32 fileFieldArtifactId = await ArtifactQueries.RetrieveFieldArtifactIdByGuid( rsapiApiOptions : RsapiApiOptions, workspaceArtifactId : exportManagerQueueRecord.WorkspaceArtifactId, fieldRepository : RsapiRepositoryGroup.FieldRepository, fieldGuid : Constant.Guids.Field.ExportUtilityJob.ExportFile ); await ArtifactQueries.AttachFileToExportJob( rsapiClient : AgentHelper.GetServicesManager().CreateProxy <IRSAPIClient>(ExecutionIdentity.CurrentUser), rsapiApiOptions : RsapiApiOptions, workspaceArtifactId : exportManagerQueueRecord.WorkspaceArtifactId, exportJobArtifactId : exportManagerQueueRecord.ExportJobArtifactId, fileFieldArtifactId : fileFieldArtifactId, fileLocation : fileLocation ); } catch (Exception ex) { throw new AdminMigrationUtilityException(Constant.ErrorMessages.AttachFileToExportJobError, ex); } }
public async Task DeleteProcessedRecordsFromExportResultsTableAsync(ExportManagerQueueRecord exportManagerQueueRecord, List <ExportWorkerResultsTableRecord> exportWorkerResultsTableRecords) { List <Int32> processedResultRecordTableRowIds = exportWorkerResultsTableRecords.OrderBy(x => x.TableRowId).Select(x => x.TableRowId).ToList(); await SqlQueryHelper.DeleteRecordsFromExportWorkerResultsTableAsync( eddsDbContext : AgentHelper.GetDBContext(-1), exportResultsTableName : exportManagerQueueRecord.ResultsTableName, tableRowIds : processedResultRecordTableRowIds); }
public async Task <Int32?> CurrentQueueRecordStatus(ExportManagerQueueRecord exportManagerQueueRecord) { return(await SqlQueryHelper.GetQueueRecordCurrentStatus( eddsDbContext : AgentHelper.GetDBContext(-1), workspaceIDColumnName : Constant.Sql.ColumnsNames.ExportManagerQueue.WorkspaceArtifactId, workspaceArtifactID : exportManagerQueueRecord.WorkspaceArtifactId, JobIDColumnName : Constant.Sql.ColumnsNames.ExportManagerQueue.ExportJobArtifactId, jobID : exportManagerQueueRecord.ExportJobArtifactId, tableName : Constant.Tables.ExportManagerQueue, statusColumnName : Constant.Sql.ColumnsNames.ExportManagerQueue.QueueStatus)); }
public void Constructor_ReceivesTable_Initializes() { // Arrange var table = GetTable(); // Act var record = new ExportManagerQueueRecord(table.Rows[0]); // Assert Assert.AreEqual(2345678, record.WorkspaceArtifactId); Assert.AreEqual(1, record.RecordId); Assert.AreEqual(3456789, record.ExportJobArtifactId); }
private async Task SendEmail(ExportManagerQueueRecord exportManagerQueueRecord, String status) { var exportJob = await RetrieveExportJobRDO(exportManagerQueueRecord); var emailTo = exportJob[Constant.Guids.Field.ExportUtilityJob.EmailAddresses].ValueAsFixedLengthText; var jobName = exportJob[Constant.Guids.Field.ExportUtilityJob.Name].ValueAsFixedLengthText; String emailSubject = String.Format(Constant.EmailSubject, "export"); String emailBody = String.Format(Constant.EmailBody, "Export", jobName, exportManagerQueueRecord.WorkspaceName, exportManagerQueueRecord.WorkspaceArtifactId, status); if (!String.IsNullOrWhiteSpace(emailTo)) { await EmailUtility.SendEmail(AgentHelper.GetDBContext(-1), emailTo, emailSubject, emailBody, new SmtpClientSettings(AgentHelper.GetDBContext(-1), SqlQueryHelper)); } }
public async Task <Int32> ProcessUsers(ExportManagerQueueRecord exportManagerQueueRecord, String exportResultTableName) { Int32 retVal; try { //query all users var userResults = await QueryAllUserArtifactIDs(); retVal = userResults.TotalCount; await InsertUserArtifactIDBatchIntoExportWorkerQueueTableAsync(exportManagerQueueRecord, exportResultTableName, userResults); } catch (Exception ex) { throw new AdminMigrationUtilityException(Constant.ErrorMessages.ExportManagerJobUserError, ex); } return(retVal); }
private async Task FinalizeJobStatiaticsAsync(ExportManagerQueueRecord exportManagerQueueRecord, Int32 actualNumberOfExportedRecords) { var jobRDO = await RetrieveExportJobRDO(exportManagerQueueRecord); var expectedNumberOfExports = jobRDO[Constant.Guids.Field.ExportUtilityJob.Expected].ValueAsWholeNumber.GetValueOrDefault(); if (expectedNumberOfExports > 0) { var notExported = (await ArtifactQueries.QueryJobErrors(RsapiApiOptions, RsapiRepositoryGroup.RdoRepository, exportManagerQueueRecord.WorkspaceArtifactId, exportManagerQueueRecord.ExportJobArtifactId)).Count(); await ArtifactQueries.UpdateExportJobStatitics( rdoRepository : RsapiRepositoryGroup.RdoRepository, rsapiApiOptions : RsapiApiOptions, workspaceArtifactId : exportManagerQueueRecord.WorkspaceArtifactId, jobArtifactID : exportManagerQueueRecord.ExportJobArtifactId, expectedNumberOfExports : expectedNumberOfExports, actualNumberOfExports : actualNumberOfExportedRecords, notExported : notExported, recordsThatHaveBeenProcessed : expectedNumberOfExports); } }
public async Task UpdateExportJobStatus(String status, ExportManagerQueueRecord exportManagerQueueRecord) { try { RaiseAndLogDebugMessage("Updating the status of export job."); await ArtifactQueries.UpdateRdoTextFieldValueAsync( rsapiApiOptions : RsapiApiOptions, workspaceArtifactId : exportManagerQueueRecord.WorkspaceArtifactId, rdoRepository : RsapiRepositoryGroup.RdoRepository, rdoArtifactId : exportManagerQueueRecord.ExportJobArtifactId, jobObjectGuid : Constant.Guids.ObjectType.ExportUtilityJob, textFieldGuid : Constant.Guids.Field.ExportUtilityJob.Status, fieldValue : status); RaiseAndLogDebugMessage("Updated the status of export job."); } catch (Exception ex) { throw new AdminMigrationUtilityException(Constant.ErrorMessages.UpdateExportJobStatusError, ex); } }
public async Task <Int32> CreateExportFileAsync(ExportManagerQueueRecord exportManagerQueueRecord, FileInfo exportFileInfo) { try { Int32 rowCountInExportWorkerResultsTable = await SqlQueryHelper.RetrieveTotalRowCountInExportWorkerResultsTableAsync(AgentHelper.GetDBContext(-1), exportManagerQueueRecord.ResultsTableName); if (rowCountInExportWorkerResultsTable == 0) { throw new AdminMigrationUtilityException(Constant.ErrorMessages.ZeroRowsInExportWorkerResultsTableError); } //insert headers into the export file await WriteColumnHeadersToExportFileAsync(exportFileInfo, exportManagerQueueRecord.ObjectType); //insert records in batch into the export file var actualNumberOfExports = await InsertExportWorkerResultTableUserRecordsIntoExportFileInBatchesAsync(exportManagerQueueRecord, exportFileInfo); return(actualNumberOfExports); } catch (Exception ex) { throw new AdminMigrationUtilityException(Constant.ErrorMessages.CreateExportFileError, ex); } }
public async Task ProcessNewRecordAsync(String delimitedListOfResourceGroupIds) { if (delimitedListOfResourceGroupIds != String.Empty) { DataTable dataTable = await RetrieveNextRecordInExportManagerQueueTableAsync(delimitedListOfResourceGroupIds); if (TableIsNotEmpty(dataTable)) { ExportManagerQueueRecord exportManagerQueueRecord = new ExportManagerQueueRecord(dataTable.Rows[0]); SetJobProperties(exportManagerQueueRecord); String context = $"[Table = {QueueTable}, ID = {TableRowId}, Workspace Artifact ID = {WorkspaceArtifactId}, Export Job Artifact ID = {ExportJobArtifactId}]"; RaiseAndLogDebugMessage($"Processing record(s). {context}"); try { Int32 numberOfExpectedExports; //update export job status to in progress await UpdateExportJobStatus(Constant.Status.Job.IN_PROGRESS_MANAGER, exportManagerQueueRecord); //update export manager queue record with export results table name String exportResultsTableName = $"{Constant.Names.ExportWorkerResultsTablePrefix}{Guid.NewGuid()}"; await SqlQueryHelper.UpdateResultsTableNameInExportManagerQueueAsync(AgentHelper.GetDBContext(-1), exportManagerQueueRecord.TableRowId, exportResultsTableName); // Add specific logic for other types of objects here #DifferentObject Constant.Enums.SupportedObjects migrationObjectType = (Constant.Enums.SupportedObjects)Enum.Parse(typeof(Constant.Enums.SupportedObjects), exportManagerQueueRecord.ObjectType); switch (migrationObjectType) { case Constant.Enums.SupportedObjects.User: numberOfExpectedExports = await ProcessUsers(exportManagerQueueRecord, exportResultsTableName); break; default: throw new AdminMigrationUtilityException(Constant.ErrorMessages.InvalidAdminObjectTypeError); } //Update status and statistics long as cancellation is not requested if (!await CancellationRequestedAsync(exportManagerQueueRecord)) { await SqlQueryHelper.UpdateStatusInExportManagerQueueByWorkspaceJobIdAsync( eddsDbContext : AgentHelper.GetDBContext(-1), workspaceArtifactId : exportManagerQueueRecord.WorkspaceArtifactId, exportJobArtifactId : exportManagerQueueRecord.ExportJobArtifactId, queueStatus : Constant.Status.Queue.WAITING_FOR_WORKERS_TO_FINISH); await UpdateExportJobStatus(Constant.Status.Job.COMPLETED_MANAGER, exportManagerQueueRecord); await UpdateExportJobExpectedNumberOfExports(numberOfExpectedExports, exportManagerQueueRecord); } } catch (Exception ex) { //update export job status to error await UpdateExportJobStatus(Constant.Status.Job.ERROR, exportManagerQueueRecord); String details = ExceptionMessageFormatter.GetExceptionMessageIncludingAllInnerExceptions(ex); RaiseAndLogDebugMessage(details); //remove record from export manager queue table await ClearRecordsFromQueueTables(exportManagerQueueRecord); } } } }
public override async Task ExecuteAsync() { try { //Check for jobs which stopped unexpectedly on this agent thread RaiseMessage($"Resetting records which failed. [Table = {QueueTable}]"); await ResetUnfishedJobsAsync(AgentHelper.GetDBContext(-1)); //Retrieve the next record to work on RaiseMessage($"Retrieving next record(s) in the queue. [Table = {QueueTable}]"); var delimitedListOfResourceGroupIds = GetCommaDelimitedListOfResourceIds(AgentResourceGroupIds); if (delimitedListOfResourceGroupIds != string.Empty) { var next = await RetrieveNextAsync(delimitedListOfResourceGroupIds); if (TableIsNotEmpty(next)) { var exportManagerQueueRecord = new ExportManagerQueueRecord(next.Rows[0]); // Sets the workspaceArtifactID and RecordID so the agent will have access to them in case of an exception WorkspaceArtifactId = exportManagerQueueRecord.WorkspaceArtifactId; RecordId = exportManagerQueueRecord.RecordId; RaiseMessage($"Retrieved record(s) in the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]"); //Check for Cancellation RequestGetDBContext var jobStatus = await RetrieveJobStatusAsync(_serviceMgr, WorkspaceArtifactId, ExecutionIdentity.System, exportManagerQueueRecord.ExportJobArtifactId); if (jobStatus != Constant.Status.Job.CANCELREQUESTED) { //Process the record(s) await ProcessRecordsAsync(exportManagerQueueRecord); //Remove the record from the manager queue await FinishAsync(); } else { //Remove the record from the manager queue await FinishAsync(); //Set the status of the Export Job to Cancelled await UpdateStatusFieldAsync(exportManagerQueueRecord.ExportJobArtifactId, Constant.Status.Job.CANCELLED); //Flush the Details of the Export Job await UpdateDetailsFieldAsync(exportManagerQueueRecord.ExportJobArtifactId, string.Empty); } } else { RaiseMessage(Constant.AgentRaiseMessages.NO_RECORDS_IN_QUEUE_FOR_THIS_RESOURCE_POOL); } } else { RaiseMessage(Constant.AgentRaiseMessages.AGENT_SERVER_NOT_PART_OF_ANY_RESOURCE_POOL); } } catch (Exception ex) { //log error RaiseMessage($"Logging error."); await LogErrorAsync(ex); } }
public async Task ProcessRecordsAsync(ExportManagerQueueRecord exportManagerQueueRecord) { RaiseMessage($"Processing record(s). [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]"); try { //Retrieve export job _markupUtilityExportJob = await _artifactQueries.RetrieveExportJobAsync(_serviceMgr, ExecutionIdentity.CurrentUser, WorkspaceArtifactId, exportManagerQueueRecord.ExportJobArtifactId); //Update status of the Export Job to Manager In Progress await UpdateStatusFieldAsync(exportManagerQueueRecord.ExportJobArtifactId, Constant.Status.Job.IN_PROGRESS_MANAGER); //Flush the Details of the Export Job await UpdateDetailsFieldAsync(exportManagerQueueRecord.ExportJobArtifactId, string.Empty); //Generate MarkupSubTypes var markupSubTypes = await GenerateMarkupSubTypesAsync(); //Create holding table await CreateHoldingTableAsync(); //Add Documents from Saved Search in the Export Worker Queue var isSuccessful = await AddDocumentsToHoldingTableAsync(); if (isSuccessful) { //Copy records from Holding table to Export Worker Queue table await CopyRecordsToExportWorkerQueueAsync(markupSubTypes, exportManagerQueueRecord.ExportJobArtifactId, exportManagerQueueRecord.ResourceGroupId); //Update status of the Export Job to Completed - Manager await UpdateStatusFieldAsync(exportManagerQueueRecord.ExportJobArtifactId, Constant.Status.Job.COMPLETED_MANAGER); //Flush the Details of the Export Job await UpdateDetailsFieldAsync(exportManagerQueueRecord.ExportJobArtifactId, string.Empty); } else { //Update status of the Export Job to Completed await UpdateStatusFieldAsync(exportManagerQueueRecord.ExportJobArtifactId, Constant.Status.Job.COMPLETED); //Update Details of the Export Job indicating no Documents in Saved Search await UpdateDetailsFieldAsync(exportManagerQueueRecord.ExportJobArtifactId, "No Documents found in the selected Saved Search."); //Update Exported Redation Count to 0 await _artifactQueries.UpdateRdoJobTextFieldAsync(_serviceMgr, WorkspaceArtifactId, ExecutionIdentity.CurrentUser, Constant.Guids.ObjectType.MarkupUtilityExportJob, _markupUtilityExportJob.ArtifactId, Constant.Guids.Field.MarkupUtilityExportJob.ExportedRedactionCount, "0"); } } catch (Exception ex) { //Update status of the Export Job to Error await UpdateStatusFieldAsync(exportManagerQueueRecord.ExportJobArtifactId, Constant.Status.Job.ERROR); //Update Details of the Export Job await UpdateDetailsFieldAsync(exportManagerQueueRecord.ExportJobArtifactId, ex.ToString()); //Delete Holding table await DeleteHoldingTableAsync(); //log error await LogErrorAsync(ex); } //delete import job from queue RaiseMessage($"Removing record(s) from the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]"); await FinishAsync(); await DeleteHoldingTableAsync(); RaiseMessage($"Removed record(s) from the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]"); RaiseMessage($"Processed record(s). [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]"); }
public async Task <Boolean> CancellationRequestedAsync(ExportManagerQueueRecord exportManagerQueueRecord) { return((await CurrentQueueRecordStatus(exportManagerQueueRecord)) == Constant.Status.Queue.CANCELLATION_REQUESTED); }
public async Task <Int32> InsertExportWorkerResultTableUserRecordsIntoExportFileInBatchesAsync(ExportManagerQueueRecord exportManagerQueueRecord, FileInfo tempFileInfo) { try { Int32 actualNumberOfExports = 0; //query for records in the results table in batches List <ExportWorkerResultsTableRecord> exportWorkerResultsTableRecords = await QueryBatchFromExportResultsTableAsync(exportManagerQueueRecord); do { foreach (ExportWorkerResultsTableRecord currentExportWorkerResultsTableRecord in exportWorkerResultsTableRecords) { IAdminObject migrationObject = await SerializationHelper.DeserializeToAdminObjectAsync(currentExportWorkerResultsTableRecord.MetaData); using (StreamWriter streamWriter = tempFileInfo.AppendText()) //creates file if it does not already exists { String[] orderedCells = (await migrationObject.ExportAsync()).ToArray(); String csvFormatedCells = await Utility.FormatCsvLineAsync(orderedCells); await streamWriter.WriteLineAsync(csvFormatedCells); actualNumberOfExports++; } } //delete processed records from export results table await DeleteProcessedRecordsFromExportResultsTableAsync(exportManagerQueueRecord, exportWorkerResultsTableRecords); //query for records in the results table in batches exportWorkerResultsTableRecords = await QueryBatchFromExportResultsTableAsync(exportManagerQueueRecord); } while (exportWorkerResultsTableRecords.Count > 0); return(actualNumberOfExports); } catch (Exception ex) { throw new AdminMigrationUtilityException(Constant.ErrorMessages.WritingResultRecordsToExportFileError, ex); } }
public async Task ClearRecordsFromQueueTables(ExportManagerQueueRecord exportManagerQueueRecord) { await SqlQueryHelper.DeleteRecordFromQueueAsync(AgentHelper.GetDBContext(-1), Constant.Sql.ColumnsNames.ExportWorkerQueue.WorkspaceArtifactId, exportManagerQueueRecord.WorkspaceArtifactId, Constant.Sql.ColumnsNames.ExportWorkerQueue.ExportJobArtifactId, exportManagerQueueRecord.ExportJobArtifactId, Constant.Tables.ExportWorkerQueue); await SqlQueryHelper.DeleteRecordFromQueueAsync(AgentHelper.GetDBContext(-1), Constant.Sql.ColumnsNames.ExportManagerQueue.WorkspaceArtifactId, exportManagerQueueRecord.WorkspaceArtifactId, Constant.Sql.ColumnsNames.ExportManagerQueue.ExportJobArtifactId, exportManagerQueueRecord.ExportJobArtifactId, Constant.Tables.ExportManagerQueue); }
public void SetJobProperties(ExportManagerQueueRecord exportManagerQueueRecord) { WorkspaceArtifactId = exportManagerQueueRecord.WorkspaceArtifactId; ExportJobArtifactId = exportManagerQueueRecord.ExportJobArtifactId; TableRowId = exportManagerQueueRecord.TableRowId; }
public async Task <List <ExportWorkerResultsTableRecord> > QueryBatchFromExportResultsTableAsync(ExportManagerQueueRecord exportManagerQueueRecord) { try { DataTable dataTable = await SqlQueryHelper.RetrieveBatchFromExportWorkerResultsTableAsync( eddsDbContext : AgentHelper.GetDBContext(-1), agentId : exportManagerQueueRecord.AgentId.GetValueOrDefault(), batchSize : Constant.BatchSizes.ExportManagerCSVQuery, exportResultsTableName : exportManagerQueueRecord.ResultsTableName); List <ExportWorkerResultsTableRecord> exportWorkerResultsTableRecords = new List <ExportWorkerResultsTableRecord>(); if (dataTable?.Rows != null && dataTable.Rows.Count > 0) { foreach (DataRow currentDataRow in dataTable.Rows) { ExportWorkerResultsTableRecord newExportWorkerResultsTableRecord = new ExportWorkerResultsTableRecord(currentDataRow); exportWorkerResultsTableRecords.Add(newExportWorkerResultsTableRecord); } } return(exportWorkerResultsTableRecords); } catch (Exception ex) { throw new AdminMigrationUtilityException(Constant.ErrorMessages.QueryBatchFromExportWorkerResultsTableError, ex); } }
public async Task UpdateExportJobExpectedNumberOfExports(Int32 expectedNumberOfExports, ExportManagerQueueRecord exportManagerQueueRecord) { try { RaiseAndLogDebugMessage("Updating export job expected number of exports."); await ArtifactQueries.UpdateExportJobStatitics(RsapiRepositoryGroup.RdoRepository, RsapiApiOptions, exportManagerQueueRecord.WorkspaceArtifactId, exportManagerQueueRecord.ExportJobArtifactId, expectedNumberOfExports, 0, 0, 0); RaiseAndLogDebugMessage("Updated export job expected number of exports."); } catch (Exception ex) { throw new AdminMigrationUtilityException(Constant.ErrorMessages.UpdateExportJobExpectedNumberError, ex); } }
private async Task <RDO> RetrieveExportJobRDO(ExportManagerQueueRecord exportManagerQueueRecord) { return(await ArtifactQueries.RetrieveJob(RsapiRepositoryGroup.RdoRepository, RsapiApiOptions, exportManagerQueueRecord.WorkspaceArtifactId, exportManagerQueueRecord.ExportJobArtifactId)); }