public override async Task ExecuteAsync()
{
try
{
//Check for jobs which stopped unexpectedly on this agent thread
RaiseMessage($"Resetting records which failed. [Table = {QueueTable}]");
await ResetUnfishedJobsAsync(AgentHelper.GetDBContext(-1));
//Retrieve the next record to work on
RaiseMessage($"Retrieving next record(s) in the queue. [Table = {QueueTable}]");
var delimitedListOfResourceGroupIds = GetCommaDelimitedListOfResourceIds(AgentResourceGroupIds);
if (delimitedListOfResourceGroupIds != string.Empty)
{
var next = await RetrieveNextAsync(delimitedListOfResourceGroupIds);
if (TableIsNotEmpty(next))
{
var firstExportWorkerQueueRecord = new ExportWorkerQueueRecord(next.Rows[0]);
WorkspaceArtifactId = firstExportWorkerQueueRecord.WorkspaceArtifactId;
//Retrieve export job
_markupUtilityExportJob = await _artifactQueries.RetrieveExportJobAsync(_serviceMgr, ExecutionIdentity.CurrentUser, WorkspaceArtifactId, firstExportWorkerQueueRecord.ExportJobArtifactId);
//Create Export Worker Job holding table
await CreateExportWorkerHoldingTableAsync();
//Set the status of the Export Job to In Progess - Export Worker
await UpdateStatusFieldAsync(firstExportWorkerQueueRecord.ExportJobArtifactId, Constant.Status.Job.IN_PROGRESS_WORKER);
//Process document redactions
await ProcessDocumentRedactionsAsync(next, firstExportWorkerQueueRecord.WorkspaceArtifactId, firstExportWorkerQueueRecord.ExportJobArtifactId);
RaiseMessage($"Retrieved record(s) in the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
}
else
{
RaiseMessage("No records in the queue for this resource pool.");
}
}
else
{
RaiseMessage(Constant.AgentRaiseMessages.AGENT_SERVER_NOT_PART_OF_ANY_RESOURCE_POOL);
}
}
catch (Exception ex)
{
RaiseMessage($"Logging error.");
await LogErrorAsync(ex);
}
finally
{
//Remove the Export Worker Holding table
await FinishHoldingTableAsync();
//Remove records from Batch table and Delete the Export Worker Batch table
await FinishAsync();
}
}
public void Constructor_ReceivesTable_Initializes()
{
// Arrange
var table = GetTable();
// Act
var record = new ExportWorkerQueueRecord(table.Rows[0]);
// Assert
Assert.AreEqual(2345678, record.WorkspaceArtifactId);
Assert.AreEqual(1, record.QueueRowId);
Assert.AreEqual(3456789, record.ArtifactId);
Assert.AreEqual(2, record.Priority);
}
public async Task ProcessUsers(Int32 workspaceArtifactID, Int32 jobArtifactID, DataTable batchDataTable)
{
try
{
RaiseAndLogDebugMessage("Processing all users in export worker batch.");
foreach (DataRow currentDataRow in batchDataTable.AsEnumerable())
{
ExportWorkerQueueRecord exportWorkerQueueRecord = new ExportWorkerQueueRecord(currentDataRow);
await ProcessSingleUserAsync(workspaceArtifactID, jobArtifactID, exportWorkerQueueRecord);
}
RaiseAndLogDebugMessage("Processed all users in export worker batch.");
}
catch (Exception ex)
{
throw new AdminMigrationUtilityException(Constant.ErrorMessages.ProcessAllUsersInExportWorkerBatchError, ex);
}
}
public async Task ProcessDocumentRedactionsSingleAsync(ExportWorkerQueueRecord exportWorkerQueueRecord)
{
_errorContext = $"[WorkspaceArtifactId = {WorkspaceArtifactId}, ExportJobArtifactId = {exportWorkerQueueRecord.ExportJobArtifactId}]";
RaiseMessage($"Processing record(s). [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}, DocumentArtifactId = {exportWorkerQueueRecord.DocumentArtifactId}, MarkupSetArtifactId = {exportWorkerQueueRecord.MarkupSetArtifactId}, MarkupSubTypes = {exportWorkerQueueRecord.MarkupSubType}]");
//Check if Document exists
await _artifactQueries.VerifyIfDocumentExistsAsync(_serviceMgr, ExecutionIdentity.CurrentUser, WorkspaceArtifactId, exportWorkerQueueRecord.DocumentArtifactId);
//Retrieve the Column Name of the Document Identifier Field
var documentIdentifierColumnName = await QueryHelper.GetDocumentIdentifierFieldColumnNameAsync(AgentHelper.GetDBContext(WorkspaceArtifactId));
//Retrieve Redactions for the selected Documents from the Workspace, store in DataTable
var dtDocumentRedactions = await QueryHelper.RetrieveRedactionsForDocumentAsync(AgentHelper.GetDBContext(exportWorkerQueueRecord.WorkspaceArtifactId), exportWorkerQueueRecord.WorkspaceArtifactId, exportWorkerQueueRecord.ExportJobArtifactId, exportWorkerQueueRecord.DocumentArtifactId, exportWorkerQueueRecord.MarkupSetArtifactId, documentIdentifierColumnName, exportWorkerQueueRecord.MarkupSubType);
//Write records to holding table in EDDS
await AddRedactionsFromHoldingToResultsTableAsync(dtDocumentRedactions, ExportWorkerHoldingTable);
RaiseMessage($"Processed record(s). [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}, DocumentArtifactId = {exportWorkerQueueRecord.DocumentArtifactId}, MarkupSetArtifactId = {exportWorkerQueueRecord.MarkupSetArtifactId}, MarkupSubTypes = {exportWorkerQueueRecord.MarkupSubType}]");
}
public void Constructor_ReceivesTable_Initializes()
{
// Arrange
var table = GetTable();
// Act
var record = new ExportWorkerQueueRecord(table.Rows[0]);
// Assert
Assert.AreEqual(2345678, record.WorkspaceArtifactId);
Assert.AreEqual(1, record.RecordId);
Assert.AreEqual(3456789, record.ExportJobArtifactId);
Assert.AreEqual(3, record.DocumentArtifactId);
Assert.AreEqual(4, record.MarkupSetArtifactId);
Assert.AreEqual(5, record.QueueStatus);
Assert.AreEqual(6, record.AgentId);
Assert.AreEqual("7", record.MarkupSubType);
Assert.AreEqual(8, record.ResourceGroupId);
}
private async Task ProcessDocumentRedactionsAsync(DataTable dtExportJobWorkerQueueRecords, int workspaceArtifactId, int exportJobArtifactId)
{
try
{
foreach (DataRow row in dtExportJobWorkerQueueRecords.Rows)
{
var exportWorkerQueueRecord = new ExportWorkerQueueRecord(row);
//Sets the workspaceArtifactID and RecordID so the agent will have access to them in case of an exception
WorkspaceArtifactId = exportWorkerQueueRecord.WorkspaceArtifactId;
RecordId = exportWorkerQueueRecord.RecordId;
_errorContext = $"[WorkspaceArtifactId = {WorkspaceArtifactId}, Id = {RecordId}]";
//Retrieve the Document redactions
await ProcessDocumentRedactionsSingleAsync(exportWorkerQueueRecord);
}
//Bulk copy from Holding table to Export Results table
await AddRecordsToExportResultsAsync(ExportWorkerHoldingTable);
//Remove the Export Worker Holding table
await FinishHoldingTableAsync();
//Remove records from the Export Worker Batch table
await DeleteRecordsFromExportWorkerBatchTableAsync();
//Check if no more records exist in the Export Worker queue, if not, proceed with creating CSV for attachment to Export Job
await VerifyIfExportWorkerQueueContainsRecordsForJobAsync(workspaceArtifactId, exportJobArtifactId);
}
catch (Exception ex)
{
//Set the status of the Export Job to Error
await UpdateStatusFieldAsync(_markupUtilityExportJob.ArtifactId, Constant.Status.Job.ERROR);
await UpdateDetailsFieldAsync(_markupUtilityExportJob.ArtifactId, await ConstructDetailsExceptionMessageAsync(ex));
RaiseMessage($"Logging error.");
await LogErrorAsync(ex);
//Remove the Export Worker Holding table
await FinishHoldingTableAsync();
}
}
public async Task <String> RetrieveExportJobStatusAsync(ExportWorkerQueueRecord exportWorkerQueueRecord)
{
try
{
RaiseAndLogDebugMessage("Retrieving the status of export job.");
String exportJobStatus = await ArtifactQueries.RetrieveRdoTextFieldValueAsync(
rsapiApiOptions : RsapiApiOptions,
workspaceArtifactId : exportWorkerQueueRecord.WorkspaceArtifactId,
rdoRepository : RsapiRepositoryGroup.RdoRepository,
rdoArtifactId : exportWorkerQueueRecord.ExportJobArtifactId,
textFieldGuid : Constant.Guids.Field.ExportUtilityJob.Status);
RaiseAndLogDebugMessage("Retrieved the status of export job.");
return(exportJobStatus);
}
catch (Exception ex)
{
throw new AdminMigrationUtilityException(Constant.ErrorMessages.RetrieveExportJobStatusError, ex);
}
}
public async Task InsertUserDataIntoExportWorkerResultsTableAsync(ExportWorkerQueueRecord exportWorkerQueueRecord, UserAdminObject userAdminObject)
{
try
{
//deserialize user admin object
String metadata = await SerializationHelper.SerializeAdminObjectAsync(userAdminObject);
//insert into export worker results table
await SqlQueryHelper.InsertRowIntoExportWorkerResultsTableAsync(
eddsDbContext : AgentHelper.GetDBContext(-1),
exportWorkerResultTableName : exportWorkerQueueRecord.ResultTableName,
workspaceArtifactId : WorkspaceArtifactId,
exportJobArtifactId : exportWorkerQueueRecord.ExportJobArtifactId,
objectType : exportWorkerQueueRecord.ObjectType,
workspaceResourceGroupArtifactId : exportWorkerQueueRecord.WorkspaceResourceGroupArtifactId,
queueStatus : Constant.Status.Queue.NOT_STARTED,
metadata : metadata,
timeStampUtc : DateTime.UtcNow);
}
catch (Exception ex)
{
throw new AdminMigrationUtilityException(Constant.ErrorMessages.InsertUserDataIntoExportWorkerResultsTableError, ex);
}
}
public override async Task ExecuteAsync()
{
try
{
Boolean isOffHours = await IsOffHoursAsync(ProcessedOnDateTime);
if (!isOffHours)
{
RaiseAndLogDebugMessage($"Current time is not between {OffHoursStartTime} and {OffHoursEndTime}. Agent execution skipped.");
}
else
{
//Check for jobs which stopped unexpectedly on this agent thread and reset them
await ResetUnfinishedExportWorkerJobsAsync();
//Retrieve the next record to work on
String commaDelimitedListOfResourceIds = GetCommaDelimitedListOfResourceIds(AgentResourceGroupIds);
if (commaDelimitedListOfResourceIds == String.Empty)
{
RaiseAndLogDebugMessage("This agent server is not part of any resource pools. Agent execution skipped.");
}
else
{
DataTable batchDataTable = await RetrieveNextBatchOfRecordsInExportWorkerQueueTableAsync(commaDelimitedListOfResourceIds);
if (IsTableEmpty(batchDataTable))
{
RaiseAndLogDebugMessage("No records in the export worker queue table for this resource pool.");
}
else
{
// Use first record to check the status of the Job RDO. All records are from the same job
var workerQueueRecord = new ExportWorkerQueueRecord(batchDataTable.Rows[0]);
SetJobProperties(workerQueueRecord);
var objectType = (Constant.Enums.SupportedObjects)Enum.Parse(typeof(Constant.Enums.SupportedObjects), workerQueueRecord.ObjectType);
String exportJobStatus = await RetrieveExportJobStatusAsync(workerQueueRecord);
if (exportJobStatus != Constant.Status.Job.IN_PROGRESS_WORKER)
{
//update export job status to in progress
await UpdateExportJobStatus(workerQueueRecord.WorkspaceArtifactId, workerQueueRecord.ExportJobArtifactId, Constant.Status.Job.IN_PROGRESS_WORKER);
}
//Create results table name if it does not already exists
await CreateExportWorkerResultsTableIfItDoesNotAlreadyExistsAsync(workerQueueRecord);
await ProcessBatchRecordsAsync(batchDataTable, workerQueueRecord.WorkspaceArtifactId, workerQueueRecord.ExportJobArtifactId, objectType);
}
}
}
}
catch (Exception ex)
{
throw new AdminMigrationUtilityException(Constant.ErrorMessages.ExportWorkerAgentError, ex);
}
finally
{
//drop batch table
await DropExportWorkerBatchTableAsync();
}
}
public async Task ProcessSingleUserAsync(Int32 workspaceArtifactID, Int32 jobArtifactID, ExportWorkerQueueRecord exportWorkerQueueRecord)
{
try
{
RaiseAndLogDebugMessage($"Processing a single user in export worker batch. [{nameof(exportWorkerQueueRecord.ArtifactId)}] = {exportWorkerQueueRecord.ArtifactId}");
kCura.Relativity.Client.DTOs.User userRDO = await ArtifactQueries.RetrieveUserAsync(RsapiApiOptions, RsapiRepositoryGroup.UserRepository, exportWorkerQueueRecord.ArtifactId);
//query user Auth data
LoginProfile userLoginProfile = await AuthenticationHelper.RetrieveExistingUserLoginProfileAsync(userRDO.ArtifactID);
//query keywords and notes for user
KeywordsNotesModel userKeywordsNotesModel = await SqlQueryHelper.RetrieveKeywordsAndNotesForUserAsync(
eddsDbContext : AgentHelper.GetDBContext(-1),
userArtifactId : userRDO.ArtifactID);
//query groups user is part of
IEnumerable <String> userGroupNameList = await QueryGroupsNamesUserIsPartOfAsync(userRDO);
UserAdminObject userAdminObject = await ConvertUserResultToUserAdminObjectAsync(userRDO);
//add keywords and notes, auth and groups data to UserAdminObject
await AddKeywordsNotesAuthInfoAndGroupsToUserAdminObjectAsync(userAdminObject, userKeywordsNotesModel, userGroupNameList, userLoginProfile);
//insert user data info into the export worker results table
await InsertUserDataIntoExportWorkerResultsTableAsync(exportWorkerQueueRecord, userAdminObject);
RaiseAndLogDebugMessage($"Processed a single user in export worker batch. [{nameof(exportWorkerQueueRecord.ArtifactId)}] = {exportWorkerQueueRecord.ArtifactId}");
}
catch (Exception ex)
{
//create export job error record
String details = ExceptionMessageFormatter.GetExceptionMessageIncludingAllInnerExceptions(ex);
await CreateExportJobErrorRecordAsync(workspaceArtifactID, jobArtifactID, exportWorkerQueueRecord.ObjectType, Constant.Status.JobErrors.ERROR, details);
}
}
public async Task CreateExportWorkerResultsTableIfItDoesNotAlreadyExistsAsync(ExportWorkerQueueRecord workerQueueRecord)
{
try
{
RaiseAndLogDebugMessage("Creating the export worker results table if it doesn't already exists.");
await SqlQueryHelper.CreateExportWorkerResultsTableIfItDoesNotAlreadyExistsAsync(
eddsDbContext : AgentHelper.GetDBContext(-1),
tableName : workerQueueRecord.ResultTableName);
RaiseAndLogDebugMessage("Created the export worker results table if it doesn't already exists.");
}
catch (Exception ex)
{
throw new AdminMigrationUtilityException(Constant.ErrorMessages.CreateExportWorkerResultsTableIfItDoesNotAlreadyExistsError, ex);
}
}
public void SetJobProperties(ExportWorkerQueueRecord exportWorkerQueueRecord)
{
WorkspaceArtifactId = exportWorkerQueueRecord.WorkspaceArtifactId;
TableRowId = exportWorkerQueueRecord.QueueRowId;
}