private async Task WriteQueueRecordsAsync(ImportManagerQueueRecord managerQueueRecord, Dictionary <long, String> queueRecordToPopulate, List <ImportJobError> violations, DateTime timeOfInsert)
{
try
{
var dt = new DataTable();
dt.Columns.Add(Constant.Sql.ColumnsNames.ImportWorkerQueue.JobID, typeof(Int32));
dt.Columns.Add(Constant.Sql.ColumnsNames.ImportWorkerQueue.WorkspaceArtifactID, typeof(Int32));
dt.Columns.Add(Constant.Sql.ColumnsNames.ImportWorkerQueue.ObjectType, typeof(String));
dt.Columns.Add(Constant.Sql.ColumnsNames.ImportWorkerQueue.JobType, typeof(String));
dt.Columns.Add(Constant.Sql.ColumnsNames.ImportWorkerQueue.MetaData, typeof(String));
dt.Columns.Add(Constant.Sql.ColumnsNames.ImportWorkerQueue.ImportRowID, typeof(Int32));
dt.Columns.Add(Constant.Sql.ColumnsNames.ImportWorkerQueue.QueueStatus, typeof(Int32));
dt.Columns.Add(Constant.Sql.ColumnsNames.ImportWorkerQueue.Priority, typeof(Int32));
dt.Columns.Add(Constant.Sql.ColumnsNames.ImportWorkerQueue.ResourceGroupID, typeof(Int32));
dt.Columns.Add(Constant.Sql.ColumnsNames.ImportWorkerQueue.TimeStampUTC, typeof(DateTime));
foreach (var record in queueRecordToPopulate)
{
dt.Rows.Add(managerQueueRecord.JobId, managerQueueRecord.WorkspaceArtifactId, managerQueueRecord.ObjectType, managerQueueRecord.JobType, record.Value, record.Key, Constant.Status.Queue.NOT_STARTED, managerQueueRecord.Priority, managerQueueRecord.ResourceGroupId, timeOfInsert);
}
await SqlQueryHelper.BulkInsertIntoTableAsync(AgentHelper.GetDBContext(-1), dt, Constant.Tables.ImportWorkerQueue, Constant.BatchSizes.ImportManagerIntoWorkerQueue);
}
catch (Exception ex)
{
var beginErrorLine = queueRecordToPopulate.Keys.Min();
var endErrorLine = queueRecordToPopulate.Keys.Max();
violations.Add(new ImportJobError()
{
Message = String.Format(Constant.ErrorMessages.ImportQueueManagerPopulatingImportWorkerQueueError, beginErrorLine, endErrorLine), LineNumber = null, Type = Constant.ImportUtilityJob.ErrorType.DataLevel, Details = ex.ToString()
});
}
}
private async Task AddRecordsToExportResultsAsync(string holdingTableName)
{
RaiseMessage($"Creating Export Worker Results from holding table for export job [ExportWorkerHoldingTable = {ExportWorkerHoldingTable}]. {_errorContext}");
await QueryHelper.CopyRecordsToExportResultsAsync(AgentHelper.GetDBContext(-1), holdingTableName);
RaiseMessage($"Created Export Worker Results from holding table for export job [ExportWorkerHoldingTable = {ExportWorkerHoldingTable}]. {_errorContext}");
}
public async Task FinishAsync()
{
RaiseMessage($"Dropping batch table. [BatchTableName = {BatchTableName}]");
await QueryHelper.DropTableAsync(AgentHelper.GetDBContext(-1), BatchTableName);
RaiseMessage($"Dropped batch table. [BatchTableName = {BatchTableName}]");
}
public async Task InsertUserArtifactIDBatchIntoExportWorkerQueueTableAsync(ExportManagerQueueRecord exportManagerQueueRecord, String exportResultsTableName, QueryResultSet <kCura.Relativity.Client.DTOs.User> users)
{
var currentTime = DateTime.UtcNow;
var dt = new DataTable();
dt.Columns.Add(Constant.Sql.ColumnsNames.ExportWorkerQueue.WorkspaceArtifactId, typeof(Int32));
dt.Columns.Add(Constant.Sql.ColumnsNames.ExportWorkerQueue.ExportJobArtifactId, typeof(Int32));
dt.Columns.Add(Constant.Sql.ColumnsNames.ExportWorkerQueue.ObjectType, typeof(String));
dt.Columns.Add(Constant.Sql.ColumnsNames.ExportWorkerQueue.Priority, typeof(Int32));
dt.Columns.Add(Constant.Sql.ColumnsNames.ExportWorkerQueue.WorkspaceResourceGroupArtifactId, typeof(Int32));
dt.Columns.Add(Constant.Sql.ColumnsNames.ExportWorkerQueue.QueueStatus, typeof(Int32));
dt.Columns.Add(Constant.Sql.ColumnsNames.ExportWorkerQueue.ArtifactId, typeof(Int32));
dt.Columns.Add(Constant.Sql.ColumnsNames.ExportWorkerQueue.ResultsTableName, typeof(String));
dt.Columns.Add(Constant.Sql.ColumnsNames.ExportWorkerQueue.MetaData, typeof(String));
dt.Columns.Add(Constant.Sql.ColumnsNames.ExportWorkerQueue.TimeStampUtc, typeof(DateTime));
try
{
foreach (var user in users.Results)
{
dt.Rows.Add(exportManagerQueueRecord.WorkspaceArtifactId, exportManagerQueueRecord.ExportJobArtifactId, exportManagerQueueRecord.ObjectType, exportManagerQueueRecord.Priority, exportManagerQueueRecord.WorkspaceResourceGroupArtifactId, Constant.Status.Queue.NOT_STARTED, user.Artifact.ArtifactID, exportResultsTableName, String.Empty, currentTime);
}
await SqlQueryHelper.BulkInsertIntoTableAsync(AgentHelper.GetDBContext(-1), dt, Constant.Tables.ExportWorkerQueue, Constant.BatchSizes.ExportManagerIntoWorkerQueue);
}
catch (Exception ex)
{
throw new AdminMigrationUtilityException(Constant.ErrorMessages.UnableToInsertUserArtifactIDsIntoWorkerQueue, ex);
}
}
public async Task <List <ExportWorkerResultsTableRecord> > QueryBatchFromExportResultsTableAsync(ExportManagerQueueRecord exportManagerQueueRecord)
{
try
{
DataTable dataTable = await SqlQueryHelper.RetrieveBatchFromExportWorkerResultsTableAsync(
eddsDbContext : AgentHelper.GetDBContext(-1),
agentId : exportManagerQueueRecord.AgentId.GetValueOrDefault(),
batchSize : Constant.BatchSizes.ExportManagerCSVQuery,
exportResultsTableName : exportManagerQueueRecord.ResultsTableName);
List <ExportWorkerResultsTableRecord> exportWorkerResultsTableRecords = new List <ExportWorkerResultsTableRecord>();
if (dataTable?.Rows != null && dataTable.Rows.Count > 0)
{
foreach (DataRow currentDataRow in dataTable.Rows)
{
ExportWorkerResultsTableRecord newExportWorkerResultsTableRecord = new ExportWorkerResultsTableRecord(currentDataRow);
exportWorkerResultsTableRecords.Add(newExportWorkerResultsTableRecord);
}
}
return(exportWorkerResultsTableRecords);
}
catch (Exception ex)
{
throw new AdminMigrationUtilityException(Constant.ErrorMessages.QueryBatchFromExportWorkerResultsTableError, ex);
}
}
private async Task RemoveRecordsFromImportWorkerQueueTableAsync()
{
RaiseMessage($"Removing record(s) from the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
await QueryHelper.RemoveBatchFromImportWorkerQueueAsync(AgentHelper.GetDBContext(-1), BatchTableName);
RaiseMessage($"Removed record(s) from the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
}
private async Task UpdateMarkupSetMultipleChoiceFieldValueAsync(int documentArtifactId, int markupType)
{
const string errorContext = "An error occured when updating markup set multiple choice field value.";
try
{
int choiceArtifactId;
switch (markupType)
{
case Constant.MarkupType.Redaction.VALUE:
choiceArtifactId = _hasRedactionsChoiceModel.ArtifactId;
break;
case Constant.MarkupType.Highlight.VALUE:
choiceArtifactId = _hasHighlightsChoiceModel.ArtifactId;
break;
default:
throw new MarkupUtilityException($"Invalid Markup Type. [MarkupType = {markupType}]. {errorContext}");
}
await QueryHelper.UpdateMarkupSetMultipleChoiceFieldAsync(
AgentHelper.GetDBContext(WorkspaceArtifactId),
documentArtifactId,
_markupSetMultiplechoiceFieldChoiceTypeId,
choiceArtifactId);
}
catch (Exception ex)
{
throw new MarkupUtilityException($"{errorContext}", ex);
}
}
public async Task FinishAsync()
{
RaiseMessage($"Removing record(s) from the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
await QueryHelper.RemoveRecordFromTableByIdAsync(AgentHelper.GetDBContext(-1), QueueTable, RecordId);
RaiseMessage($"Removed record(s) from the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
}
public async Task DeleteRecordFromReproduceWorkerQueueAsync(int id)
{
RaiseMessage($"Deleting record from ReproduceWorkerQueue. [ID = {id}, Workspace Artifact ID = {WorkspaceArtifactId}, Agent ID = {AgentId}]");
await QueryHelper.RemoveRecordFromReproduceWorkerQueueAsync(AgentHelper.GetDBContext(-1), id);
RaiseMessage($"Deleted record from ReproduceWorkerQueue. [ID = {id}, Workspace Artifact ID = {WorkspaceArtifactId}, Agent ID = {AgentId}]");
}
private async Task CreateExportWorkerHoldingTableAsync()
{
RaiseMessage($"Creating Export Worker holding table for export job [ExportWorkerHoldingTable = {ExportWorkerHoldingTable}]. {_errorContext}");
await QueryHelper.CreateExportWorkerHoldingTableAsync(AgentHelper.GetDBContext(-1), ExportWorkerHoldingTable);
RaiseMessage($"Created Export Worker holding table for export job [ExportWorkerHoldingTable = {ExportWorkerHoldingTable}]. {_errorContext}");
}
private async Task GetOffHoursTimesAsync()
{
try
{
RaiseAndLogDebugMessage("Retrieving off hours.");
DataTable dataTable = await SqlQueryHelper.RetrieveOffHoursAsync(AgentHelper.GetDBContext(-1));
if (dataTable?.Rows == null ||
dataTable.Rows.Count == 0 ||
String.IsNullOrEmpty(dataTable.Rows[0]["AgentOffHourStartTime"].ToString()) ||
String.IsNullOrEmpty(dataTable.Rows[0]["AgentOffHourEndTime"].ToString()) ||
dataTable.Rows[0]["AgentOffHourStartTime"] == null ||
dataTable.Rows[0]["AgentOffHourEndTime"] == null)
{
throw new AdminMigrationUtilityException(Constant.Messages.Agent.AGENT_OFF_HOURS_NOT_FOUND);
}
OffHoursStartTime = dataTable.Rows[0]["AgentOffHourStartTime"].ToString();
OffHoursEndTime = dataTable.Rows[0]["AgentOffHourEndTime"].ToString();
RaiseAndLogDebugMessage("Retrieved off hours.");
}
catch (Exception ex)
{
throw new AdminMigrationUtilityException(Constant.ErrorMessages.RetrieveOffHoursError, ex);
}
}
public override async Task ExecuteAsync()
{
try
{
//Check for jobs which stopped unexpectedly on this agent thread
RaiseMessage($"Resetting records which failed. [Table = {QueueTable}]");
await ResetUnfishedJobsAsync(AgentHelper.GetDBContext(-1));
//Retrieve the next record to work on
RaiseMessage($"Retrieving next record(s) in the queue. [Table = {QueueTable}]");
var delimitedListOfResourceGroupIds = GetCommaDelimitedListOfResourceIds(AgentResourceGroupIds);
if (delimitedListOfResourceGroupIds != string.Empty)
{
var next = await RetrieveNextAsync(delimitedListOfResourceGroupIds);
if (TableIsNotEmpty(next))
{
var firstExportWorkerQueueRecord = new ExportWorkerQueueRecord(next.Rows[0]);
WorkspaceArtifactId = firstExportWorkerQueueRecord.WorkspaceArtifactId;
//Retrieve export job
_markupUtilityExportJob = await _artifactQueries.RetrieveExportJobAsync(_serviceMgr, ExecutionIdentity.CurrentUser, WorkspaceArtifactId, firstExportWorkerQueueRecord.ExportJobArtifactId);
//Create Export Worker Job holding table
await CreateExportWorkerHoldingTableAsync();
//Set the status of the Export Job to In Progess - Export Worker
await UpdateStatusFieldAsync(firstExportWorkerQueueRecord.ExportJobArtifactId, Constant.Status.Job.IN_PROGRESS_WORKER);
//Process document redactions
await ProcessDocumentRedactionsAsync(next, firstExportWorkerQueueRecord.WorkspaceArtifactId, firstExportWorkerQueueRecord.ExportJobArtifactId);
RaiseMessage($"Retrieved record(s) in the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
}
else
{
RaiseMessage("No records in the queue for this resource pool.");
}
}
else
{
RaiseMessage(Constant.AgentRaiseMessages.AGENT_SERVER_NOT_PART_OF_ANY_RESOURCE_POOL);
}
}
catch (Exception ex)
{
RaiseMessage($"Logging error.");
await LogErrorAsync(ex);
}
finally
{
//Remove the Export Worker Holding table
await FinishHoldingTableAsync();
//Remove records from Batch table and Delete the Export Worker Batch table
await FinishAsync();
}
}
public async Task DeleteRecordsFromExportWorkerBatchTableAsync()
{
//Remove records from the Export Worker Batch table
RaiseMessage($"Removing record(s) from the Export Worker Batch table. [Table = {BatchTableName}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}, Agent ID = {AgentId}]");
await QueryHelper.RemoveBatchFromExportWorkerQueueAsync(AgentHelper.GetDBContext(-1), BatchTableName);
RaiseMessage($"Removed record(s) from the Export Worker Batch table. [Table = {BatchTableName}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}, Agent ID = {AgentId}]");
}
public async Task FinishAsync()
{
//Delete the Export Worker Batch table
RaiseMessage($"Deleting the Export Worker Batch table. [Table = {BatchTableName}, Agent ID = {AgentId}]");
await QueryHelper.DropTableAsync(AgentHelper.GetDBContext(-1), BatchTableName);
RaiseMessage($"Deleted the Export Worker Batch table. [Table = {BatchTableName}, Agent ID = {AgentId}]");
}
public async Task DeleteRedactionsHoldingTableAsync()
{
//Delete the Redactions Holding table
RaiseMessage($"Deleting the Reproduce Manager Redactions Holding table. [Table = {RedactionsHoldingTable}, Agent ID = {AgentId}]");
await QueryHelper.DropTableAsync(AgentHelper.GetDBContext(-1), RedactionsHoldingTable);
RaiseMessage($"Deleted the Reproduce Manager Redactions Holding table. [Table = {RedactionsHoldingTable}, Agent ID = {AgentId}]");
}
public async Task DeleteProcessedRecordsFromExportResultsTableAsync(ExportManagerQueueRecord exportManagerQueueRecord, List <ExportWorkerResultsTableRecord> exportWorkerResultsTableRecords)
{
List <Int32> processedResultRecordTableRowIds = exportWorkerResultsTableRecords.OrderBy(x => x.TableRowId).Select(x => x.TableRowId).ToList();
await SqlQueryHelper.DeleteRecordsFromExportWorkerResultsTableAsync(
eddsDbContext : AgentHelper.GetDBContext(-1),
exportResultsTableName : exportManagerQueueRecord.ResultsTableName,
tableRowIds : processedResultRecordTableRowIds);
}
public override async Task ExecuteAsync()
{
//reset count properties
_importFileRedactionCount = 0;
_expectedRedactionCount = 0;
try
{
//Check for jobs which stopped unexpectedly on this agent thread
RaiseMessage($"Resetting records which failed. [Table = {QueueTable}]");
await ResetUnfishedJobsAsync(AgentHelper.GetDBContext(-1));
//Retrieve the next record to work on
RaiseMessage($"Retrieving next record(s) in the queue. [Table = {QueueTable}]");
var commaDelimitedListOfResourceIds = GetCommaDelimitedListOfResourceIds(AgentResourceGroupIds);
if (commaDelimitedListOfResourceIds != string.Empty)
{
var next = await RetrieveNextAsync(commaDelimitedListOfResourceIds);
if (TableIsNotEmpty(next))
{
var importManagerQueueRecord = new ImportManagerQueueRecord(next.Rows[0]);
// Sets the workspaceArtifactID and RecordID so the agent will have access to them in case of an exception
WorkspaceArtifactId = importManagerQueueRecord.WorkspaceArtifactId;
RecordId = importManagerQueueRecord.Id;
RaiseMessage($"Retrieved record(s) in the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
//set _importJobArtifactId
_importJobArtifactId = importManagerQueueRecord.ImportJobArtifactId;
//Process the record(s)
await ProcessRecordsAsync(importManagerQueueRecord);
//delete import job from queue
await FinishAsync();
}
else
{
RaiseMessage(Constant.AgentRaiseMessages.NO_RECORDS_IN_QUEUE_FOR_THIS_RESOURCE_POOL);
}
}
else
{
RaiseMessage(Constant.AgentRaiseMessages.AGENT_SERVER_NOT_PART_OF_ANY_RESOURCE_POOL);
}
}
catch (Exception ex)
{
var innerMostExceptionMessage = await ConstructDetailsExceptionMessageAsync(ex);
//update import job status field to complete with errors and details field to inner most exception message
await UpdateImportJobStatusAndDetailsFieldAsync(Constant.Status.Job.ERROR, innerMostExceptionMessage);
//log error
await LogErrorAsync(ex);
}
}
public override async Task ExecuteAsync()
{
try
{
//Check for jobs which stopped unexpectedly on this agent thread
RaiseMessage($"Resetting records which failed. [Table = {QueueTable}]");
await ResetUnfishedJobsAsync(AgentHelper.GetDBContext(-1));
//Retrieve the next record to work on
RaiseMessage($"Retrieving next record(s) in the queue. [Table = {QueueTable}]");
var delimitedListOfResourceGroupIds = GetCommaDelimitedListOfResourceIds(AgentResourceGroupIds);
if (delimitedListOfResourceGroupIds != string.Empty)
{
var next = await RetrieveNextAsync(delimitedListOfResourceGroupIds);
if (TableIsNotEmpty(next))
{
var workerQueueRecord = new ReproduceWorkerQueueRecord(next.Rows[0]);
WorkspaceArtifactId = workerQueueRecord.WorkspaceArtifactId;
//Process document redactions
var markupUtilityReproduceJob = await RetrieveReproduceJobAsync(workerQueueRecord);
//someone cancelled the job or managed to delete it
if (markupUtilityReproduceJob == null)
{
await Finish(workerQueueRecord, null);
}
else if (markupUtilityReproduceJob.Status == Constant.Status.Job.CANCELREQUESTED || markupUtilityReproduceJob.Status == Constant.Status.Job.CANCELLED)
{
//you can end up in state Cancel Requestd if you cancel just after the ManagerQueue job completed, in which case update the the status to Cancelled
await Finish(workerQueueRecord, Constant.Status.Job.CANCELLED);
}
else
{
await UpdateStatusFieldAsync(workerQueueRecord.ReproduceJobArtifactId, Constant.Status.Job.IN_PROGRESS_WORKER);
await ProcessRedactionsAsync(workerQueueRecord, markupUtilityReproduceJob);
}
RaiseMessage($"Retrieved record(s) in the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
}
else
{
RaiseMessage("No records in the queue for this resource pool.");
}
}
else
{
RaiseMessage(Constant.AgentRaiseMessages.AGENT_SERVER_NOT_PART_OF_ANY_RESOURCE_POOL);
}
}
catch (Exception ex)
{
RaiseMessage($"Logging error.");
await LogErrorAsync(ex);
}
}
public async Task FinishHoldingTableAsync()
{
//Delete the Export Worker Holding table
RaiseMessage($"Deleting the Export Worker Holding table. [Table = {ExportWorkerHoldingTable}, Agent ID = {AgentId}]");
await QueryHelper.DropTableAsync(AgentHelper.GetDBContext(-1), ExportWorkerHoldingTable);
//await Task.Run(() => { });
RaiseMessage($"Deleted the Export Worker Holding table. [Table = {ExportWorkerHoldingTable}, Agent ID = {AgentId}]");
}
public async Task CheckIfWorkersCompletedJob(IEnumerable <ExportManagerQueueRecord> exportManagerQueueRecords)
{
var jobsInTheWorkerQueue = exportManagerQueueRecords.Where(x => x.QueueStatus == Constant.Status.Queue.WAITING_FOR_WORKERS_TO_FINISH);
foreach (var exportManagerQueueRecord in jobsInTheWorkerQueue)
{
try
{
SetJobProperties(exportManagerQueueRecord);
Int32 numberOfWorkerRecords = await SqlQueryHelper.CountNumberOfExportWorkerRecordsAsync(
eddsDbContext : AgentHelper.GetDBContext(-1),
workspaceArtifactId : exportManagerQueueRecord.WorkspaceArtifactId,
exportJobArtifactId : exportManagerQueueRecord.ExportJobArtifactId);
if (numberOfWorkerRecords == 0)
{
FileInfo exportFileInfo = await ConstructExportFileNameAsync(exportManagerQueueRecord.ObjectType, exportManagerQueueRecord.ExportJobArtifactId);
//create export file
Int32 numberOfRecordsExported = await CreateExportFileAsync(exportManagerQueueRecord, exportFileInfo);
//attach export file to RDO
await AttachFileToExportJobAsync(exportManagerQueueRecord, exportFileInfo.FullName);
//delete export file from temp directory
await DeleteExportFileAsync(exportFileInfo);
//drop results table
await DropExportWorkerResultsTableAsync(exportManagerQueueRecord.ResultsTableName);
//delete record from export manager queue
await ClearRecordsFromQueueTables(exportManagerQueueRecord);
Boolean jobErrorsRecorded = await ArtifactQueries.JobErrorRecordExistsAsync(RsapiApiOptions, exportManagerQueueRecord.WorkspaceArtifactId, RsapiRepositoryGroup.RdoRepository, Constant.Guids.ObjectType.ExportUtilityJobErrors, Constant.Guids.Field.ExportUtilityJobErrors.ExportUtilityJob, exportManagerQueueRecord.ExportJobArtifactId);
await FinalizeJobStatiaticsAsync(exportManagerQueueRecord, numberOfRecordsExported);
//update status of export job to complete
if (jobErrorsRecorded)
{
await UpdateExportJobStatus(Constant.Status.Job.COMPLETED_WITH_ERRORS, exportManagerQueueRecord);
await SendEmail(exportManagerQueueRecord, Constant.Status.Job.COMPLETED_WITH_ERRORS);
}
else
{
await UpdateExportJobStatus(Constant.Status.Job.COMPLETED, exportManagerQueueRecord);
await SendEmail(exportManagerQueueRecord, Constant.Status.Job.COMPLETED);
}
}
}
catch (Exception ex)
{
throw new AdminMigrationUtilityException(Constant.ErrorMessages.CheckIfExportWorkersAreStillWorkingOnExportJobInExportWorkerQueueTableError, ex);
}
}
}
private async Task <Boolean> ProcessEachLineAsync(ImportFileRecord importFileRecord)
{
Boolean retVal = false;
try
{
if (importFileRecord != null)
{
int resourceGroupId = await WorkspaceQueryHelper.GetResourcePoolAsync(AgentHelper.GetServicesManager(), ExecutionIdentity.System, WorkspaceArtifactId);
//get selected markup sub types in the import job
List <int> selectedMarkupSubTypes = _markupUtilityImportJob
.MarkupUtilityTypes
.Select(x =>
MarkupTypeHelper.GetMarkupSubTypeValueAsync(x.Name).Result)
.ToList();
//insert into import manager holding table only if the sub type is selected in the import job
if (selectedMarkupSubTypes.Contains(importFileRecord.MarkupSubType))
{
await QueryHelper.InsertRowIntoImportManagerHoldingTableAsync(
AgentHelper.GetDBContext(-1),
WorkspaceArtifactId,
importFileRecord.DocumentIdentifier,
importFileRecord.FileOrder,
resourceGroupId,
_markupUtilityImportJob.ArtifactId,
_markupUtilityImportJob.MarkupSetArtifactId,
_markupUtilityImportJob.JobType,
importFileRecord,
_markupUtilityImportJob.SkipDuplicateRedactions,
ImportManagerHoldingTable);
_expectedRedactionCount++;
}
retVal = true;
}
}
catch (Exception ex)
{
throw new MarkupUtilityException("An error occured when inserting redaction record to the import manager holding table.", ex);
}
finally
{
_importFileRedactionCount++;
if (_importFileRedactionCount % 100 == 0)
{
RaiseMessage($"Parsed {_importFileRedactionCount} records in import file.");
}
}
return(retVal);
}
private async Task <List <ImportManagerQueueRecord> > GetAllManagerJobsAsync()
{
var retVal = new List <ImportManagerQueueRecord>();
var dt = await SqlQueryHelper.RetrieveAllImportManagerQueueRecords(AgentHelper.GetDBContext(-1));
foreach (DataRow row in dt.Rows)
{
retVal.Add(new ImportManagerQueueRecord(row));
}
return(retVal);
}
public async Task <Int32?> CurrentQueueRecordStatus(ExportManagerQueueRecord exportManagerQueueRecord)
{
return(await SqlQueryHelper.GetQueueRecordCurrentStatus(
eddsDbContext : AgentHelper.GetDBContext(-1),
workspaceIDColumnName : Constant.Sql.ColumnsNames.ExportManagerQueue.WorkspaceArtifactId,
workspaceArtifactID : exportManagerQueueRecord.WorkspaceArtifactId,
JobIDColumnName : Constant.Sql.ColumnsNames.ExportManagerQueue.ExportJobArtifactId,
jobID : exportManagerQueueRecord.ExportJobArtifactId,
tableName : Constant.Tables.ExportManagerQueue,
statusColumnName : Constant.Sql.ColumnsNames.ExportManagerQueue.QueueStatus));
}
private async Task GetOffHoursTimesAsync()
{
var dt = await QueryHelper.RetrieveOffHoursAsync(AgentHelper.GetDBContext(-1));
if (dt?.Rows == null || dt.Rows.Count == 0 || string.IsNullOrEmpty(dt.Rows[0]["AgentOffHourStartTime"].ToString()) || string.IsNullOrEmpty(dt.Rows[0]["AgentOffHourEndTime"].ToString()) || dt.Rows[0]["AgentOffHourStartTime"] == null || dt.Rows[0]["AgentOffHourEndTime"] == null)
{
throw new Helpers.Exceptions.MarkupUtilityException(Constant.Messages.AGENT_OFF_HOURS_NOT_FOUND);
}
OffHoursStartTime = dt.Rows[0]["AgentOffHourStartTime"].ToString();
OffHoursEndTime = dt.Rows[0]["AgentOffHourEndTime"].ToString();
}
public async Task DropExportWorkerResultsTableAsync(String exportWorkerResultsTableName)
{
try
{
await SqlQueryHelper.DropTableAsync(AgentHelper.GetDBContext(-1), $"[{exportWorkerResultsTableName}]");
}
catch (Exception ex)
{
throw new AdminMigrationUtilityException(Constant.ErrorMessages.DropExportWorkerResultsTableError, ex);
}
}
private async Task <IEnumerable <int> > UpdateHasRedactionsOrHighlightsAsync(ReproduceWorkerQueueRecord reproduceWorkerQueueRecord, int destinationMarkupSet, bool relationalGroup)
{
RaiseMessage($"Updating documents with Has Redactions/Highligths for worker record id {reproduceWorkerQueueRecord.RecordId}. {_errorContext}");
Task <int> redactions;
Task <int> highlights;
if (relationalGroup)
{
redactions = QueryHelper.UpdateHasRedactionsOrHighlightsAsync(
AgentHelper.GetDBContext(reproduceWorkerQueueRecord.WorkspaceArtifactId),
reproduceWorkerQueueRecord.RedactionCodeTypeId,
reproduceWorkerQueueRecord.MarkupSetRedactionCodeArtifactId, Constant.MarkupType.Redaction.VALUE,
reproduceWorkerQueueRecord.SavedSearchHoldingTable,
reproduceWorkerQueueRecord.RedactionsHoldingTable,
destinationMarkupSet, reproduceWorkerQueueRecord.RelationalGroup);
highlights = QueryHelper.UpdateHasRedactionsOrHighlightsAsync(
AgentHelper.GetDBContext(reproduceWorkerQueueRecord.WorkspaceArtifactId),
reproduceWorkerQueueRecord.RedactionCodeTypeId,
reproduceWorkerQueueRecord.MarkupSetAnnotationCodeArtifactId,
Constant.MarkupType.Highlight.VALUE,
reproduceWorkerQueueRecord.SavedSearchHoldingTable,
reproduceWorkerQueueRecord.RedactionsHoldingTable,
destinationMarkupSet, reproduceWorkerQueueRecord.RelationalGroup);
}
else
{
redactions = QueryHelper.UpdateHasRedactionsOrHighlightsAsync(
AgentHelper.GetDBContext(reproduceWorkerQueueRecord.WorkspaceArtifactId),
reproduceWorkerQueueRecord.RedactionCodeTypeId,
reproduceWorkerQueueRecord.MarkupSetRedactionCodeArtifactId, Constant.MarkupType.Redaction.VALUE,
reproduceWorkerQueueRecord.SavedSearchHoldingTable,
reproduceWorkerQueueRecord.RedactionsHoldingTable,
destinationMarkupSet, reproduceWorkerQueueRecord.DocumentIdStart,
reproduceWorkerQueueRecord.DocumentIdEnd);
highlights = QueryHelper.UpdateHasRedactionsOrHighlightsAsync(
AgentHelper.GetDBContext(reproduceWorkerQueueRecord.WorkspaceArtifactId),
reproduceWorkerQueueRecord.RedactionCodeTypeId,
reproduceWorkerQueueRecord.MarkupSetAnnotationCodeArtifactId,
Constant.MarkupType.Highlight.VALUE,
reproduceWorkerQueueRecord.SavedSearchHoldingTable,
reproduceWorkerQueueRecord.RedactionsHoldingTable,
destinationMarkupSet, reproduceWorkerQueueRecord.DocumentIdStart,
reproduceWorkerQueueRecord.DocumentIdEnd);
}
var ints = await Task.WhenAll(redactions, highlights);
RaiseMessage($"Updated {string.Join("/", ints)} Redaction/Highlight records with Has Redactions/Highligths for worker record id {reproduceWorkerQueueRecord.RecordId}. {_errorContext}");
return(ints);
}
public async Task ResetUnfinishedExportManagerJobsAsync()
{
try
{
RaiseAndLogDebugMessage($"Resetting records which failed in Export Manager Queue Table.");
await SqlQueryHelper.ResetUnfishedExportManagerJobsAsync(AgentHelper.GetDBContext(-1), AgentId);
}
catch (Exception ex)
{
throw new AdminMigrationUtilityException(Constant.ErrorMessages.ResetUnfinishedJobsInExportManagerQueueTableError, ex);
}
}
public async Task DropExportWorkerBatchTableAsync()
{
try
{
await SqlQueryHelper.DropTableAsync(
eddsDbContext : AgentHelper.GetDBContext(-1),
tableName : BatchTableName);
}
catch (Exception ex)
{
throw new AdminMigrationUtilityException(Constant.ErrorMessages.DropExportWorkerBatchTableError, ex);
}
}
private async Task CreateAuditRecordAsync(DataRow datarow, MarkupUtilityReproduceJob job, int documentArtifactId)
{
RaiseMessage($"creating audit record. {_errorContext}");
await _auditRecordHelper.CreateRedactionAuditRecordAsync(
AgentHelper.GetDBContext(WorkspaceArtifactId),
Constant.AuditRecord.AuditAction.REDACTION_CREATED,
documentArtifactId,
job.CreatedBy,
(string)datarow["FileGuid"],
(int)datarow["ID"],
(int)datarow["MarkupSetArtifactID"],
(int)datarow["Order"] + 1);
}
private async Task LogErrorAsync(Exception ex)
{
//Add the error to our custom Errors table
await QueryHelper.InsertRowIntoJobErrorLogAsync(
AgentHelper.GetDBContext(-1),
WorkspaceArtifactId,
QueueTable,
RecordId,
AgentId,
ex.ToString(), Constant.Tables.ReproduceErrorLog);
//Add the error to the Relativity Errors tab
ErrorQueries.WriteError(AgentHelper.GetServicesManager(), ExecutionIdentity.System, WorkspaceArtifactId, ex);
}
