private async Task <int> ReproduceAcrossRelationalGroup(ReproduceManagerQueueRecord reproduceManagerQueueRecord, MarkupUtilityReproduceJob reproduceJob, int codeTypeId, int markupSetRedactionCodeArtifactId, int markupSetAnnotationCodeArtifactId)
{
var dataTable = await RetrieveRelationalGroupsAsync(reproduceManagerQueueRecord.WorkspaceArtifactId);
//create a new record for each relational group
var insertTasks = new List <Task>();
foreach (DataRow row in dataTable.Rows)
{
var relationalGroup = (string)row["RelationalGroup"];
insertTasks.Add(InsertRecordIntoToReproduceWorkerQueueAsync(
reproduceManagerQueueRecord.WorkspaceArtifactId,
-1,
-1,
-1,
reproduceJob.ArtifactId,
reproduceManagerQueueRecord.ResourceGroupId, codeTypeId, markupSetRedactionCodeArtifactId,
markupSetAnnotationCodeArtifactId, reproduceJob.RelationalFieldColumnName,
reproduceJob.HasAutoRedactionsFieldColumnName, relationalGroup));
}
await Task.WhenAll(insertTasks.ToArray());
return(insertTasks.Count);
}
private async Task <MarkupUtilityReproduceJob> RetrieveReproduceJobAsync(ReproduceManagerQueueRecord reproduceManagerQueueRecord)
{
return(await _artifactQueries.RetrieveReproduceJobAsync(
AgentHelper.GetServicesManager(),
ExecutionIdentity.CurrentUser,
WorkspaceArtifactId,
reproduceManagerQueueRecord.ReproduceJobArtifactId));
}
public void ReproduceManagerQueueRecordInitTest()
{
var table = new DataTable();
table.Columns.Add("WorkspaceArtifactID", typeof(int));
table.Columns.Add("ID", typeof(int));
table.Columns.Add("ReproduceJobArtifactID", typeof(int));
table.Columns.Add("ResourceGroupID", typeof(int));
table.Rows.Add(1, 2, 3, 4);
var reproduceManagerQueueRecord = new ReproduceManagerQueueRecord(table.Rows[0]);
Assert.AreEqual(1, reproduceManagerQueueRecord.WorkspaceArtifactId);
Assert.AreEqual(2, reproduceManagerQueueRecord.RecordId);
Assert.AreEqual(3, reproduceManagerQueueRecord.ReproduceJobArtifactId);
Assert.AreEqual(4, reproduceManagerQueueRecord.ResourceGroupId);
Assert.Throws <ArgumentNullException>(delegate { new ReproduceManagerQueueRecord(null); });
}
private async Task <int> ReproduceAcrossDocumentSet(ReproduceManagerQueueRecord reproduceManagerQueueRecord, MarkupUtilityReproduceJob reproduceJob, int min, int max, int codeTypeId, int markupSetRedactionCodeArtifactId, int markupSetAnnotationCodeArtifactId)
{
//create Worker records splitting the records
var insertTasks = new List <Task>();
for (var i = min; i <= max; i = i + Constant.Sizes.ReproduceJobInsertBatchSize)
{
insertTasks.Add(InsertRecordIntoToReproduceWorkerQueueAsync(
reproduceManagerQueueRecord.WorkspaceArtifactId,
i,
i + Constant.Sizes.ReproduceJobInsertBatchSize - 1,
reproduceJob.DestinationMarkupSetArtifactId,
reproduceJob.ArtifactId,
reproduceManagerQueueRecord.ResourceGroupId, codeTypeId, markupSetRedactionCodeArtifactId,
markupSetAnnotationCodeArtifactId, null, reproduceJob.HasAutoRedactionsFieldColumnName, null));
}
await Task.WhenAll(insertTasks.ToArray());
return(insertTasks.Count);
}
public override async Task ExecuteAsync()
{
try
{
//Check for jobs which stopped unexpectedly on this agent thread
RaiseMessage($"Resetting records which failed. [Table = {QueueTable}]");
await ResetUnfishedJobsAsync(AgentHelper.GetDBContext(-1));
//Retrieve the next record to work on
RaiseMessage($"Retrieving next record(s) in the queue. [Table = {QueueTable}]");
var delimitedListOfResourceGroupIds = GetCommaDelimitedListOfResourceIds(AgentResourceGroupIds);
if (delimitedListOfResourceGroupIds != string.Empty)
{
var next = await RetrieveNextAsync(delimitedListOfResourceGroupIds);
if (TableIsNotEmpty(next))
{
var reproduceManagerQueueRecord = new ReproduceManagerQueueRecord(next.Rows[0]);
// Sets the workspaceArtifactId and RecordID so the agent will have access to them in case of an exception
WorkspaceArtifactId = reproduceManagerQueueRecord.WorkspaceArtifactId;
RecordId = reproduceManagerQueueRecord.RecordId;
RaiseMessage(
$"Retrieved record(s) in the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
var reproduceJob = await RetrieveReproduceJobAsync(reproduceManagerQueueRecord);
//Check for Cancellation RequestGetDBContext
if (reproduceJob != null && reproduceJob.Status != Constant.Status.Job.CANCELREQUESTED)
{
//Process the record(s)
await ProcessRecordsAsync(reproduceManagerQueueRecord, reproduceJob);
//Remove the record from the manager queue
await FinishAsync();
}
else
{
//Remove the record from the manager queue
await FinishAsync();
if (reproduceJob != null)
{
//Set the status of the Export Job to Cancelled
await UpdateStatusFieldAsync(reproduceManagerQueueRecord.ReproduceJobArtifactId,
Constant.Status.Job.CANCELLED);
//Flush the Details of the Export Job
await UpdateDetailsFieldAsync(reproduceManagerQueueRecord.ReproduceJobArtifactId,
string.Empty);
}
}
}
else
{
RaiseMessage(Constant.AgentRaiseMessages.NO_RECORDS_IN_QUEUE_FOR_THIS_RESOURCE_POOL);
}
}
else
{
RaiseMessage(Constant.AgentRaiseMessages.AGENT_SERVER_NOT_PART_OF_ANY_RESOURCE_POOL);
}
}
catch (Exception ex)
{
//log error
RaiseMessage($"Logging error.");
await LogErrorAsync(ex);
}
finally
{
//Remove the record from the manager queue
await FinishAsync();
//Delete Holding Tables
await DeleteSavedSearchHoldingTableAsync();
await DeleteRedactionsHoldingTableAsync();
}
}
public async Task ProcessRecordsAsync(ReproduceManagerQueueRecord reproduceManagerQueueRecord, MarkupUtilityReproduceJob reproduceJob)
{
RaiseMessage($"Processing record(s). [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
try
{
//Update status of the Job to Manager In Progress
await UpdateStatusFieldAsync(reproduceManagerQueueRecord.ReproduceJobArtifactId, Constant.Status.Job.IN_PROGRESS_MANAGER);
//Flush the Details of the Job
await UpdateDetailsFieldAsync(reproduceManagerQueueRecord.ReproduceJobArtifactId, string.Empty);
var includeRelationalGroup = reproduceJob.RelationalField > 0;
if (reproduceJob.HasAutoRedactionsField > 0)
{
var columnName = await RetrieveDocumentColumnAsync(reproduceManagerQueueRecord.WorkspaceArtifactId, reproduceJob.HasAutoRedactionsField);
reproduceJob.HasAutoRedactionsFieldColumnName = columnName;
}
//Create holding tables
if (includeRelationalGroup)
{
await CreateSavedSearchHoldingTableAsync(reproduceManagerQueueRecord.WorkspaceArtifactId, true);
var relationalColumnName = await RetrieveDocumentColumnAsync(reproduceManagerQueueRecord.WorkspaceArtifactId, reproduceJob.RelationalField);
reproduceJob.RelationalFieldColumnName = relationalColumnName;
}
else
{
await CreateSavedSearchHoldingTableAsync(reproduceManagerQueueRecord.WorkspaceArtifactId, false);
}
await CreateRedactionsHoldingTableAsync(reproduceManagerQueueRecord.WorkspaceArtifactId, includeRelationalGroup);
//Add Documents from Saved Search in the Export Worker Queue
var isSuccessful = await AddDocumentsToSavedSearchHoldingTableAsync(reproduceManagerQueueRecord.WorkspaceArtifactId, reproduceJob);
var jobIsComplete = false;
var jobMessage = "";
if (isSuccessful)
{
var dataTable = await RetrieveMinMaxIdAsync(reproduceManagerQueueRecord.WorkspaceArtifactId);
var min = (int)dataTable.Rows[0][0];
var max = (int)dataTable.Rows[0][1];
var tasks = new List <Task <int> >();
for (var i = min; i <= max; i = i + Constant.Sizes.ReproduceJobBatchSize)
{
//sql between has inclusive end
tasks.Add(AddDocumentsToRedactionsHoldingTableAsync(reproduceManagerQueueRecord.WorkspaceArtifactId, reproduceJob.SourceMarkupSetArtifactId, reproduceJob.DestinationMarkupSetArtifactId, i, i + Constant.Sizes.ReproduceJobBatchSize - 1, includeRelationalGroup));
}
var results = await Task.WhenAll(tasks.ToArray());
var existRedactions = Array.Exists(results, r => r > 0);
if (existRedactions)
{
var result = await RetrieveZCodesAsync(reproduceManagerQueueRecord.WorkspaceArtifactId, reproduceJob.DestinationMarkupSetArtifactId);
var codeTypeId = (int)result.Rows[0]["CodeTypeID"];
var markupSetRedactionCodeArtifactId = (int)result.Rows[0]["RedactionCodeArtifactID"];
var markupSetAnnotationCodeArtifactId = (int)result.Rows[0]["AnnotationCodeArtifactID"];
int count;
if (reproduceJob.RelationalField > 0)
{
count = await ReproduceAcrossRelationalGroup(reproduceManagerQueueRecord, reproduceJob, codeTypeId, markupSetRedactionCodeArtifactId, markupSetAnnotationCodeArtifactId);
}
else
{
count = await ReproduceAcrossDocumentSet(reproduceManagerQueueRecord, reproduceJob, min, max, codeTypeId, markupSetRedactionCodeArtifactId, markupSetAnnotationCodeArtifactId);
}
if (count == 0)
{
jobIsComplete = true;
jobMessage = "No redactions found for selected criteria.";
}
}
else
{
jobIsComplete = true;
jobMessage = "No redactions found for selected criteria.";
}
}
else
{
jobIsComplete = true;
jobMessage = "No Documents found in the selected Saved Search.";
}
if (jobIsComplete)
{
var tasks = new List <Task>
{
UpdateStatusFieldAsync(reproduceManagerQueueRecord.ReproduceJobArtifactId, Constant.Status.Job.COMPLETED), UpdateDetailsFieldAsync(reproduceManagerQueueRecord.ReproduceJobArtifactId, jobMessage), QueryHelper.DropTableAsync(AgentHelper.GetDBContext(reproduceManagerQueueRecord.WorkspaceArtifactId), RedactionsHoldingTable), QueryHelper.DropTableAsync(AgentHelper.GetDBContext(reproduceManagerQueueRecord.WorkspaceArtifactId), SavedSearchHoldingTable)
};
//Update status of the Export Job to Completed
//Update Details of the Export Job indicating no Documents in Saved Search
await Task.WhenAll(tasks);
}
else
{
//Update status of the Export Job to Completed - Manager
await UpdateStatusFieldAsync(reproduceManagerQueueRecord.ReproduceJobArtifactId, Constant.Status.Job.COMPLETED_MANAGER);
//Flush the Details of the Export Job
await UpdateDetailsFieldAsync(reproduceManagerQueueRecord.ReproduceJobArtifactId, string.Empty);
}
//delete import job from queue
RaiseMessage($"Removing record(s) from the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
await FinishAsync();
await DeleteSavedSearchHoldingTableAsync();
await DeleteRedactionsHoldingTableAsync();
RaiseMessage($"Removed record(s) from the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
RaiseMessage($"Processed record(s). [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
}
catch (Exception ex)
{
//Update status of the Export Job to Error
await UpdateStatusFieldAsync(reproduceManagerQueueRecord.ReproduceJobArtifactId, Constant.Status.Job.ERROR);
//Update Details of the Export Job
await UpdateDetailsFieldAsync(reproduceManagerQueueRecord.ReproduceJobArtifactId, ex.ToString());
//log error
await LogErrorAsync(ex);
//delete import job from queue
RaiseMessage($"Removing record(s) from the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
await FinishAsync();
await DeleteSavedSearchHoldingTableAsync();
await DeleteRedactionsHoldingTableAsync();
RaiseMessage($"Removed record(s) from the queue. [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
RaiseMessage($"Processed record(s). [Table = {QueueTable}, ID = {RecordId}, Workspace Artifact ID = {WorkspaceArtifactId}]");
}
}
