public override void Execute()
{
//retrieve ResourceServerId for agent for checking resource pool. this value will be used when retrieving records from queue
ResourceServerId = SqlQueryHelper.GetResourceServerByAgentId(EddsDbContext, AgentId);
if (ResourceServerId == 0)
{
TextExtractorLog.RaiseUpdate(string.Format("Resource Server for Agent ID {0} cannot be detected.", AgentId));
return;
}
TextExtractorLog.RaiseUpdate("Processing Worker Queue Batch.");
var workerQueue = new WorkerQueue(SqlQueryHelper, ArtifactQueries, ArtifactFactory, EddsDbContext, ServicesMgr, ExecutionIdentity, AgentId, ResourceServerId, BatchTableName, TextExtractorLog, TextExtractorJobReporting);
if (workerQueue.HasRecords)
{
WorkspaceArtifactId = workerQueue.WorkspaceArtifactId;
var extractorSet = ArtifactFactory.GetInstanceOfExtractorSet(ExecutionIdentity.CurrentUser, workerQueue.WorkspaceArtifactId, workerQueue.ExtractorSetArtifactId);
//check for ExtractorSet cancellation
Boolean isCancelled = CheckForExtractorSetCancellation(extractorSet, true);
if (!isCancelled)
{
//process worker queue records in current batch
workerQueue.ProcessAllRecords();
//check for ExtractorSet cancellation
CheckForExtractorSetCancellation(extractorSet, false);
}
}
TextExtractorLog.RaiseUpdate("Worker Queue Batch processed.");
}
public AgentJobExceptionWrapper(ISqlQueryHelper sqlQueryHelper, IArtifactQueries artifactQueries, IServicesMgr servicesMgr, IDBContext eddsDbContext, int agentId)
{
SqlQueryHelper = sqlQueryHelper;
ArtifactQueries = artifactQueries;
ServicesMgr = servicesMgr;
EddsDbContext = eddsDbContext;
AgentId = agentId;
TextExtractorLog = new TextExtractorLog();
}
public override void Execute()
{
var sqlQueryHelper = new SqlQueryHelper();
var artifactQueries = new ArtifactQueries();
var errorLogModel = new ErrorLogModel(sqlQueryHelper, Helper.GetDBContext(-1), AgentID, Constant.Tables.WorkerQueue);
var uniqueBatchTableName = "[" + Constant.Names.TablePrefix + "Worker_" + Guid.NewGuid() + "_" + AgentID + "]";
var artifactFactory = new ArtifactFactory(artifactQueries, Helper.GetServicesManager(), errorLogModel);
var agentWorkerJob = new WorkerJob(AgentID, Helper.GetServicesManager(), ExecutionIdentity.CurrentUser, new SqlQueryHelper(), new ArtifactQueries(), artifactFactory, Constant.AgentType.Worker, uniqueBatchTableName, Helper.GetDBContext(-1));
var textExtractorLog = new TextExtractorLog();
textExtractorLog.OnUpdate += MessageRaised;
agentWorkerJob.TextExtractorLog = textExtractorLog;
try
{
RaiseMessage("Enter Agent", 10);
agentWorkerJob.Execute();
RaiseMessage("Exit Agent", 10);
}
catch (Exception ex)
{
//Raise an error on the agents tab and event viewer
RaiseError(ex.ToString(), ex.ToString());
//Add the error to our custom Errors table
sqlQueryHelper.InsertRowIntoErrorLog(Helper.GetDBContext(-1), agentWorkerJob.WorkspaceArtifactId, Constant.Tables.WorkerQueue, agentWorkerJob.RecordId, agentWorkerJob.AgentId, ex.ToString());
//Add the error to the Relativity Errors tab
//this second try catch is in case we have a problem connecting to the RSAPI
try
{
ErrorQueries.WriteError(Helper.GetServicesManager(), ExecutionIdentity.CurrentUser, agentWorkerJob.WorkspaceArtifactId, ex);
}
catch (Exception rsapiException)
{
RaiseError(rsapiException.ToString(), rsapiException.ToString());
}
//Set the status in the queue to error
if (uniqueBatchTableName != string.Empty)
{
sqlQueryHelper.UpdateStatusInWorkerQueue(Helper.GetDBContext(-1), Constant.QueueStatus.Error, uniqueBatchTableName);
}
}
finally
{
textExtractorLog.RaiseUpdate("Dropping Worker Queue Batch table.");
//drop temp table
sqlQueryHelper.DropTable(Helper.GetDBContext(-1), uniqueBatchTableName);
}
}
private Boolean CheckForExtractorSetCancellation(ExtractorSet extractorSet, Boolean deleteCurrentWorkerQueueBatch)
{
Boolean retVal = false;
if (extractorSet.IsCancellationRequested())
{
TextExtractorLog.RaiseUpdate("Cancellation Requested.");
//Delete records in worker queue for current ExtractorSet which is cancelled and are assigned to current agent
if (deleteCurrentWorkerQueueBatch)
{
SqlQueryHelper.DeleteRecordsInWorkerQueueForCancelledExtractorSetAndAgentId(EddsDbContext, WorkspaceArtifactId, extractorSet.ArtifactId, AgentId);
}
//Delete records in worker queue for current ExtractorSet which is cancelled and are not assigned to other agents
SqlQueryHelper.DeleteRecordsInWorkerQueueForCancelledExtractorSet(EddsDbContext, WorkspaceArtifactId, extractorSet.ArtifactId);
retVal = true;
}
return(retVal);
}
public void ProcessAllRecordsTest()
{
IntegrationSettings testSettings = new IntegrationSettings();
IntegrationDBContext testDBContext = new IntegrationDBContext(testSettings.DBContextSettings);
IntegrationServicesMgr testServiceManager = new IntegrationServicesMgr(testSettings.RsapiSettings);
SqlQueryHelper testSqlQueryHelpers = new SqlQueryHelper();
ArtifactQueries artifactQueries = new ArtifactQueries();
ArtifactFactory artifactFactory = new ArtifactFactory(artifactQueries, testServiceManager, null);
Int32 agentId = 1016595;
Int32 resourceServerId = 1016158;
string batchTableName = "[" + Constant.Names.TablePrefix + "Worker_" + Guid.NewGuid() + "_" + agentId + "]";
TextExtractorLog textExtractorLog = new TextExtractorLog();
ExtractorSetReporting extractorSetReporting = new ExtractorSetReporting(artifactQueries, testServiceManager);
WorkerQueue workerQueue = new WorkerQueue(testSqlQueryHelpers, artifactQueries, artifactFactory, testDBContext, testServiceManager, ExecutionIdentity.System,
agentId, resourceServerId, batchTableName, textExtractorLog, extractorSetReporting);
workerQueue.ProcessAllRecords();
}
public override void Execute()
{
var managerQueue = GetManagerQueue();
// Reset unfinished jobs
TextExtractorLog.RaiseUpdate(string.Format("Resetting records which failed. [Table = {0}]", Constant.Tables.ManagerQueue));
managerQueue.ResetUnfinishedRecords();
// Retrieve the records in the manager queue
TextExtractorLog.RaiseUpdate("Retrieving next batch of records in the manager queue.");
managerQueue.GetNextBatchOfRecords();
if (managerQueue.HasRecords == false)
{
TextExtractorLog.RaiseUpdate("No jobs found in the manager queue.");
return;
}
foreach (ManagerQueueRecord record in managerQueue.Records)
{
try
{
var extractorSet = GetExtractorSet(record.ExtractorSetArtifactId, record.WorkspaceArtifactId);
// Checks to see if the user's cancelled the job between the time it was entered
// and agent picked it up
if (extractorSet.IsCancellationRequested())
{
TextExtractorLog.RaiseUpdate("Job was cancelled by user.");
record.Remove();
continue;
}
// Process the record
TextExtractorLog.RaiseUpdate(String.Format("Processing record. [Table = {0}, ExtractorSetArtifactID = {1}, Workspace Artifact ID = {2}]", Constant.Tables.ManagerQueue, record.ExtractorSetArtifactId, record.WorkspaceArtifactId));
if (extractorSet.Exists == false)
{
TextExtractorLog.RaiseUpdate("No Text Extraction Job found.");
continue;
}
// Sets the job status to processing
extractorSet.UpdateStatus(Constant.ExtractorSetStatus.IN_PROGRESS_MANAGER_PROCESSING);
var documentBatch = GetDocumentBatch();
// Gets document artifact IDs from the saved search and bulk copies them into
// the worker queue
do
{
if (HasReachedInterval(documentBatch.Index))
{
// Check for cancellation
if (extractorSet.IsCancellationRequested())
{
// Stop inserting any more records into the Worker queue
record.Remove();
break;
}
}
var nextBatch = documentBatch.GetNext(record.WorkspaceArtifactId, record.SavedSearchArtifactId);
// End execution if there's a failure, so the job will be re-tried when it's
// picked up next
if (nextBatch == null || nextBatch.TotalCount == 0)
{
//Saved Search is emnpty
TextExtractorLog.RaiseUpdate((String.Format("Failed to retrieve a batch of documents for saved search {0}", record.SavedSearchArtifactId)));
extractorSet.UpdateStatus(Constant.ExtractorSetStatus.COMPLETE_WITH_ERRORS);
extractorSet.UpdateDetails(Constant.ErrorMessages.SAVED_SEARCH_IS_EMPTY);
record.Remove();
break;
}
var batchTable = documentBatch.ConvertToWorkerQueueTable(nextBatch, record);
// End execution if there's a failure, so the job will be re-tried when it's
// picked up next
if (batchTable == null)
{
TextExtractorLog.RaiseUpdate("Failed to convert values into worker queue table");
extractorSet.UpdateStatus(Constant.ExtractorSetStatus.ERROR);
break;
}
var recordsAddedToWorkerQueue = managerQueue.AddRecordsToWorkerQueue(batchTable);
if (!recordsAddedToWorkerQueue)
{
TextExtractorLog.RaiseUpdate("Failed to add records to the worker queue");
}
}while (documentBatch.AreMoreBatches);
// If the extractorSet was cancelled while processing, will break to this and
// continue to the next record
if (extractorSet.IsCancellationRequested())
{
TextExtractorLog.RaiseUpdate("Job was cancelled by user.");
record.Remove();
continue;
}
if (documentBatch.Total == 0)
{
continue; //Saved Search is emnpty
}
TextExtractorLog.RaiseUpdate(String.Format("Processed record. [Table = {0}, ExtractorSetArtifactID = {1}, Workspace Artifact ID = {2}]", Constant.Tables.ManagerQueue, record.ExtractorSetArtifactId, record.WorkspaceArtifactId));
// Update Total Expected Update Count on Reporting RDO
var extractorProfileRdo = ArtifactQueries.GetExtractorProfileRdo(ServicesMgr, ExecutionIdentity.CurrentUser, record.WorkspaceArtifactId, record.ExtractorProfileArtifactId);
var targetTexts = extractorProfileRdo.Fields.Get(Constant.Guids.Fields.ExtractorProfile.TargetText).GetValueAsMultipleObject <Artifact>();
var extractorProfileTargetCount = (targetTexts != null && targetTexts.Count > 0) ? targetTexts.Count : 0;
extractorSet.ExtractorSetReporting.UpdateTotalExpectedUpdates(record.WorkspaceArtifactId, record.ExtractorSetArtifactId, documentBatch.Total, extractorProfileTargetCount);
//Remove the record from the manager queue
TextExtractorLog.RaiseUpdate(String.Format("Removing record from the queue. [Table = {0}, ExtractorSetArtifactID = {1}, Workspace Artifact ID = {2}]", Constant.Tables.ManagerQueue, record.ExtractorSetArtifactId, record.WorkspaceArtifactId));
// Remove all records associated to this agent
managerQueue.RemoveRecordsByAgent();
TextExtractorLog.RaiseUpdate(String.Format("Removed record from the queue. [Table = {0}, ExtractorSetArtifactID = {1}, Workspace Artifact ID = {2}]", Constant.Tables.ManagerQueue, record.ExtractorSetArtifactId, record.WorkspaceArtifactId));
// Update the status of job in Relativity
extractorSet.UpdateStatus(Constant.ExtractorSetStatus.IN_PROGRESS_MANAGER_COMPLETE);
}
// Catches any exception thrown in this record, sets it to error, and processes the next
catch (Exception exception)
{
TextExtractorLog.RaiseUpdate(exception.Message);
record.Update(Constant.QueueStatus.Error);
}
}
}
