public override void Execute() { //retrieve ResourceServerId for agent for checking resource pool. this value will be used when retrieving records from queue ResourceServerId = SqlQueryHelper.GetResourceServerByAgentId(EddsDbContext, AgentId); if (ResourceServerId == 0) { TextExtractorLog.RaiseUpdate(string.Format("Resource Server for Agent ID {0} cannot be detected.", AgentId)); return; } TextExtractorLog.RaiseUpdate("Processing Worker Queue Batch."); var workerQueue = new WorkerQueue(SqlQueryHelper, ArtifactQueries, ArtifactFactory, EddsDbContext, ServicesMgr, ExecutionIdentity, AgentId, ResourceServerId, BatchTableName, TextExtractorLog, TextExtractorJobReporting); if (workerQueue.HasRecords) { WorkspaceArtifactId = workerQueue.WorkspaceArtifactId; var extractorSet = ArtifactFactory.GetInstanceOfExtractorSet(ExecutionIdentity.CurrentUser, workerQueue.WorkspaceArtifactId, workerQueue.ExtractorSetArtifactId); //check for ExtractorSet cancellation Boolean isCancelled = CheckForExtractorSetCancellation(extractorSet, true); if (!isCancelled) { //process worker queue records in current batch workerQueue.ProcessAllRecords(); //check for ExtractorSet cancellation CheckForExtractorSetCancellation(extractorSet, false); } } TextExtractorLog.RaiseUpdate("Worker Queue Batch processed."); }
public AgentJobExceptionWrapper(ISqlQueryHelper sqlQueryHelper, IArtifactQueries artifactQueries, IServicesMgr servicesMgr, IDBContext eddsDbContext, int agentId) { SqlQueryHelper = sqlQueryHelper; ArtifactQueries = artifactQueries; ServicesMgr = servicesMgr; EddsDbContext = eddsDbContext; AgentId = agentId; TextExtractorLog = new TextExtractorLog(); }
public override void Execute() { var sqlQueryHelper = new SqlQueryHelper(); var artifactQueries = new ArtifactQueries(); var errorLogModel = new ErrorLogModel(sqlQueryHelper, Helper.GetDBContext(-1), AgentID, Constant.Tables.WorkerQueue); var uniqueBatchTableName = "[" + Constant.Names.TablePrefix + "Worker_" + Guid.NewGuid() + "_" + AgentID + "]"; var artifactFactory = new ArtifactFactory(artifactQueries, Helper.GetServicesManager(), errorLogModel); var agentWorkerJob = new WorkerJob(AgentID, Helper.GetServicesManager(), ExecutionIdentity.CurrentUser, new SqlQueryHelper(), new ArtifactQueries(), artifactFactory, Constant.AgentType.Worker, uniqueBatchTableName, Helper.GetDBContext(-1)); var textExtractorLog = new TextExtractorLog(); textExtractorLog.OnUpdate += MessageRaised; agentWorkerJob.TextExtractorLog = textExtractorLog; try { RaiseMessage("Enter Agent", 10); agentWorkerJob.Execute(); RaiseMessage("Exit Agent", 10); } catch (Exception ex) { //Raise an error on the agents tab and event viewer RaiseError(ex.ToString(), ex.ToString()); //Add the error to our custom Errors table sqlQueryHelper.InsertRowIntoErrorLog(Helper.GetDBContext(-1), agentWorkerJob.WorkspaceArtifactId, Constant.Tables.WorkerQueue, agentWorkerJob.RecordId, agentWorkerJob.AgentId, ex.ToString()); //Add the error to the Relativity Errors tab //this second try catch is in case we have a problem connecting to the RSAPI try { ErrorQueries.WriteError(Helper.GetServicesManager(), ExecutionIdentity.CurrentUser, agentWorkerJob.WorkspaceArtifactId, ex); } catch (Exception rsapiException) { RaiseError(rsapiException.ToString(), rsapiException.ToString()); } //Set the status in the queue to error if (uniqueBatchTableName != string.Empty) { sqlQueryHelper.UpdateStatusInWorkerQueue(Helper.GetDBContext(-1), Constant.QueueStatus.Error, uniqueBatchTableName); } } finally { textExtractorLog.RaiseUpdate("Dropping Worker Queue Batch table."); //drop temp table sqlQueryHelper.DropTable(Helper.GetDBContext(-1), uniqueBatchTableName); } }
private Boolean CheckForExtractorSetCancellation(ExtractorSet extractorSet, Boolean deleteCurrentWorkerQueueBatch) { Boolean retVal = false; if (extractorSet.IsCancellationRequested()) { TextExtractorLog.RaiseUpdate("Cancellation Requested."); //Delete records in worker queue for current ExtractorSet which is cancelled and are assigned to current agent if (deleteCurrentWorkerQueueBatch) { SqlQueryHelper.DeleteRecordsInWorkerQueueForCancelledExtractorSetAndAgentId(EddsDbContext, WorkspaceArtifactId, extractorSet.ArtifactId, AgentId); } //Delete records in worker queue for current ExtractorSet which is cancelled and are not assigned to other agents SqlQueryHelper.DeleteRecordsInWorkerQueueForCancelledExtractorSet(EddsDbContext, WorkspaceArtifactId, extractorSet.ArtifactId); retVal = true; } return(retVal); }
public void ProcessAllRecordsTest() { IntegrationSettings testSettings = new IntegrationSettings(); IntegrationDBContext testDBContext = new IntegrationDBContext(testSettings.DBContextSettings); IntegrationServicesMgr testServiceManager = new IntegrationServicesMgr(testSettings.RsapiSettings); SqlQueryHelper testSqlQueryHelpers = new SqlQueryHelper(); ArtifactQueries artifactQueries = new ArtifactQueries(); ArtifactFactory artifactFactory = new ArtifactFactory(artifactQueries, testServiceManager, null); Int32 agentId = 1016595; Int32 resourceServerId = 1016158; string batchTableName = "[" + Constant.Names.TablePrefix + "Worker_" + Guid.NewGuid() + "_" + agentId + "]"; TextExtractorLog textExtractorLog = new TextExtractorLog(); ExtractorSetReporting extractorSetReporting = new ExtractorSetReporting(artifactQueries, testServiceManager); WorkerQueue workerQueue = new WorkerQueue(testSqlQueryHelpers, artifactQueries, artifactFactory, testDBContext, testServiceManager, ExecutionIdentity.System, agentId, resourceServerId, batchTableName, textExtractorLog, extractorSetReporting); workerQueue.ProcessAllRecords(); }
public override void Execute() { var managerQueue = GetManagerQueue(); // Reset unfinished jobs TextExtractorLog.RaiseUpdate(string.Format("Resetting records which failed. [Table = {0}]", Constant.Tables.ManagerQueue)); managerQueue.ResetUnfinishedRecords(); // Retrieve the records in the manager queue TextExtractorLog.RaiseUpdate("Retrieving next batch of records in the manager queue."); managerQueue.GetNextBatchOfRecords(); if (managerQueue.HasRecords == false) { TextExtractorLog.RaiseUpdate("No jobs found in the manager queue."); return; } foreach (ManagerQueueRecord record in managerQueue.Records) { try { var extractorSet = GetExtractorSet(record.ExtractorSetArtifactId, record.WorkspaceArtifactId); // Checks to see if the user's cancelled the job between the time it was entered // and agent picked it up if (extractorSet.IsCancellationRequested()) { TextExtractorLog.RaiseUpdate("Job was cancelled by user."); record.Remove(); continue; } // Process the record TextExtractorLog.RaiseUpdate(String.Format("Processing record. [Table = {0}, ExtractorSetArtifactID = {1}, Workspace Artifact ID = {2}]", Constant.Tables.ManagerQueue, record.ExtractorSetArtifactId, record.WorkspaceArtifactId)); if (extractorSet.Exists == false) { TextExtractorLog.RaiseUpdate("No Text Extraction Job found."); continue; } // Sets the job status to processing extractorSet.UpdateStatus(Constant.ExtractorSetStatus.IN_PROGRESS_MANAGER_PROCESSING); var documentBatch = GetDocumentBatch(); // Gets document artifact IDs from the saved search and bulk copies them into // the worker queue do { if (HasReachedInterval(documentBatch.Index)) { // Check for cancellation if (extractorSet.IsCancellationRequested()) { // Stop inserting any more records into the Worker queue record.Remove(); break; } } var nextBatch = documentBatch.GetNext(record.WorkspaceArtifactId, record.SavedSearchArtifactId); // End execution if there's a failure, so the job will be re-tried when it's // picked up next if (nextBatch == null || nextBatch.TotalCount == 0) { //Saved Search is emnpty TextExtractorLog.RaiseUpdate((String.Format("Failed to retrieve a batch of documents for saved search {0}", record.SavedSearchArtifactId))); extractorSet.UpdateStatus(Constant.ExtractorSetStatus.COMPLETE_WITH_ERRORS); extractorSet.UpdateDetails(Constant.ErrorMessages.SAVED_SEARCH_IS_EMPTY); record.Remove(); break; } var batchTable = documentBatch.ConvertToWorkerQueueTable(nextBatch, record); // End execution if there's a failure, so the job will be re-tried when it's // picked up next if (batchTable == null) { TextExtractorLog.RaiseUpdate("Failed to convert values into worker queue table"); extractorSet.UpdateStatus(Constant.ExtractorSetStatus.ERROR); break; } var recordsAddedToWorkerQueue = managerQueue.AddRecordsToWorkerQueue(batchTable); if (!recordsAddedToWorkerQueue) { TextExtractorLog.RaiseUpdate("Failed to add records to the worker queue"); } }while (documentBatch.AreMoreBatches); // If the extractorSet was cancelled while processing, will break to this and // continue to the next record if (extractorSet.IsCancellationRequested()) { TextExtractorLog.RaiseUpdate("Job was cancelled by user."); record.Remove(); continue; } if (documentBatch.Total == 0) { continue; //Saved Search is emnpty } TextExtractorLog.RaiseUpdate(String.Format("Processed record. [Table = {0}, ExtractorSetArtifactID = {1}, Workspace Artifact ID = {2}]", Constant.Tables.ManagerQueue, record.ExtractorSetArtifactId, record.WorkspaceArtifactId)); // Update Total Expected Update Count on Reporting RDO var extractorProfileRdo = ArtifactQueries.GetExtractorProfileRdo(ServicesMgr, ExecutionIdentity.CurrentUser, record.WorkspaceArtifactId, record.ExtractorProfileArtifactId); var targetTexts = extractorProfileRdo.Fields.Get(Constant.Guids.Fields.ExtractorProfile.TargetText).GetValueAsMultipleObject <Artifact>(); var extractorProfileTargetCount = (targetTexts != null && targetTexts.Count > 0) ? targetTexts.Count : 0; extractorSet.ExtractorSetReporting.UpdateTotalExpectedUpdates(record.WorkspaceArtifactId, record.ExtractorSetArtifactId, documentBatch.Total, extractorProfileTargetCount); //Remove the record from the manager queue TextExtractorLog.RaiseUpdate(String.Format("Removing record from the queue. [Table = {0}, ExtractorSetArtifactID = {1}, Workspace Artifact ID = {2}]", Constant.Tables.ManagerQueue, record.ExtractorSetArtifactId, record.WorkspaceArtifactId)); // Remove all records associated to this agent managerQueue.RemoveRecordsByAgent(); TextExtractorLog.RaiseUpdate(String.Format("Removed record from the queue. [Table = {0}, ExtractorSetArtifactID = {1}, Workspace Artifact ID = {2}]", Constant.Tables.ManagerQueue, record.ExtractorSetArtifactId, record.WorkspaceArtifactId)); // Update the status of job in Relativity extractorSet.UpdateStatus(Constant.ExtractorSetStatus.IN_PROGRESS_MANAGER_COMPLETE); } // Catches any exception thrown in this record, sets it to error, and processes the next catch (Exception exception) { TextExtractorLog.RaiseUpdate(exception.Message); record.Update(Constant.QueueStatus.Error); } } }