private void ProcessJob(IServicesMgr servicesMgr, int workspaceArtifactId, int jobArtifactId) { try { //Update job status to In Progress _apiChooser.UpdateJobField(servicesMgr, workspaceArtifactId, jobArtifactId, Constants.Guids.Fields.InstanceMetricsJob.Status_LongText, Constants.JobStatus.IN_PROGRESS); //Update job metrics if (selectedApiType == Helpers.Constants.ApiType.Rsapi) { RDO jobRdo = _apiChooser.RetrieveJob(servicesMgr, workspaceArtifactId, jobArtifactId); RaiseMessage("Calculating metrics for the job", 10); ProcessAllMetrics(servicesMgr, workspaceArtifactId, jobArtifactId, jobRdo); RaiseMessage("Calculated metrics for the job", 10); } else if (selectedApiType == Helpers.Constants.ApiType.Gravity) { } //Update job status to Completed _apiChooser.UpdateJobField(servicesMgr, workspaceArtifactId, jobArtifactId, Constants.Guids.Fields.InstanceMetricsJob.Status_LongText, Constants.JobStatus.COMPLETED); } catch (Exception ex) { //Update job status to Error string errorMessage = ExceptionMessageFormatter.GetInnerMostExceptionMessage(ex); _apiChooser.UpdateJobField(servicesMgr, workspaceArtifactId, jobArtifactId, Constants.Guids.Fields.InstanceMetricsJob.Status_LongText, Constants.JobStatus.ERROR); _apiChooser.UpdateJobField(servicesMgr, workspaceArtifactId, jobArtifactId, Constants.Guids.Fields.InstanceMetricsJob.Errors_LongText, errorMessage); } }
private void ProcessJob(int workspaceArtifactId, int jobArtifactId) { try { //Update job status to In Progress _rsapiHelper.UpdateJobField(workspaceArtifactId, jobArtifactId, Constants.Guids.Fields.InstanceMetricsJob.Status_LongText, Constants.JobStatus.IN_PROGRESS); //Update job metrics RDO jobRdo = _rsapiHelper.RetrieveJob(workspaceArtifactId, jobArtifactId); RaiseMessage("Calculating metrics for the job", 10); ProcessAllMetrics(workspaceArtifactId, jobArtifactId, jobRdo); RaiseMessage("Calculated metrics for the job", 10); //Update job status to Completed _rsapiHelper.UpdateJobField(workspaceArtifactId, jobArtifactId, Constants.Guids.Fields.InstanceMetricsJob.Status_LongText, Constants.JobStatus.COMPLETED); } catch (Exception ex) { //Update job status to Error string errorMessage = ExceptionMessageFormatter.GetInnerMostExceptionMessage(ex); _rsapiHelper.UpdateJobField(workspaceArtifactId, jobArtifactId, Constants.Guids.Fields.InstanceMetricsJob.Status_LongText, Constants.JobStatus.ERROR); _rsapiHelper.UpdateJobField(workspaceArtifactId, jobArtifactId, Constants.Guids.Fields.InstanceMetricsJob.Errors_LongText, errorMessage); } }
public void ProcessAllRecords() { string[] emailRecepients = null; try { if (HasRecords) { //update job status to worker - in progress TextExtractorLog.RaiseUpdate(String.Format("Updating record status to {0}. [WorkspaceArtifactId: {1}, ExtractorSetArtifactID: {2}]", Constant.ExtractorSetStatus.IN_PROGRESS_WORKER_PROCESSING, WorkspaceArtifactId, ExtractorSetArtifactId)); ArtifactQueries.UpdateRdoStringFieldValue(ServicesMgr, ExecutionIdentity.CurrentUser, WorkspaceArtifactId, Constant.Guids.ObjectType.ExtractorSet, Constant.Guids.Fields.ExtractorSet.Status, ExtractorSetArtifactId, Constant.ExtractorSetStatus.IN_PROGRESS_WORKER_PROCESSING); var countForLogging = 1; foreach (var workerQueueRecord in Records) { TextExtractorLog.RaiseUpdate(String.Format("Processing record - {0}. [WorkspaceArtifactId: {1}, ExtractorSetArtifactID: {2}, DocumentArtifactId: {3}]", countForLogging++, WorkspaceArtifactId, ExtractorSetArtifactId, workerQueueRecord.DocumentArtifactId)); //check for ExtractorSet cancellation var extractorSet = ArtifactFactory.GetInstanceOfExtractorSet(ExecutionIdentity.CurrentUser, WorkspaceArtifactId, ExtractorSetArtifactId); emailRecepients = extractorSet.EmailRecepients; if (extractorSet.IsCancellationRequested()) { TextExtractorLog.RaiseUpdate("Cancellation Requested."); //delete all remaining records in current batch worker queue because the ExtractorSet is cancelled TextExtractorLog.RaiseUpdate("Deleting all the remaining records in current worker queue batch."); SqlQueryHelper.RemoveBatchFromQueue(EddsDbContext, BatchTableName); return; } try { workerQueueRecord.Process(); } catch (Exception ex) { var errorContext = String.Format("An error occured when processing the document. [WorkspaceArtifactId: {0}, ExtractorSetArtifactId: {1}, DocumentArtifactId: {2}]", workerQueueRecord.WorkspaceArtifactId, workerQueueRecord.ExtractorSetArtifactId, workerQueueRecord.DocumentArtifactId); //log error message to ErrorLog table ErrorLogModel.InsertRecord(errorContext, ex, workerQueueRecord.QueueId, workerQueueRecord.WorkspaceArtifactId); //Update TextExtractorErrors field on the Document object incase of error var exceptionMessage = ExceptionMessageFormatter.GetInnerMostExceptionMessage(ex); String extractorSetName = ArtifactQueries.GetExtractorSetNameForArtifactId(ServicesMgr, ExecutionIdentity, WorkspaceArtifactId, workerQueueRecord.ExtractorSetArtifactId); var fieldValue = String.Format(Constant.ErrorMessages.DOCUMENT_ERROR_ENCOUNTERED, exceptionMessage, extractorSetName); ArtifactQueries.AppendToDocumentLongTextFieldValue(ServicesMgr, ExecutionIdentity, workerQueueRecord.WorkspaceArtifactId, workerQueueRecord.DocumentArtifactId, Constant.Guids.Fields.Document.TextExtractorErrors, fieldValue); //Update ExtractorSet Details field ArtifactQueries.UpdateExtractorSetDetails(ServicesMgr, ExecutionIdentity, workerQueueRecord.WorkspaceArtifactId, workerQueueRecord.ExtractorSetArtifactId, Constant.ExtractorSetStatus.DetailMessages.COMPLETE_WITH_ERRORS_DETAILS); } finally { //delete current record from worker queue SqlQueryHelper.RemoveRecordFromWorkerQueue(EddsDbContext, workerQueueRecord.QueueTableName, workerQueueRecord.QueueId); } } } } catch (Exception ex) { throw new CustomExceptions.TextExtractorException("An error occured when processing records in the worker queue.", ex); } finally { VerifyAndUpdateWorkerStatusToComplete(); if (emailRecepients != null) { VerifyAndSendEmailForExtractorSetComplete(emailRecepients); } } }
private void SetMyProperties(RDO extractorSetRdo) { if (extractorSetRdo == null) { Exists = false; return; } try { var savedSearchArtifact = extractorSetRdo.Fields.Get(Constant.Guids.Fields.ExtractorSet.SavedSearch).ValueAsSingleObject; var extractorProfileArtifact = extractorSetRdo.Fields.Get(Constant.Guids.Fields.ExtractorSet.ExtractorProfile).ValueAsSingleObject; var sourceLongTextFieldArtifact = extractorSetRdo.Fields.Get(Constant.Guids.Fields.ExtractorSet.SourceLongTextField).ValueAsSingleObject; String emailRecepients = extractorSetRdo.Fields.Get(Constant.Guids.Fields.ExtractorSet.EmailNotificationRecipients).ValueAsLongText; EmailRecepients = (String.IsNullOrWhiteSpace(emailRecepients) == true) ? null : emailRecepients.Split(';'); SetName = extractorSetRdo.Fields.Get(Constant.Guids.Fields.ExtractorSet.SetName).ValueAsFixedLengthText; ArtifactId = extractorSetRdo.ArtifactID; Status = extractorSetRdo.Fields.Get(Constant.Guids.Fields.ExtractorSet.Status).ValueAsFixedLengthText; SavedSearchArtifactId = savedSearchArtifact.ArtifactID; ExtractorProfileArtifactId = extractorProfileArtifact.ArtifactID; SourceLongTextFieldArtifactId = sourceLongTextFieldArtifact.ArtifactID; Exists = true; } catch (Exception ex) { throw new CustomExceptions.TextExtractorException(Constant.ErrorMessages.DEFAULT_CONVERT_TO_EXTRACTOR_SET_ERROR_MESSAGE + ". " + ExceptionMessageFormatter.GetInnerMostExceptionMessage(ex), ex); } }
private void SetMyProperties(RDO extractorProfileRdo) { try { ProfileName = extractorProfileRdo.Fields.Get(Constant.Guids.Fields.ExtractorProfile.ProfileName).ValueAsFixedLengthText; if (ProfileName == null) { throw new CustomExceptions.TextExtractorException(Constant.ErrorMessages.EXTRACTOR_PROFILE_PROFILE_NAME_IS_EMPTY); } ArtifactId = extractorProfileRdo.ArtifactID; if (ArtifactId < 1) { throw new CustomExceptions.TextExtractorException(Constant.ErrorMessages.EXTRACTOR_PROFILE_ARTIFACT_ID_CANNOT_BE_LESS_THAN_OR_EQUAL_TO_ZERO); } var targetTextArtifacts = extractorProfileRdo.Fields.Get(Constant.Guids.Fields.ExtractorProfile.TargetText).GetValueAsMultipleObject <Artifact>(); if (targetTextArtifacts == null || targetTextArtifacts.Count == 0) { throw new CustomExceptions.TextExtractorException(Constant.ErrorMessages.EXTRACTOR_PROFILE_TARGET_TEXT_IS_EMPTY); } foreach (var targetTextArtifact in targetTextArtifacts) { var targetText = ArtifactFactory.GetInstanceOfExtractorTargetText(ExecutionIdentity.CurrentUser, WorkspaceArtifactId, targetTextArtifact.ArtifactID); ExtractorTargetTexts.Add(targetText); } } catch (Exception ex) { throw new CustomExceptions.TextExtractorException(Constant.ErrorMessages.DEFAULT_CONVERT_TO_EXTRACTOR_PROFILE_ERROR_MESSAGE + ". " + ExceptionMessageFormatter.GetInnerMostExceptionMessage(ex), ex); } }
// Virtual for testing purpose public virtual Boolean Process(ExtractorSetDocument extractorSetDocument, ExtractorSet extractorSet) { if (extractorSetDocument == null) { throw new ArgumentNullException("extractorSetDocument"); } if (extractorSetDocument.TextSource == null) { throw new CustomExceptions.TextExtractorException(Constant.ErrorMessages.EXTRACTOR_SET_SOURCE_LONG_TEXT_FIELD_IS_EMPTY); } var documentUpdatedWithExtractedText = false; ExtractorSetHistory extractorSetHistory = null; var errorContext = String.Format("An error occured when extracting text for field. [WorkspaceArtifactId: {0}, DocumentArtifactId: {1}, TextExtractorFieldArtifactId: {2}]", WorkspaceArtifactId, extractorSetDocument.ArtifactId, ArtifactId); try { string historyStartMarkerName = null; string historyStopMarkerName = null; string historyMarkerType = null; switch (TargetRule.MarkerEnum) { case Constant.MarkerEnum.RegEx: historyStartMarkerName = RegExStartMarker.Name; historyStopMarkerName = (RegExStopMarker == null) ? null : RegExStopMarker.Name; historyMarkerType = "Regular Expression"; break; case Constant.MarkerEnum.PlainText: historyStartMarkerName = StartMarker; historyStopMarkerName = StopMarker; historyMarkerType = "Plain Text"; break; } extractorSetHistory = new ExtractorSetHistory(ServicesMgr, ExecutionIdentity, ArtifactQueries, extractorSet.ArtifactId, extractorSetDocument.ArtifactId, DestinationField.ArtifactID, WorkspaceArtifactId, TargetName, historyStartMarkerName, historyStopMarkerName, historyMarkerType); if (!String.IsNullOrEmpty(this.StopMarker)) { TextExtractionUtility.StopMarker = this.StopMarker; } var extractedText = TextExtractionUtility.ExtractText(extractorSetDocument.TextSource, StartMarker, StopMarker, TargetRule); if (TextExtractionUtility.IsMarkerFound == false) { //create extractor set history record extractorSetHistory.CreateRecord(Constant.ExtractionSetHistoryStatus.COMPLETE_MARKER_NOT_FOUND); } else { if (String.IsNullOrEmpty(extractedText)) { //create extractor set history record extractorSetHistory.CreateRecord(Constant.ExtractionSetHistoryStatus.COMPLETE_TEXT_NOT_FOUND); } else { //update Document field with extracted text ArtifactQueries.UpdateDocumentTextFieldValue(ServicesMgr, ExecutionIdentity.CurrentUser, WorkspaceArtifactId, extractorSetDocument.ArtifactId, DestinationField.ArtifactID, extractedText); //check if text is truncated if (TextExtractionUtility.IsTextTruncated) { //Update TextExtractorDetails field on the Document object if extracted text is truncated var fieldValue = String.Format(Constant.TextExtractorDetailsMessages.TRUNCATED, ArtifactQueries.GetFieldNameForArtifactId(ServicesMgr, ExecutionIdentity, WorkspaceArtifactId, DestinationField.ArtifactID)); ArtifactQueries.AppendToDocumentLongTextFieldValue(ServicesMgr, ExecutionIdentity, WorkspaceArtifactId, extractorSetDocument.ArtifactId, Constant.Guids.Fields.Document.TextExtractorDetails, fieldValue); } //create extractor set history record extractorSetHistory.CreateRecord(Constant.ExtractionSetHistoryStatus.COMPLETE_TEXT_EXTRACTED); documentUpdatedWithExtractedText = true; } } } catch (Exception ex) { //create extractor set history record if (extractorSetHistory != null) { extractorSetHistory.CreateRecord(Constant.ExtractionSetHistoryStatus.ERROR, ExceptionMessageFormatter.GetInnerMostExceptionMessage(ex)); } else { throw new Exception("An error occured when creating Extractor Set History record."); } //log error message to ErrorLog table ErrorLogModel.InsertRecord(errorContext, ex, ArtifactId, WorkspaceArtifactId); } return(documentUpdatedWithExtractedText); }
public void SetMyProperties(RDO extractorTargetTextRdo) { try { ArtifactId = extractorTargetTextRdo.ArtifactID; if (ArtifactId < 1) { throw new CustomExceptions.TextExtractorException(Constant.ErrorMessages.TARGET_TEXT_ARTIFACT_ID_CANNOT_BE_LESS_THAN_OR_EQUAL_TO_ZERO); } TargetName = extractorTargetTextRdo.Fields.Get(Constant.Guids.Fields.ExtractorTargetText.TargetName).ValueAsFixedLengthText; if (TargetName == null) { throw new CustomExceptions.TextExtractorException(Constant.ErrorMessages.TARGET_TEXT_TARGET_NAME_IS_EMPTY); } switch (TargetRule.MarkerEnum) { case Constant.MarkerEnum.RegEx: RegeExStartMarkerArtifact = extractorTargetTextRdo.Fields.Get(Constant.Guids.Fields.ExtractorTargetText.RegularExpressionStartMarker).ValueAsSingleObject; if (RegeExStartMarkerArtifact == null) { throw new CustomExceptions.TextExtractorException(Constant.ErrorMessages.TARGET_TEXT_REGULAR_EXPRESSION_IS_EMPTY); } ArtifactFactory artifactFactory = new ArtifactFactory(ArtifactQueries, ServicesMgr, ErrorLogModel); RegExStartMarker = artifactFactory.GetInstanceOfExtractorRegullarExpression(ExecutionIdentity.CurrentUser, WorkspaceArtifactId, RegeExStartMarkerArtifact.ArtifactID); StartMarker = RegExStartMarker.RegularExpression; RegeExStopMarkerArtifact = extractorTargetTextRdo.Fields.Get(Constant.Guids.Fields.ExtractorTargetText.RegularExpressionStopMarker).ValueAsSingleObject; if (RegeExStopMarkerArtifact == null || RegeExStopMarkerArtifact.ArtifactID < 1) { RegExStopMarker = null; } else { RegExStopMarker = artifactFactory.GetInstanceOfExtractorRegullarExpression(ExecutionIdentity.CurrentUser, WorkspaceArtifactId, RegeExStopMarkerArtifact.ArtifactID); StopMarker = RegExStopMarker.RegularExpression; } break; case Constant.MarkerEnum.PlainText: PlainTextStartMarker = extractorTargetTextRdo.Fields.Get(Constant.Guids.Fields.ExtractorTargetText.PlainTextStartMarker).ValueAsFixedLengthText; if (PlainTextStartMarker == null) { throw new CustomExceptions.TextExtractorException(Constant.ErrorMessages.TARGET_TEXT_PLAIN_TEXT_MARKER_IS_EMPTY); } StartMarker = PlainTextStartMarker; PlainTextStopMarker = extractorTargetTextRdo.Fields.Get(Constant.Guids.Fields.ExtractorTargetText.PlainTextStopMarker).ValueAsFixedLengthText; StopMarker = PlainTextStopMarker; break; } if (StartMarker == null) { throw new CustomExceptions.TextExtractorException(Constant.ErrorMessages.TARGET_TEXT_START_MARKER_IS_EMPTY); } if (StopMarker != null) { TargetRule.DirectionEnum = Constant.DirectionEnum.RightToStopMarker; } DestinationField = extractorTargetTextRdo.Fields.Get(Constant.Guids.Fields.ExtractorTargetText.DestinationField).ValueAsSingleObject; if (DestinationField == null) { throw new CustomExceptions.TextExtractorException(Constant.ErrorMessages.TARGET_TEXT_DESTINATION_FIELD_IS_EMPTY); } } catch (Exception ex) { throw new CustomExceptions.TextExtractorException(Constant.ErrorMessages.DEFAULT_CONVERT_TO_EXTRACTOR_TARGET_TEXT_ERROR_MESSAGE + ". " + ExceptionMessageFormatter.GetInnerMostExceptionMessage(ex), ex); } }
public void SetMyProperties(RDO extractorRegullarExpressionRdo) { try { ArtifactId = extractorRegullarExpressionRdo.ArtifactID; if (ArtifactId < 1) { throw new CustomExceptions.TextExtractorException(Constant.ErrorMessages.REGULAR_EXPRESSION_ARTIFACT_ID_CANNOT_BE_LESS_THAN_OR_EQUAL_TO_ZERO); } Name = extractorRegullarExpressionRdo.Fields.Get(Constant.Guids.Fields.ExtractorRegularExpression.Name).ValueAsFixedLengthText; if (Name == null) { throw new CustomExceptions.TextExtractorException(Constant.ErrorMessages.REGULAR_EXPRESSION_NAME_IS_EMPTY); } RegularExpression = extractorRegullarExpressionRdo.Fields.Get(Constant.Guids.Fields.ExtractorRegularExpression.RegularExpression).ValueAsFixedLengthText; if (RegularExpression == null) { throw new CustomExceptions.TextExtractorException(Constant.ErrorMessages.REGULAR_EXPRESSION_REGULAR_EXPRESSION_IS_EMPTY); } Description = extractorRegullarExpressionRdo.Fields.Get(Constant.Guids.Fields.ExtractorRegularExpression.Description).ValueAsFixedLengthText; if (Description == null) { throw new CustomExceptions.TextExtractorException(Constant.ErrorMessages.REGULAR_EXPRESSION_DESCRIPTION_IS_EMPTY); } } catch (Exception ex) { throw new CustomExceptions.TextExtractorException(Constant.ErrorMessages.DEFAULT_CONVERT_TO_EXTRACTOR_REGULAR_EXPRESSION_ERROR_MESSAGE + ". " + ExceptionMessageFormatter.GetInnerMostExceptionMessage(ex), ex); } }