private void CheckConversionStateAndSyncImages(LawSyncDocumentDetail lawImagingDocument, List <LawSyncDocumentDetail> documentNotReady, List <LawSyncDocumentDetail> documentReady) { try { if (!lawImagingDocument.IsImagesXdlAvailable) { return; } var documentStatus = RedactItHeartbeatWatcher.CheckDocumentState(lawImagingDocument.RedactItHeartBeatFilePath, 1); switch (documentStatus.DocumentState) { case RedactItHeartbeatWatcher.DocumentStateEnum.NotFound: if (!HandleConversionImageNotFoundState(lawImagingDocument, documentStatus, documentNotReady)) { // timeout // Bug 169138: When conversion time out, Job log table display the error message "Failure in image conversion". It should display "Conversion timeout". HandleConversionImageFailureState(lawImagingDocument, documentStatus, true); // timeout } break; case RedactItHeartbeatWatcher.DocumentStateEnum.NotReady: if (!HandleConversionImageNotReadyState(lawImagingDocument, documentStatus, documentNotReady)) { // timeout // Bug 169138: When conversion time out, Job log table display the error message "Failure in image conversion". It should display "Conversion timeout". HandleConversionImageFailureState(lawImagingDocument, documentStatus, true); // timeout } break; case RedactItHeartbeatWatcher.DocumentStateEnum.Success: HandleConversionImageSucessState(documentReady, lawImagingDocument); break; case RedactItHeartbeatWatcher.DocumentStateEnum.Failure: HandleConversionImageFailureState(lawImagingDocument, documentStatus, false); // not timeout break; } } catch (Exception ex) { ex.AddDbgMsg("Law Document Id:{0}", lawImagingDocument.LawDocumentId); ex.Trace().Swallow(); ReportToDirector(ex); } }
protected override void ProcessMessage(PipeMessageEnvelope envelope) { try { var documentIdentifierEntityBeos = new List <DocumentIdentifierEntityBEO>(); var productionDocuments = envelope.Body as List <ProductionDocumentDetail>; if (productionDocuments == null || productionDocuments.Count == 0) { return; } ProductionDocumentDetail productionDocument = productionDocuments.First(); if (productionDocument == null) { return; } productionDocument.MatterId.ShouldNotBeEmpty(); long.TryParse(productionDocument.MatterId, out _matterId); _matterId.ShouldBeGreaterThan(0); productionDocument.dataSetBeo.ShouldNotBe(null); productionDocument.dataSetBeo.CollectionId.ShouldNotBeEmpty(); datasetCollectionGuid = productionDocument.dataSetBeo.CollectionId; var ready = new List <ProductionDocumentDetail>(); var notReady = new List <ProductionDocumentDetail>(); var documentConversionLogBeos = new List <DocumentConversionLogBeo>(); foreach (var productionDocumentDetail in productionDocuments) { var documentStatus = RedactItHeartbeatWatcher.CheckDocumentState(productionDocumentDetail.HeartBeatFile, 2); switch (documentStatus.DocumentState) { case RedactItHeartbeatWatcher.DocumentStateEnum.NotFound: if (!productionDocumentDetail.ConversionEnqueueTime.HasValue) { productionDocumentDetail.ConversionEnqueueTime = DateTime.UtcNow; } //the document conversion global timeout is the maximum waiting time for a document to be converted //Below are the possible reasons for this kind of timeout //1)heart beat file is not generated for this documents //2)document is waiting in queue then the expected global or waiting timeout configured in the CCMS if (DateTime.UtcNow - productionDocumentDetail.ConversionEnqueueTime > documentConverisonGlobalTimeout) { documentStatus.DocumentState = RedactItHeartbeatWatcher.DocumentStateEnum.Failure; documentStatus.ErrorReason = "RedactIt heartbeat file creation timeout warning"; Tracer.Trace( "Global Conversion Timeout For Document with document id - {0} ,dcn {1},collection id - {2} , matter id - {3} , HeartBeatFile {4} and Timeout Value {5}", productionDocumentDetail.DocumentId, productionDocumentDetail.DCNNumber, productionDocumentDetail.OriginalCollectionId, _matterId, productionDocumentDetail.HeartBeatFile, documentConverisonGlobalTimeout); goto case RedactItHeartbeatWatcher.DocumentStateEnum.Failure; } notReady.Add(productionDocumentDetail); break; case RedactItHeartbeatWatcher.DocumentStateEnum.NotReady: if (!productionDocumentDetail.ConversionStartTime.HasValue) { productionDocumentDetail.ConversionStartTime = DateTime.UtcNow; } if (DateTime.UtcNow - productionDocumentDetail.ConversionStartTime > documentConversionTimeout) { documentStatus.DocumentState = RedactItHeartbeatWatcher.DocumentStateEnum.Failure; documentStatus.ErrorMessage = documentStatus.ErrorReason = "Document conversion validation timeout warning"; Tracer.Trace( " Conversion Timeout For Document with document id - {0} ,dcn {1},collection id - {2} , matter id - {3} , HeartBeatFile {4} and Timeout Value {5}", productionDocumentDetail.DocumentId, productionDocumentDetail.DCNNumber, productionDocumentDetail.OriginalCollectionId, _matterId, productionDocumentDetail.HeartBeatFile, documentConversionTimeout); goto case RedactItHeartbeatWatcher.DocumentStateEnum.Failure; } notReady.Add(productionDocumentDetail); break; case RedactItHeartbeatWatcher.DocumentStateEnum.Success: ready.Add(productionDocumentDetail); SafeDeleteFolder(productionDocumentDetail.SourceDestinationPath); SafeDeleteFile(productionDocumentDetail.HeartBeatFile); documentConversionLogBeos.Add(ConvertToDocumentConversionLogBeo(productionDocumentDetail, EVRedactItErrorCodes. Completed)); var documentIdentifierEntity = ConstructAuditLog(productionDocumentDetail); documentIdentifierEntityBeos.Add(documentIdentifierEntity); RenameImagesBasedOnBatesNumber(productionDocumentDetail); //Delete all the source file for the document // Heart beat file and source file are deleted by callback in Production Notify service break; case RedactItHeartbeatWatcher.DocumentStateEnum.Failure: SafeDeleteFolder(productionDocumentDetail.SourceDestinationPath); //Delete all the source file for the document string message = "Redact-It reported error converting document DCN: " + productionDocumentDetail.DCNNumber + ". Heartbeat error: " + documentStatus.ErrorMessage + ". Refer heartbeat file " + productionDocumentDetail.HeartBeatFile + " for more info."; if (string.IsNullOrEmpty(documentStatus.ErrorReason)) { documentStatus.ErrorReason = EVRedactItErrorCodes.UnKnownConversionFailure; } documentConversionLogBeos.Add(ConvertToDocumentConversionLogBeo(productionDocumentDetail, EVRedactItErrorCodes.Failed, documentStatus.ErrorReason, documentStatus.ErrorMessage)); if (message.ToLower().Contains(TimeoutError)) { LogMessage(productionDocumentDetail, true, message, true); } else { LogMessage(productionDocumentDetail, false, message, false); } RenameImagesBasedOnBatesNumber(productionDocumentDetail); //Handle partially converted documents IncreaseProcessedDocumentsCount(1); // Failed document counted as processed break; } } if (notReady.Any()) { InputDataPipe.Send(new PipeMessageEnvelope() { Body = notReady, IsPostback = true }); } BulkUpdateProcessSetStatus(documentConversionLogBeos); if (m_BootParameters != null && m_BootParameters.Profile != null && !string.IsNullOrEmpty(m_BootParameters.Profile.ProductionSetName) && documentIdentifierEntityBeos.Count > 0) { DoAuditLog(documentIdentifierEntityBeos, m_BootParameters.Profile.ProductionSetName); } IncreaseProcessedDocumentsCount(ready.Count); } catch (Exception ex) { ex.Trace().Swallow(); ReportToDirector(ex); } }
/// <summary> /// Validate Import Conversion Reprocessing /// </summary> /// <param name="message">Input message</param> private void ValidateImportReprocessing(PipeMessageEnvelope message) { var docCollection = message.Body as ConversionDocCollection; if (docCollection == null || docCollection.Documents == null || !docCollection.Documents.Any()) //no document to validate { return; } var succeededDocs = new List <ReconversionDocumentBEO>(); var notReadyDocs = new List <ReconversionDocumentBEO>(); var failedDocs = new List <ReconversionDocumentBEO>(); foreach (var document in docCollection.Documents) { try { //number of calls to redactIt with the same DCNNumber, which all share the same heartbeat file int callCount = docCollection.Documents.Count(x => x.DCNNumber == document.DCNNumber); string docHbFilename = docCollection.GetDefaultHeartbeatFileFullPath(document); RedactItHeartbeatWatcher.DocumentStatus documentStatus = RedactItHeartbeatWatcher.CheckDocumentState(docHbFilename, callCount); switch (documentStatus.DocumentState) { case RedactItHeartbeatWatcher.DocumentStateEnum.NotFound: if (!document.ConversionEnqueueTime.HasValue) { document.ConversionEnqueueTime = DateTime.UtcNow; } if (DateTime.UtcNow - document.ConversionEnqueueTime > documentGlobalConversionTimeout) { documentStatus.DocumentState = RedactItHeartbeatWatcher.DocumentStateEnum.Failure; documentStatus.ErrorMessage = "Global document conversion timeout."; Tracer.Trace( "Global Conversion Timeout For Document with document id - {0} ,dcn -{1},collection id - {2} , matter id - {3} and Timeout value {4} ", document.DocumentId, document.DCNNumber, document.CollectionId, matterId, documentGlobalConversionTimeout); goto case RedactItHeartbeatWatcher.DocumentStateEnum.Failure; } notReadyDocs.Add(document); break; case RedactItHeartbeatWatcher.DocumentStateEnum.NotReady: if (!document.ConversionStartTime.HasValue) { document.ConversionStartTime = DateTime.UtcNow; } if (DateTime.UtcNow - document.ConversionStartTime > documentConversionTimeout) { documentStatus.DocumentState = RedactItHeartbeatWatcher.DocumentStateEnum.Failure; documentStatus.ErrorMessage = "Document conversion timeout."; Tracer.Trace( "Conversion Timeout For Document with document id - {0} ,dcn -{1},collection id - {2} matter id - {3} and Timeout value {4} ", document.DocumentId, document.DCNNumber, document.CollectionId, matterId, documentConversionTimeout); goto case RedactItHeartbeatWatcher.DocumentStateEnum.Failure; } notReadyDocs.Add(document); break; case RedactItHeartbeatWatcher.DocumentStateEnum.Success: succeededDocs.Add(document); break; case RedactItHeartbeatWatcher.DocumentStateEnum.Failure: //ToDo:For now we don't have job log for conversion reprocess job string error = "Redact-It reported error converting document DCN: " + document.DCNNumber + ". Heartbeat error: " + documentStatus.ErrorMessage + ". Refer heartbeat file " + docHbFilename + " for more info."; document.ConversionError = error; failedDocs.Add(document); break; } } catch (Exception ex) { ex.AddResMsg( "Problem in validating the conversion for the document document id - {0} ,dcn -{1},collection id - {2} and matter id - {3}", document.DocumentId, document.DCNNumber, document.CollectionId, matterId); ReportToDirector(ex); ex.Trace().Swallow(); failedDocs.Add(document); } } if (notReadyDocs.Any()) { docCollection.Documents = notReadyDocs; InputDataPipe.Send(new PipeMessageEnvelope { Body = docCollection, IsPostback = true }); } if (failedDocs.Any()) { IncreaseProcessedDocumentsCount(failedDocs.Count); } //ProcessConversionResults(failedDocs, docCollection, false); if (succeededDocs.Any()) { IncreaseProcessedDocumentsCount(succeededDocs.Count); //ProcessConversionResults(succeededDocs, docCollection, true); } }
/// <summary> /// Validate production Conversion Reprocessing /// </summary> /// <param name="message">Input message</param> private void ValidateProdctionReprocessing(PipeMessageEnvelope message) { var docList = message.Body as List <ProductionDocumentDetail>; if (docList == null || docList.Count == 0) //no document to validate { return; } var succeededDocs = new List <ProductionDocumentDetail>(); var notReadyDocs = new List <ProductionDocumentDetail>(); var failedDocs = new List <ProductionDocumentDetail>(); matterId = Convert.ToInt64(docList[0].MatterId); matterId.ShouldBeGreaterThan(0); var documentConversionLogBeos = new List <DocumentConversionLogBeo>(); foreach (var document in docList) { try { document.ConversionCheckCounter++; //production heartbeat file expect 2 calls for each conversion RedactItHeartbeatWatcher.DocumentStatus documentStatus = RedactItHeartbeatWatcher.CheckDocumentState(document.HeartBeatFile, 2); switch (documentStatus.DocumentState) { case RedactItHeartbeatWatcher.DocumentStateEnum.NotFound: if (!document.ConversionEnqueueTime.HasValue) { document.ConversionEnqueueTime = DateTime.UtcNow; } if (DateTime.UtcNow - document.ConversionEnqueueTime > documentGlobalConversionTimeout) { documentStatus.DocumentState = RedactItHeartbeatWatcher.DocumentStateEnum.Failure; documentStatus.ErrorMessage = "Document conversion timeout."; Tracer.Trace( "Global Conversion Timeout For Document with document id - {0} ,dcn -{1},collection id - {2} and matter id - {3} ", document.DocumentId, document.DCNNumber, document.OriginalCollectionId, matterId); goto case RedactItHeartbeatWatcher.DocumentStateEnum.Failure; } notReadyDocs.Add(document); break; case RedactItHeartbeatWatcher.DocumentStateEnum.NotReady: if (!document.ConversionStartTime.HasValue) { document.ConversionStartTime = DateTime.UtcNow; } if (DateTime.UtcNow - document.ConversionStartTime > documentConversionTimeout) { documentStatus.DocumentState = RedactItHeartbeatWatcher.DocumentStateEnum.Failure; documentStatus.ErrorMessage = "Document conversion timeout. Heartbeat file path: " + document.HeartBeatFile; Tracer.Trace( "Conversion Timeout For Document with document id - {0} ,dcn-{1},collection id - {2} and matter id - {3} ", document.DocumentId, document.DCNNumber, document.OriginalCollectionId, matterId); goto case RedactItHeartbeatWatcher.DocumentStateEnum.Failure; } notReadyDocs.Add(document); break; case RedactItHeartbeatWatcher.DocumentStateEnum.Success: succeededDocs.Add(document); SafeDeleteFolder(document.SourceDestinationPath); //Delete all the source file for the document documentConversionLogBeos.Add(ConvertToDocumentConversionLogBeo(document, 2)); SafeDeleteFile(document.HeartBeatFile); RenameImagesBasedOnBatesNumber(document); break; case RedactItHeartbeatWatcher.DocumentStateEnum.Failure: failedDocs.Add(document); if (string.IsNullOrEmpty(documentStatus.ErrorReason)) { documentStatus.ErrorReason = EVRedactItErrorCodes.UnKnownConversionFailure; } documentConversionLogBeos.Add(ConvertToDocumentConversionLogBeo(document, 3, documentStatus.ErrorReason, documentStatus.ErrorMessage)); SafeDeleteFolder(document.SourceDestinationPath); //Delete all the source file for the document string error = "Redact-It reported error converting document DCN: " + document.DCNNumber + ". Heartbeat error: " + documentStatus.ErrorMessage + ". Refer heartbeat file " + document.HeartBeatFile + " for more info."; RenameImagesBasedOnBatesNumber(document); //Handle partially converted documents LogMessage(document, false, error); break; } } catch (Exception ex) { ReportToDirector(ex); ex.Trace().Swallow(); //LogMessage(document, false, ex.ToUserString()); } } if (notReadyDocs.Any()) { InputDataPipe.Send(new PipeMessageEnvelope { Body = notReadyDocs, IsPostback = true }); } if (failedDocs.Any()) { IncreaseProcessedDocumentsCount(failedDocs.Count); } if (succeededDocs.Any()) { IncreaseProcessedDocumentsCount(succeededDocs.Count); } BulkUpdateProcessSetStatus(documentConversionLogBeos); }
/// <summary> /// Handle Image Not Found /// </summary> private bool HandleConversionImageNotFoundState(LawSyncDocumentDetail lawImagingDocument, RedactItHeartbeatWatcher.DocumentStatus documentStatus, List<LawSyncDocumentDetail> documentNotReady) { if (!lawImagingDocument.ConversionEnqueueTime.HasValue) { lawImagingDocument.ConversionEnqueueTime = DateTime.UtcNow; } if (DateTime.UtcNow - lawImagingDocument.ConversionEnqueueTime > _documentConverisonGlobalTimeout) { documentStatus.DocumentState = RedactItHeartbeatWatcher.DocumentStateEnum.Failure; documentStatus.ErrorMessage = Constants.GlobalDocumentconversionTimeoutMessage; Tracer.Trace("Global Conversion Timeout for Law document id - {0} ,dcn {1},collection id - {2} , HeartBeatFile {3} and Job Run Id {4}", lawImagingDocument.LawDocumentId, lawImagingDocument.DocumentControlNumber, lawImagingDocument.RedactItHeartBeatFilePath, PipelineId); return false; } documentNotReady.Add(lawImagingDocument); return true; }
/// <summary> /// Handle Image Conversion Not Ready /// </summary> private bool HandleConversionImageNotReadyState(LawSyncDocumentDetail lawImagingDocument, RedactItHeartbeatWatcher.DocumentStatus documentStatus, List<LawSyncDocumentDetail> documentNotReady) { if (!lawImagingDocument.ConversionStartTime.HasValue) { lawImagingDocument.ConversionStartTime = DateTime.UtcNow; } if (DateTime.UtcNow - lawImagingDocument.ConversionStartTime > _documentConversionTimeout) { documentStatus.DocumentState = RedactItHeartbeatWatcher.DocumentStateEnum.Failure; documentStatus.ErrorMessage = Constants.DocumentconversionTimeoutMessage; Tracer.Error("Conversion Timeout for Law document id - {0} ,dcn {1} , HeartBeatFile {2}, timeoutValue: {3}", lawImagingDocument.LawDocumentId, lawImagingDocument.DocumentControlNumber, lawImagingDocument.RedactItHeartBeatFilePath, _documentConversionTimeout.TotalSeconds); return false; } documentNotReady.Add(lawImagingDocument); return true; }
/// <summary> /// Handle Image Conversion Failure /// </summary> private void HandleConversionImageFailureState(LawSyncDocumentDetail lawImagingDocument, RedactItHeartbeatWatcher.DocumentStatus documentStatus, bool isTimeOut) { SafeDeleteFolder(lawImagingDocument.DocumentExtractionPath); // Bug 169138: When conversion time out, Job log table display the error message "Failure in image conversion". It should display "Conversion timeout". string errorMessage = Constants.LawSyncFailureinConversionMessage; if (isTimeOut) { errorMessage = Constants.LawSyncFailureinConversionTimeOutMessage; } Tracer.Error("Image conversion error: {0}", errorMessage); ConstructLog(lawImagingDocument.LawDocumentId, lawImagingDocument.CorrelationId, lawImagingDocument.DocumentControlNumber, errorMessage, lawImagingDocument.RedactItHeartBeatFilePath, lawImagingDocument.ImagesFolderPath); IncreaseProcessedDocumentsCount(1); // Failed document counted as processed if (string.IsNullOrEmpty(documentStatus.ErrorReason)) { documentStatus.ErrorReason = EVRedactItErrorCodes.UnKnownConversionFailure; } //Filling error details in ProcessSet table for future use var documentProcessState = GetDocumentProcessStateInformationForImageConversion(lawImagingDocument, documentStatus.ErrorReason); documentProcessState.ErrorDetails = documentStatus.ErrorMessage; _documentProcessStateList.Add(documentProcessState); //Handle partially converted documents RenameAndSetImagesInDocument(lawImagingDocument); }