protected override void ProcessMessage(PipeMessageEnvelope envelope) { DocumentCollection documentCollection = null; #region Extract document details from incoming message object. documentCollection = envelope.Body as DocumentCollection; #region Assertion documentCollection.ShouldNotBe(null); #endregion try { //get the documents yet to converted List <ConversionDocumentBEO> documentsInConversionQueue = _nearNativeConverter.GetDocumentsQueuedForConversion(WorkAssignment.JobId); if (documentsInConversionQueue != null && documentsInConversionQueue.Any()) { //this is to avoid tight loop which consumes resources, especially network bandwidth int waitTime = GetConversionValidationWaitTime(documentsInConversionQueue); Thread.Sleep(waitTime); InputDataPipe.Purge(); // We don't need more than one "try again later" message in that pipe InputDataPipe.Send(envelope); } //IncreaseProcessedDocumentsCount(0); } catch (Exception ex) { ReportToDirector(ex); ex.Trace().Swallow(); } }
/// <summary> /// Processes the documents. /// </summary> /// <param name="documentDetails">The document details.</param> /// <param name="documentCollection">The document collection.</param> /// <param name="isNativeDocuments">if set to <c>true</c> [is native documents].</param> private void ProcessDocuments(IEnumerable <DocumentDetail> documentDetails, DocumentCollection documentCollection, bool isNativeDocuments) { if (documentDetails == null || !documentDetails.Any()) { return; } // get list of documents from Document Details. var documents = documentDetails.Select(x => x.document); // Instantiation var documentCollectionRetryList = new DocumentCollection() { documents = new List <DocumentDetail>(), dataset = documentCollection.dataset, Originator = Guid.NewGuid().ToString() }; List <RVWDocumentBEO> queuedDocuments = new List <RVWDocumentBEO>(); var docToPush = new Dictionary <string, IEnumerable <string> >(); // loop through documents to push each document for conversion foreach (RVWDocumentBEO document in documents) { if (document.MatterId == 0 || String.IsNullOrEmpty(document.CollectionId)) { continue; } //Gets the List of documents for conversion GetDocumentForConversion(document, documentCollectionRetryList, m_Converter, queuedDocuments, docToPush, PipelineType.Moniker); } //Push documents for conversion PushDocumentsForConversion(queuedDocuments, docToPush, isNativeDocuments); IncreaseProcessedDocumentsCount(queuedDocuments.Count); #region If there are any documents in retry list - put back to Conversion worker. Those files are yet being created. if (documentCollectionRetryList.documents.Any()) { InputDataPipe.Send(new PipeMessageEnvelope() { Body = documentCollectionRetryList, IsPostback = true }); } #endregion If there are any documents in retry list - put back to Conversion worker. Those files are yet being created. }
protected override void ProcessMessage(PipeMessageEnvelope message) { message.ShouldNotBe(null); var bulkTagRecord = (BulkTagRecord)message.Body; try { bulkTagRecord.ShouldNotBe(null); bulkTagRecord.CreatedByUserGuid.ShouldNotBeEmpty(); MockSession(bulkTagRecord.CreatedByUserGuid); var failedDocs = new List <KeyValuePair <string, string> >(); if (MatterBO.IsCrawlComplete(bulkTagRecord.MatterId.ToString(CultureInfo.InvariantCulture), bulkTagRecord.Originator, failedDocs)) { try { SendLog(CreateLogs(bulkTagRecord.Documents, failedDocs, bulkTagRecord)); } catch (Exception excep) { var errMsg = string.Format("Unable to create / send logs to be sent to log pipe. Exception details: {0}", excep.ToUserString()); excep.Trace().Swallow(); } Send(bulkTagRecord); } else { InputDataPipe.Send(new PipeMessageEnvelope() { Body = bulkTagRecord, IsPostback = true }); } } catch (Exception ex) { if (bulkTagRecord != null) { ex.Data["Originator"] = bulkTagRecord.Originator; } ex.Trace().Swallow(); ReportToDirector(ex); } }
private void ProcessDocumentImages(LawSyncDocumentCollection lawDocumentsList) { var documentReady = new List <LawSyncDocumentDetail>(); var documentNotReady = new List <LawSyncDocumentDetail>(); foreach (var lawImagingDocument in lawDocumentsList.Documents) { CheckConversionStateAndSyncImages(lawImagingDocument, documentNotReady, documentReady); } if (documentNotReady.Any()) { lawDocumentsList.Documents.Clear(); lawDocumentsList.Documents = documentNotReady; InputDataPipe.Send(new PipeMessageEnvelope { Body = lawDocumentsList, IsPostback = true }); } IncreaseProcessedDocumentsCount(documentReady.Count); }
protected override void ProcessMessage(PipeMessageEnvelope envelope) { try { var documentIdentifierEntityBeos = new List <DocumentIdentifierEntityBEO>(); var productionDocuments = envelope.Body as List <ProductionDocumentDetail>; if (productionDocuments == null || productionDocuments.Count == 0) { return; } ProductionDocumentDetail productionDocument = productionDocuments.First(); if (productionDocument == null) { return; } productionDocument.MatterId.ShouldNotBeEmpty(); long.TryParse(productionDocument.MatterId, out _matterId); _matterId.ShouldBeGreaterThan(0); productionDocument.dataSetBeo.ShouldNotBe(null); productionDocument.dataSetBeo.CollectionId.ShouldNotBeEmpty(); datasetCollectionGuid = productionDocument.dataSetBeo.CollectionId; var ready = new List <ProductionDocumentDetail>(); var notReady = new List <ProductionDocumentDetail>(); var documentConversionLogBeos = new List <DocumentConversionLogBeo>(); foreach (var productionDocumentDetail in productionDocuments) { var documentStatus = RedactItHeartbeatWatcher.CheckDocumentState(productionDocumentDetail.HeartBeatFile, 2); switch (documentStatus.DocumentState) { case RedactItHeartbeatWatcher.DocumentStateEnum.NotFound: if (!productionDocumentDetail.ConversionEnqueueTime.HasValue) { productionDocumentDetail.ConversionEnqueueTime = DateTime.UtcNow; } //the document conversion global timeout is the maximum waiting time for a document to be converted //Below are the possible reasons for this kind of timeout //1)heart beat file is not generated for this documents //2)document is waiting in queue then the expected global or waiting timeout configured in the CCMS if (DateTime.UtcNow - productionDocumentDetail.ConversionEnqueueTime > documentConverisonGlobalTimeout) { documentStatus.DocumentState = RedactItHeartbeatWatcher.DocumentStateEnum.Failure; documentStatus.ErrorReason = "RedactIt heartbeat file creation timeout warning"; Tracer.Trace( "Global Conversion Timeout For Document with document id - {0} ,dcn {1},collection id - {2} , matter id - {3} , HeartBeatFile {4} and Timeout Value {5}", productionDocumentDetail.DocumentId, productionDocumentDetail.DCNNumber, productionDocumentDetail.OriginalCollectionId, _matterId, productionDocumentDetail.HeartBeatFile, documentConverisonGlobalTimeout); goto case RedactItHeartbeatWatcher.DocumentStateEnum.Failure; } notReady.Add(productionDocumentDetail); break; case RedactItHeartbeatWatcher.DocumentStateEnum.NotReady: if (!productionDocumentDetail.ConversionStartTime.HasValue) { productionDocumentDetail.ConversionStartTime = DateTime.UtcNow; } if (DateTime.UtcNow - productionDocumentDetail.ConversionStartTime > documentConversionTimeout) { documentStatus.DocumentState = RedactItHeartbeatWatcher.DocumentStateEnum.Failure; documentStatus.ErrorMessage = documentStatus.ErrorReason = "Document conversion validation timeout warning"; Tracer.Trace( " Conversion Timeout For Document with document id - {0} ,dcn {1},collection id - {2} , matter id - {3} , HeartBeatFile {4} and Timeout Value {5}", productionDocumentDetail.DocumentId, productionDocumentDetail.DCNNumber, productionDocumentDetail.OriginalCollectionId, _matterId, productionDocumentDetail.HeartBeatFile, documentConversionTimeout); goto case RedactItHeartbeatWatcher.DocumentStateEnum.Failure; } notReady.Add(productionDocumentDetail); break; case RedactItHeartbeatWatcher.DocumentStateEnum.Success: ready.Add(productionDocumentDetail); SafeDeleteFolder(productionDocumentDetail.SourceDestinationPath); SafeDeleteFile(productionDocumentDetail.HeartBeatFile); documentConversionLogBeos.Add(ConvertToDocumentConversionLogBeo(productionDocumentDetail, EVRedactItErrorCodes. Completed)); var documentIdentifierEntity = ConstructAuditLog(productionDocumentDetail); documentIdentifierEntityBeos.Add(documentIdentifierEntity); RenameImagesBasedOnBatesNumber(productionDocumentDetail); //Delete all the source file for the document // Heart beat file and source file are deleted by callback in Production Notify service break; case RedactItHeartbeatWatcher.DocumentStateEnum.Failure: SafeDeleteFolder(productionDocumentDetail.SourceDestinationPath); //Delete all the source file for the document string message = "Redact-It reported error converting document DCN: " + productionDocumentDetail.DCNNumber + ". Heartbeat error: " + documentStatus.ErrorMessage + ". Refer heartbeat file " + productionDocumentDetail.HeartBeatFile + " for more info."; if (string.IsNullOrEmpty(documentStatus.ErrorReason)) { documentStatus.ErrorReason = EVRedactItErrorCodes.UnKnownConversionFailure; } documentConversionLogBeos.Add(ConvertToDocumentConversionLogBeo(productionDocumentDetail, EVRedactItErrorCodes.Failed, documentStatus.ErrorReason, documentStatus.ErrorMessage)); if (message.ToLower().Contains(TimeoutError)) { LogMessage(productionDocumentDetail, true, message, true); } else { LogMessage(productionDocumentDetail, false, message, false); } RenameImagesBasedOnBatesNumber(productionDocumentDetail); //Handle partially converted documents IncreaseProcessedDocumentsCount(1); // Failed document counted as processed break; } } if (notReady.Any()) { InputDataPipe.Send(new PipeMessageEnvelope() { Body = notReady, IsPostback = true }); } BulkUpdateProcessSetStatus(documentConversionLogBeos); if (m_BootParameters != null && m_BootParameters.Profile != null && !string.IsNullOrEmpty(m_BootParameters.Profile.ProductionSetName) && documentIdentifierEntityBeos.Count > 0) { DoAuditLog(documentIdentifierEntityBeos, m_BootParameters.Profile.ProductionSetName); } IncreaseProcessedDocumentsCount(ready.Count); } catch (Exception ex) { ex.Trace().Swallow(); ReportToDirector(ex); } }
/// <summary> /// Validate Import Conversion Reprocessing /// </summary> /// <param name="message">Input message</param> private void ValidateImportReprocessing(PipeMessageEnvelope message) { var docCollection = message.Body as ConversionDocCollection; if (docCollection == null || docCollection.Documents == null || !docCollection.Documents.Any()) //no document to validate { return; } var succeededDocs = new List <ReconversionDocumentBEO>(); var notReadyDocs = new List <ReconversionDocumentBEO>(); var failedDocs = new List <ReconversionDocumentBEO>(); foreach (var document in docCollection.Documents) { try { //number of calls to redactIt with the same DCNNumber, which all share the same heartbeat file int callCount = docCollection.Documents.Count(x => x.DCNNumber == document.DCNNumber); string docHbFilename = docCollection.GetDefaultHeartbeatFileFullPath(document); RedactItHeartbeatWatcher.DocumentStatus documentStatus = RedactItHeartbeatWatcher.CheckDocumentState(docHbFilename, callCount); switch (documentStatus.DocumentState) { case RedactItHeartbeatWatcher.DocumentStateEnum.NotFound: if (!document.ConversionEnqueueTime.HasValue) { document.ConversionEnqueueTime = DateTime.UtcNow; } if (DateTime.UtcNow - document.ConversionEnqueueTime > documentGlobalConversionTimeout) { documentStatus.DocumentState = RedactItHeartbeatWatcher.DocumentStateEnum.Failure; documentStatus.ErrorMessage = "Global document conversion timeout."; Tracer.Trace( "Global Conversion Timeout For Document with document id - {0} ,dcn -{1},collection id - {2} , matter id - {3} and Timeout value {4} ", document.DocumentId, document.DCNNumber, document.CollectionId, matterId, documentGlobalConversionTimeout); goto case RedactItHeartbeatWatcher.DocumentStateEnum.Failure; } notReadyDocs.Add(document); break; case RedactItHeartbeatWatcher.DocumentStateEnum.NotReady: if (!document.ConversionStartTime.HasValue) { document.ConversionStartTime = DateTime.UtcNow; } if (DateTime.UtcNow - document.ConversionStartTime > documentConversionTimeout) { documentStatus.DocumentState = RedactItHeartbeatWatcher.DocumentStateEnum.Failure; documentStatus.ErrorMessage = "Document conversion timeout."; Tracer.Trace( "Conversion Timeout For Document with document id - {0} ,dcn -{1},collection id - {2} matter id - {3} and Timeout value {4} ", document.DocumentId, document.DCNNumber, document.CollectionId, matterId, documentConversionTimeout); goto case RedactItHeartbeatWatcher.DocumentStateEnum.Failure; } notReadyDocs.Add(document); break; case RedactItHeartbeatWatcher.DocumentStateEnum.Success: succeededDocs.Add(document); break; case RedactItHeartbeatWatcher.DocumentStateEnum.Failure: //ToDo:For now we don't have job log for conversion reprocess job string error = "Redact-It reported error converting document DCN: " + document.DCNNumber + ". Heartbeat error: " + documentStatus.ErrorMessage + ". Refer heartbeat file " + docHbFilename + " for more info."; document.ConversionError = error; failedDocs.Add(document); break; } } catch (Exception ex) { ex.AddResMsg( "Problem in validating the conversion for the document document id - {0} ,dcn -{1},collection id - {2} and matter id - {3}", document.DocumentId, document.DCNNumber, document.CollectionId, matterId); ReportToDirector(ex); ex.Trace().Swallow(); failedDocs.Add(document); } } if (notReadyDocs.Any()) { docCollection.Documents = notReadyDocs; InputDataPipe.Send(new PipeMessageEnvelope { Body = docCollection, IsPostback = true }); } if (failedDocs.Any()) { IncreaseProcessedDocumentsCount(failedDocs.Count); } //ProcessConversionResults(failedDocs, docCollection, false); if (succeededDocs.Any()) { IncreaseProcessedDocumentsCount(succeededDocs.Count); //ProcessConversionResults(succeededDocs, docCollection, true); } }
/// <summary> /// Validate production Conversion Reprocessing /// </summary> /// <param name="message">Input message</param> private void ValidateProdctionReprocessing(PipeMessageEnvelope message) { var docList = message.Body as List <ProductionDocumentDetail>; if (docList == null || docList.Count == 0) //no document to validate { return; } var succeededDocs = new List <ProductionDocumentDetail>(); var notReadyDocs = new List <ProductionDocumentDetail>(); var failedDocs = new List <ProductionDocumentDetail>(); matterId = Convert.ToInt64(docList[0].MatterId); matterId.ShouldBeGreaterThan(0); var documentConversionLogBeos = new List <DocumentConversionLogBeo>(); foreach (var document in docList) { try { document.ConversionCheckCounter++; //production heartbeat file expect 2 calls for each conversion RedactItHeartbeatWatcher.DocumentStatus documentStatus = RedactItHeartbeatWatcher.CheckDocumentState(document.HeartBeatFile, 2); switch (documentStatus.DocumentState) { case RedactItHeartbeatWatcher.DocumentStateEnum.NotFound: if (!document.ConversionEnqueueTime.HasValue) { document.ConversionEnqueueTime = DateTime.UtcNow; } if (DateTime.UtcNow - document.ConversionEnqueueTime > documentGlobalConversionTimeout) { documentStatus.DocumentState = RedactItHeartbeatWatcher.DocumentStateEnum.Failure; documentStatus.ErrorMessage = "Document conversion timeout."; Tracer.Trace( "Global Conversion Timeout For Document with document id - {0} ,dcn -{1},collection id - {2} and matter id - {3} ", document.DocumentId, document.DCNNumber, document.OriginalCollectionId, matterId); goto case RedactItHeartbeatWatcher.DocumentStateEnum.Failure; } notReadyDocs.Add(document); break; case RedactItHeartbeatWatcher.DocumentStateEnum.NotReady: if (!document.ConversionStartTime.HasValue) { document.ConversionStartTime = DateTime.UtcNow; } if (DateTime.UtcNow - document.ConversionStartTime > documentConversionTimeout) { documentStatus.DocumentState = RedactItHeartbeatWatcher.DocumentStateEnum.Failure; documentStatus.ErrorMessage = "Document conversion timeout. Heartbeat file path: " + document.HeartBeatFile; Tracer.Trace( "Conversion Timeout For Document with document id - {0} ,dcn-{1},collection id - {2} and matter id - {3} ", document.DocumentId, document.DCNNumber, document.OriginalCollectionId, matterId); goto case RedactItHeartbeatWatcher.DocumentStateEnum.Failure; } notReadyDocs.Add(document); break; case RedactItHeartbeatWatcher.DocumentStateEnum.Success: succeededDocs.Add(document); SafeDeleteFolder(document.SourceDestinationPath); //Delete all the source file for the document documentConversionLogBeos.Add(ConvertToDocumentConversionLogBeo(document, 2)); SafeDeleteFile(document.HeartBeatFile); RenameImagesBasedOnBatesNumber(document); break; case RedactItHeartbeatWatcher.DocumentStateEnum.Failure: failedDocs.Add(document); if (string.IsNullOrEmpty(documentStatus.ErrorReason)) { documentStatus.ErrorReason = EVRedactItErrorCodes.UnKnownConversionFailure; } documentConversionLogBeos.Add(ConvertToDocumentConversionLogBeo(document, 3, documentStatus.ErrorReason, documentStatus.ErrorMessage)); SafeDeleteFolder(document.SourceDestinationPath); //Delete all the source file for the document string error = "Redact-It reported error converting document DCN: " + document.DCNNumber + ". Heartbeat error: " + documentStatus.ErrorMessage + ". Refer heartbeat file " + document.HeartBeatFile + " for more info."; RenameImagesBasedOnBatesNumber(document); //Handle partially converted documents LogMessage(document, false, error); break; } } catch (Exception ex) { ReportToDirector(ex); ex.Trace().Swallow(); //LogMessage(document, false, ex.ToUserString()); } } if (notReadyDocs.Any()) { InputDataPipe.Send(new PipeMessageEnvelope { Body = notReadyDocs, IsPostback = true }); } if (failedDocs.Any()) { IncreaseProcessedDocumentsCount(failedDocs.Count); } if (succeededDocs.Any()) { IncreaseProcessedDocumentsCount(succeededDocs.Count); } BulkUpdateProcessSetStatus(documentConversionLogBeos); }
/// <summary> /// Processes the data. /// </summary> public void ValidateDocuments(List <DocumentResult> printDocuments) { if (bulkPrintServiceRequestBEO == null) { return; } var targetDirectoryPath = Path.Combine(sharedLocation, bulkPrintServiceRequestBEO.Name); var notReady = new List <DocumentResult>(); var sourcedir = new DirectoryInfo(targetDirectoryPath); var separatorFileList = sourcedir.GetFiles(); try { foreach (DocumentResult documentResult in printDocuments) { string fieldValue; fieldValue = documentResult.DocumentControlNumber; if (documentResult.Fields != null && documentResult.Fields.Any()) { foreach (var field in documentResult.Fields) { if (field == null) { continue; } if (String.IsNullOrEmpty(field.Name)) { continue; } if (field.Name.Equals(bulkPrintServiceRequestBEO.FieldName)) { fieldValue = !string.IsNullOrEmpty(field.Value) ? field.Value.Trim() : fieldValue; } } } var document = separatorFileList.Where(x => x.Name.Equals(string.Format("{0}.pdf", fieldValue))); if (!document.Any()) { if (documentResult.CreatedDate > DateTime.Now.AddMinutes(-5)) { notReady.Add(documentResult); } else { Tracer.Info("Print Validation Worker - Failed Document: {0}", documentResult.DocumentControlNumber); LogMessage(documentResult, false, DocumentFailedDuetoTimeOutError); } } else { LogMessage(documentResult, true, "Document queued to printer"); } } if (notReady.Any()) { var documentCollection = new PrintDocumentCollection { Documents = notReady, TotalDocumentCount = totalDocumentCount }; var message = new PipeMessageEnvelope { Body = documentCollection, IsPostback = true }; InputDataPipe.Send(message); } else { UpdateAuditLog(); } } catch (IOException ex) { ex.AddDbgMsg("Directory = {0}", sourceLocation); ReportToDirector(ex); ex.Trace().Swallow(); } catch (Exception ex) { ReportToDirector(ex); ex.Trace().Swallow(); UpdateAuditLog(); } }