protected override void ProcessMessage(PipeMessageEnvelope message) { try { _exportDocumentCollection = (ExportDocumentCollection)message.Body; #region Assertion //Pre Condition PipelineId.ShouldNotBeEmpty(); BootParameters.ShouldNotBe(null); BootParameters.ShouldBeTypeOf <string>(); _exportDocumentCollection.ShouldNotBe(null); _exportDocumentCollection.Documents.ShouldNotBe(null); _exportDocumentCollection.Documents.LongCount().ShouldBeGreaterThan(0); #endregion if (_exportDocumentCollection == null || _exportDocumentCollection.Documents == null) { Tracer.Error("ExportOption Volume Worker: Document detail is not set in pipe message for job run id: {0}", PipelineId); return; } if (_volume == null) { InitializeForProcessing(BootParameters); } CalculateVolume(_exportDocumentCollection.Documents); } catch (Exception ex) { ex.Trace().Swallow(); ReportToDirector(ex); } }
private void Send(ExportDocumentCollection exportDocumentCollection) { if (OutputDataPipe == null) { Tracer.Error("OutputDataPipe == null"); return; } exportDocumentCollection.Dataset = null; var message = new PipeMessageEnvelope() { Body = exportDocumentCollection }; OutputDataPipe.Send(message); IncreaseProcessedDocumentsCount(exportDocumentCollection.Documents.Count()); }
private void SendData(List <ExportDocumentDetail> documentList) { var documentCollection = new ExportDocumentCollection { Documents = documentList, ExportOption = _exportOption }; var message = new PipeMessageEnvelope { Body = documentCollection }; if (OutputDataPipe != null) { OutputDataPipe.Send(message); } IncreaseProcessedDocumentsCount(documentList.Count); }
protected override void ProcessMessage(PipeMessageEnvelope message) { try { _exportDocumentCollection = (ExportDocumentCollection)message.Body; #region Assertion //Pre Condition PipelineId.ShouldNotBeEmpty(); BootParameters.ShouldNotBe(null); BootParameters.ShouldBeTypeOf <string>(); _exportDocumentCollection.ShouldNotBe(null); _exportDocumentCollection.Documents.ShouldNotBe(null); _exportDocumentCollection.Documents.LongCount().ShouldBeGreaterThan(0); #endregion if (_exportDocumentCollection == null) { Tracer.Error("ExportOption File Copy Worker: Pipe message body contains empty data for job run id:{0}", PipelineId); return; } _exportDocumentCollection.Dataset = _dataset; InitializeForProcessing(BootParameters); List <JobWorkerLog <ExportFileCopyLogInfo> > fileCopyLogList; PerformCopy(out fileCopyLogList); //Audit Log InsertAuditLog(_exportDocumentCollection, _exportLoadJobDetailBeo.JobName); #region Send Message Send(); if (fileCopyLogList != null && fileCopyLogList.Count > 0) { SendLog(fileCopyLogList); } #endregion } catch (Exception ex) { ex.Trace().Swallow(); ReportToDirector(ex); } }
/// <summary> /// Inserts the audit log. /// </summary> /// <param name="exportDocumentCollection">The export document collection.</param> /// <param name="sJobName">Name of the s job.</param> private void InsertAuditLog(ExportDocumentCollection exportDocumentCollection, string sJobName) { try { Utility.SetUserSession(_exportLoadJobDetailBeo.CreatedBy); var documentIdentifierEntities = exportDocumentCollection.Documents.Select(exportDocumentDetail => new DocumentIdentifierEntityBEO { CollectionId = exportDocumentCollection.Dataset.CollectionId, CollectionName = exportDocumentCollection.Dataset.FolderName, Dcn = exportDocumentDetail.DCN, DocumentReferenceId = exportDocumentDetail.DocumentId }).ToList(); AuditBO.LogDocumentsExported(exportDocumentCollection.Dataset.Matter.FolderID, documentIdentifierEntities, sJobName); } catch (Exception ex) { Tracer.Info("Failure on insert audit log for export {0}", ex.Message); } }
private void GetAllData(ExportDocumentCollection exportDocumentCollection) { #region Get Meta Data var documentDetails = new List <ExportDocumentDetail>(); var metaDataLogList = new List <JobWorkerLog <ExportMetadataLogInfo> >(); foreach (var doc in exportDocumentCollection.Documents) { JobWorkerLog <ExportMetadataLogInfo> metaDataLog; var exportDocument = ConstructDocumentData(exportDocumentCollection, doc.DocumentId, doc.CorrelationId, out metaDataLog); lock (documentDetails) { if (exportDocument != null) { documentDetails.Add(exportDocument); } if (metaDataLog != null) { metaDataLogList.Add(metaDataLog); } } } exportDocumentCollection.Documents.Clear(); exportDocumentCollection.Documents.AddRange(documentDetails); #endregion #region Send Message //Send Message Send(exportDocumentCollection); //Send Log if (metaDataLogList.Count > 0) { SendLog(metaDataLogList); metaDataLogList.Clear(); } #endregion }
private ExportDocumentDetail ConstructDocumentData(ExportDocumentCollection exportDocumentCollection, string documentId, string correlationId, out JobWorkerLog<ExportMetadataLogInfo> metadataLog) { var exportDocument = new ExportDocumentDetail(); try { exportDocument.DocumentId = documentId; exportDocument.CorrelationId = correlationId; bool isError; bool isErrorInTag = false; var document = DocumentBO.GetDocumentDataForExportJob(matterId, _dataset.CollectionId, documentId, out isError); if (document != null) { exportDocument.DCN = document.DocumentControlNumber; if (!String.IsNullOrEmpty(exportDocumentCollection.ExportOption.FieldForTextFileName)) { var textFieldId = DocumentBO.GetFieldIdByNameForExportJob(_dataset, exportDocumentCollection.ExportOption.FieldForTextFileName, false); exportDocument.TextFileName = DocumentBO.GetFieldValueForExportJob(matterId, _dataset.CollectionId, documentId, textFieldId); } if (!String.IsNullOrEmpty(exportDocumentCollection.ExportOption.FieldForNativeFileName)) { var nativeFieldId = DocumentBO.GetFieldIdByNameForExportJob(_dataset, exportDocumentCollection.ExportOption.FieldForNativeFileName, false); exportDocument.NativeFileName = DocumentBO.GetFieldValueForExportJob(matterId, _dataset.CollectionId, documentId, nativeFieldId); } #region "Recreate family group" //2)Attachment Range if (_beginsEnds != null) { Tuple<string, string> beginEnd; _beginsEnds.TryGetValue(document.Id.ToString(), out beginEnd); if (beginEnd != null) { exportDocument.BeginDoc = beginEnd.Item1; exportDocument.EndDoc = beginEnd.Item2; } } #endregion //3) File Path (Native File & Text) if (document.DocumentBinary.FileList != null) { if (exportDocumentCollection.ExportOption.IsNative) { //Get the tag name to check the tag exists for the document. If exists then export the native files. string tagNameToIncludeNative = exportDocumentCollection.ExportOption.IncludeNativeTagName; if (!string.IsNullOrEmpty(tagNameToIncludeNative)) //If "Include native for tagged document only" option is selected { //Check whether the document tagged with the selected tag. If yes, set the IsNativeTagExists property as true and get all native files to export out. exportDocument.IsNativeTagExists = CheckTagExistsToIncludeNative(document.CollectionId, document.DocumentId, tagNameToIncludeNative); if (exportDocument.IsNativeTagExists) //If document tagged then get the native files list to export. exportDocument.NativeFiles = GetFileList(document.DocumentBinary.FileList.Where(f => f.Type == Constants.NativeFileTypeId).ToList()); } else //If "Include native for tagged document only" option not selected, get native files to export out. { exportDocument.NativeFiles = GetFileList(document.DocumentBinary.FileList.Where(f => f.Type == Constants.NativeFileTypeId).ToList()); } } if (exportDocumentCollection.ExportOption.IsText) { if (exportDocumentCollection.ExportOption.IsTextFieldToExportSelected && !string.IsNullOrEmpty(exportDocumentCollection.ExportOption.TextFieldToExport)) { var fileList = new List<ExportFileInformation> { new ExportFileInformation() { IsTextFieldExportEnabled = true } }; exportDocument.TextFiles = fileList; } else { if (exportDocumentCollection.ExportOption.IsProduction && !string.IsNullOrEmpty(exportDocumentCollection.ExportOption.ProductionSetCollectionId)) { string[] lstOfProductionSets = exportDocumentCollection.ExportOption.ProductionSetCollectionId.Split(','); foreach (string productionSetId in lstOfProductionSets) { var productionDocumentData = GetDocumentData(documentId, productionSetId, out isError); if (exportDocumentCollection.ExportOption.TextOption1 == Constants.ScrubbedText) //If Text Prority 1 is Printed/Scrubbed text { //Get the scrubbed text files for one or more productions... if (exportDocument.TextFiles == null) { //Get the scrubbed text files for the first time.... exportDocument.TextFiles = GetFileList(productionDocumentData.DocumentBinary.FileList.Where(f => f.Type == Constants.ScrubbedFileTypeId).ToList()); } else { //Appnd to existing text files in case user chooses more than one production exportDocument.TextFiles.AddRange(GetFileList(productionDocumentData.DocumentBinary.FileList.Where(f => f.Type == Constants.ScrubbedFileTypeId).ToList())); } if ((exportDocument.TextFiles == null || exportDocument.TextFiles.Count == 0 || !File.Exists(exportDocument.TextFiles.FirstOrDefault().SourceFilePath)) && exportDocumentCollection.ExportOption.TextOption2 == Constants.ExtractedText) //If scrubbed text not exists for this document then get the Extracted text { exportDocument.TextFiles = GetFileList(document.DocumentBinary.FileList.Where(f => f.Type == Constants.TextFileTypeId).ToList()); exportDocument.TextFiles.SafeForEach(x => x.SourceFilePath = x.SourceFilePath.Contains(Constants.QuestionMark) ? x.SourceFilePath.Substring(0, x.SourceFilePath.LastIndexOf(Constants.QuestionMark)) : x.SourceFilePath); } else exportDocument.TextFiles.ForEach(s => s.IsScrubbedText = true); } else //Else If Text Prority 1 is Extracted text { //Get the extracted text files exportDocument.TextFiles = GetFileList(document.DocumentBinary.FileList.Where(f => f.Type == Constants.TextFileTypeId).ToList()); exportDocument.TextFiles.SafeForEach(x => x.SourceFilePath = x.SourceFilePath.Contains(Constants.QuestionMark) ? x.SourceFilePath.Substring(0, x.SourceFilePath.LastIndexOf(Constants.QuestionMark)) : x.SourceFilePath); //Remove the querystring if ((exportDocument.TextFiles == null || exportDocument.TextFiles.Count == 0 || !File.Exists(exportDocument.TextFiles.FirstOrDefault().SourceFilePath)) && exportDocumentCollection.ExportOption.TextOption2 == Constants.ScrubbedText) //If extracted text not exists for this document then get the Scrubbed text if exists { exportDocument.TextFiles = GetFileList(productionDocumentData.DocumentBinary.FileList.Where(f => f.Type == Constants.ScrubbedFileTypeId).ToList()); exportDocument.TextFiles.ForEach(s => s.IsScrubbedText = true); } } } } else { exportDocument.TextFiles = GetFileList(document.DocumentBinary.FileList.Where(f => f.Type == Constants.TextFileTypeId).ToList()); exportDocument.TextFiles.SafeForEach(x => x.SourceFilePath = x.SourceFilePath.Contains(Constants.QuestionMark) ? x.SourceFilePath.Substring(0, x.SourceFilePath.LastIndexOf(Constants.QuestionMark)) : x.SourceFilePath); //Remove the querystring } } } } //2.1) File Path (Images for Imageset or Production Set) exportDocument.ImageFiles = new List<ExportFileInformation>(); } //3) Tag if (exportDocumentCollection.ExportOption.IsTag && exportDocumentCollection.ExportOption.TagList != null && exportDocumentCollection.ExportOption.TagList.Count > 0) { var tag = GetDocumentTags(documentId, out isErrorInTag); if (tag != null && tag.Count > 0) { exportDocument.Tags = tag.Where(t => exportDocumentCollection.ExportOption.TagList.Contains(t.TagId.ToString(CultureInfo.InvariantCulture)) && t.Status).ToList(); } } #region Log metadataLog = ConstructLog(correlationId, (!isError), (!string.IsNullOrEmpty(exportDocument.DCN) ? exportDocument.DCN : string.Empty), isError, isErrorInTag, string.Empty); #endregion } catch (Exception ex) { ex.Trace().Swallow(); metadataLog = ConstructLog(correlationId, false, string.Empty, true, true, "Error in get Metadata."); } return exportDocument; }
private void GetAllData(ExportDocumentCollection exportDocumentCollection) { #region Get Meta Data var documentDetails = new List<ExportDocumentDetail>(); var metaDataLogList = new List<JobWorkerLog<ExportMetadataLogInfo>>(); foreach (var doc in exportDocumentCollection.Documents) { JobWorkerLog<ExportMetadataLogInfo> metaDataLog; var exportDocument = ConstructDocumentData(exportDocumentCollection, doc.DocumentId, doc.CorrelationId, out metaDataLog); lock (documentDetails) { if (exportDocument != null) { documentDetails.Add(exportDocument); } if (metaDataLog != null) { metaDataLogList.Add(metaDataLog); } } } exportDocumentCollection.Documents.Clear(); exportDocumentCollection.Documents.AddRange(documentDetails); #endregion #region Send Message //Send Message Send(exportDocumentCollection); //Send Log if (metaDataLogList.Count > 0) { SendLog(metaDataLogList); metaDataLogList.Clear(); } #endregion }
protected override void ProcessMessage(PipeMessageEnvelope envelope) { if (envelope.Label == "PleaseFinalize") { FinalizeFiles(); return; } exportDocumentCollection = (ExportDocumentCollection)envelope.Body; #region Assertion //Pre Condition PipelineId.ShouldNotBeEmpty(); BootParameters.ShouldNotBe(null); BootParameters.ShouldBeTypeOf <string>(); exportDocumentCollection.ShouldNotBe(null); exportDocumentCollection.Documents.ShouldNotBe(null); exportDocumentCollection.Documents.LongCount().ShouldBeGreaterThan(0); #endregion exportDocumentCollection.Dataset = _dataset; try { if (parametersExportLoadFile == null) { InitializeForProcessing(BootParameters); } GetDocumentFields(exportDocumentCollection.Documents); #region Get Content-Field value from Text file GetDocumentsContentField(); #endregion if (exportDocumentCollection.ExportOption.IsImage || exportDocumentCollection.ExportOption.IsProduction) { //Set Images File Path.. var loadFileHelper = new ExportLoadFileHelper(BootParameters); Parallel.ForEach(exportDocumentCollection.Documents, new ParallelOptions { MaxDegreeOfParallelism = _maxParallelThread }, (docDetail) => loadFileHelper.SetImageSourceFiles(docDetail, exportDocumentCollection.ExportOption)); } var fileWriterLogList = WriteLoadFiles(); #region Send Log if (fileWriterLogList != null && fileWriterLogList.Any()) { //Send to Log pipe SendLog(fileWriterLogList); } #endregion documentContentFieldsValueCollection.Clear(); } catch (Exception ex) { ex.Trace().Swallow(); ReportToDirector(ex); } }
private ExportDocumentDetail ConstructDocumentData(ExportDocumentCollection exportDocumentCollection, string documentId, string correlationId, out JobWorkerLog <ExportMetadataLogInfo> metadataLog) { var exportDocument = new ExportDocumentDetail(); try { exportDocument.DocumentId = documentId; exportDocument.CorrelationId = correlationId; bool isError; bool isErrorInTag = false; var document = DocumentBO.GetDocumentDataForExportJob(matterId, _dataset.CollectionId, documentId, out isError); if (document != null) { exportDocument.DCN = document.DocumentControlNumber; if (!String.IsNullOrEmpty(exportDocumentCollection.ExportOption.FieldForTextFileName)) { var textFieldId = DocumentBO.GetFieldIdByNameForExportJob(_dataset, exportDocumentCollection.ExportOption.FieldForTextFileName, false); exportDocument.TextFileName = DocumentBO.GetFieldValueForExportJob(matterId, _dataset.CollectionId, documentId, textFieldId); } if (!String.IsNullOrEmpty(exportDocumentCollection.ExportOption.FieldForNativeFileName)) { var nativeFieldId = DocumentBO.GetFieldIdByNameForExportJob(_dataset, exportDocumentCollection.ExportOption.FieldForNativeFileName, false); exportDocument.NativeFileName = DocumentBO.GetFieldValueForExportJob(matterId, _dataset.CollectionId, documentId, nativeFieldId); } #region "Recreate family group" //2)Attachment Range if (_beginsEnds != null) { Tuple <string, string> beginEnd; _beginsEnds.TryGetValue(document.Id.ToString(), out beginEnd); if (beginEnd != null) { exportDocument.BeginDoc = beginEnd.Item1; exportDocument.EndDoc = beginEnd.Item2; } } #endregion //3) File Path (Native File & Text) if (document.DocumentBinary.FileList != null) { if (exportDocumentCollection.ExportOption.IsNative) { //Get the tag name to check the tag exists for the document. If exists then export the native files. string tagNameToIncludeNative = exportDocumentCollection.ExportOption.IncludeNativeTagName; if (!string.IsNullOrEmpty(tagNameToIncludeNative)) //If "Include native for tagged document only" option is selected { //Check whether the document tagged with the selected tag. If yes, set the IsNativeTagExists property as true and get all native files to export out. exportDocument.IsNativeTagExists = CheckTagExistsToIncludeNative(document.CollectionId, document.DocumentId, tagNameToIncludeNative); if (exportDocument.IsNativeTagExists) //If document tagged then get the native files list to export. { exportDocument.NativeFiles = GetFileList(document.DocumentBinary.FileList.Where(f => f.Type == Constants.NativeFileTypeId).ToList()); } } else //If "Include native for tagged document only" option not selected, get native files to export out. { exportDocument.NativeFiles = GetFileList(document.DocumentBinary.FileList.Where(f => f.Type == Constants.NativeFileTypeId).ToList()); } } if (exportDocumentCollection.ExportOption.IsText) { if (exportDocumentCollection.ExportOption.IsTextFieldToExportSelected && !string.IsNullOrEmpty(exportDocumentCollection.ExportOption.TextFieldToExport)) { var fileList = new List <ExportFileInformation> { new ExportFileInformation() { IsTextFieldExportEnabled = true } }; exportDocument.TextFiles = fileList; } else { if (exportDocumentCollection.ExportOption.IsProduction && !string.IsNullOrEmpty(exportDocumentCollection.ExportOption.ProductionSetCollectionId)) { string[] lstOfProductionSets = exportDocumentCollection.ExportOption.ProductionSetCollectionId.Split(','); foreach (string productionSetId in lstOfProductionSets) { var productionDocumentData = GetDocumentData(documentId, productionSetId, out isError); if (exportDocumentCollection.ExportOption.TextOption1 == Constants.ScrubbedText) //If Text Prority 1 is Printed/Scrubbed text { //Get the scrubbed text files for one or more productions... if (exportDocument.TextFiles == null) { //Get the scrubbed text files for the first time.... exportDocument.TextFiles = GetFileList(productionDocumentData.DocumentBinary.FileList.Where(f => f.Type == Constants.ScrubbedFileTypeId).ToList()); } else { //Appnd to existing text files in case user chooses more than one production exportDocument.TextFiles.AddRange(GetFileList(productionDocumentData.DocumentBinary.FileList.Where(f => f.Type == Constants.ScrubbedFileTypeId).ToList())); } if ((exportDocument.TextFiles == null || exportDocument.TextFiles.Count == 0 || !File.Exists(exportDocument.TextFiles.FirstOrDefault().SourceFilePath)) && exportDocumentCollection.ExportOption.TextOption2 == Constants.ExtractedText) //If scrubbed text not exists for this document then get the Extracted text { exportDocument.TextFiles = GetFileList(document.DocumentBinary.FileList.Where(f => f.Type == Constants.TextFileTypeId).ToList()); exportDocument.TextFiles.SafeForEach(x => x.SourceFilePath = x.SourceFilePath.Contains(Constants.QuestionMark) ? x.SourceFilePath.Substring(0, x.SourceFilePath.LastIndexOf(Constants.QuestionMark)) : x.SourceFilePath); } else { exportDocument.TextFiles.ForEach(s => s.IsScrubbedText = true); } } else //Else If Text Prority 1 is Extracted text { //Get the extracted text files exportDocument.TextFiles = GetFileList(document.DocumentBinary.FileList.Where(f => f.Type == Constants.TextFileTypeId).ToList()); exportDocument.TextFiles.SafeForEach(x => x.SourceFilePath = x.SourceFilePath.Contains(Constants.QuestionMark) ? x.SourceFilePath.Substring(0, x.SourceFilePath.LastIndexOf(Constants.QuestionMark)) : x.SourceFilePath); //Remove the querystring if ((exportDocument.TextFiles == null || exportDocument.TextFiles.Count == 0 || !File.Exists(exportDocument.TextFiles.FirstOrDefault().SourceFilePath)) && exportDocumentCollection.ExportOption.TextOption2 == Constants.ScrubbedText) //If extracted text not exists for this document then get the Scrubbed text if exists { exportDocument.TextFiles = GetFileList(productionDocumentData.DocumentBinary.FileList.Where(f => f.Type == Constants.ScrubbedFileTypeId).ToList()); exportDocument.TextFiles.ForEach(s => s.IsScrubbedText = true); } } } } else { exportDocument.TextFiles = GetFileList(document.DocumentBinary.FileList.Where(f => f.Type == Constants.TextFileTypeId).ToList()); exportDocument.TextFiles.SafeForEach(x => x.SourceFilePath = x.SourceFilePath.Contains(Constants.QuestionMark) ? x.SourceFilePath.Substring(0, x.SourceFilePath.LastIndexOf(Constants.QuestionMark)) : x.SourceFilePath); //Remove the querystring } } } } //2.1) File Path (Images for Imageset or Production Set) exportDocument.ImageFiles = new List <ExportFileInformation>(); } //3) Tag if (exportDocumentCollection.ExportOption.IsTag && exportDocumentCollection.ExportOption.TagList != null && exportDocumentCollection.ExportOption.TagList.Count > 0) { var tag = GetDocumentTags(documentId, out isErrorInTag); if (tag != null && tag.Count > 0) { exportDocument.Tags = tag.Where(t => exportDocumentCollection.ExportOption.TagList.Contains(t.TagId.ToString(CultureInfo.InvariantCulture)) && t.Status).ToList(); } } #region Log metadataLog = ConstructLog(correlationId, (!isError), (!string.IsNullOrEmpty(exportDocument.DCN) ? exportDocument.DCN : string.Empty), isError, isErrorInTag, string.Empty); #endregion } catch (Exception ex) { ex.Trace().Swallow(); metadataLog = ConstructLog(correlationId, false, string.Empty, true, true, "Error in get Metadata."); } return(exportDocument); }