/// <summary>
        /// Process each document 
        /// </summary>
        private void ProcessDocuments(DocumentCollection recordParserResponse)
        {
            var documentDetailList = new List<DocumentDetail>();
            var overlayLogList = new List<JobWorkerLog<OverlaySearchLogInfo>>();
            var docManager = new OverlayDocumentManager(_jobParameter, _dataset, PipelineId, WorkerId);

            //Bulk Search for entire batch
            documentDetailList = docManager.BulkSearch(recordParserResponse.documents, _userInfo, out overlayLogList);

            SendRelationshipsInfo(documentDetailList);

            #region Send Message
            //Send response   
            if (documentDetailList != null && documentDetailList.Any())
            {
                if (documentDetailList.Count > _batchSize)
                {
                    Send(documentDetailList.Take(_batchSize).ToList());
                    var remainDocumentList = documentDetailList.Skip(_batchSize).ToList();
                    Send(remainDocumentList); //send remaining list
                    documentDetailList.Clear();
                }
                else
                {
                    Send(documentDetailList);
                    documentDetailList.Clear();
                }
            }
            //Send Log   
            if (overlayLogList != null && overlayLogList.Any())
            {
                SendLog(overlayLogList);
                overlayLogList.Clear();
            }
            #endregion
        }
 /// <summary>
 /// Send data to Data pipe
 /// </summary>
 /// <param name="docDetails"></param>
 private void Send(List<DocumentDetail> docDetails)
 {
     bool overlayIsRemoveTag = _jobParameter != null && _jobParameter != null && !_jobParameter.IsOverlayRetainTags;
     var documentList = new DocumentCollection() { documents = docDetails, dataset = _dataset, IsDeleteTagsForOverlay = overlayIsRemoveTag, IsIncludeNativeFile = _isIncludeNativeFile };
     var message = new PipeMessageEnvelope()
     {
         Body = documentList
     };
     OutputDataPipe.Send(message);
     IncreaseProcessedDocumentsCount(docDetails.Count);
 }
        /// <summary>
        /// Sends the specified document batch to next worker in the pipeline.
        /// </summary>
        /// <param name="documentBatch">The document batch.</param>
        /// <param name="isIncludeOutlookMailStoreDataEntityIfAvailable">if set to <c>true</c> [includes outlook mail store data entity in the send list (when available)].</param>
        private void Send(IEnumerable<RVWDocumentBEO> documentBatch, bool isIncludeOutlookMailStoreDataEntityIfAvailable)
        {
            try
            {
                DocumentCollection documentCollection = null;
                List<DocumentDetail> documentDetailList = new List<DocumentDetail>();

                if (documentBatch != null)
                {
                    foreach (RVWDocumentBEO document in documentBatch)
                    {
                        m_CounterForCorrelationId += 1;
                        DocumentDetail documentDetail = new DocumentDetail
                        {
                            CorrelationId = m_CounterForCorrelationId.ToString(),
                            docType = DocumentsetType.NativeSet,
                            document = document,
                            IsNewDocument = true
                        };

                        documentDetailList.Add(documentDetail);
                    }

                    documentCollection = new DocumentCollection
                    {
                        dataset = m_Dataset,
                        documents = documentDetailList
                    };
                }
                if (documentCollection != null)
                {
                    Pipe vaultOutDataPipe = GetOutputDataPipe("Vault");
                    var message = new PipeMessageEnvelope()
                    {
                        Body = documentCollection
                    };
                    if (vaultOutDataPipe != null)
                    {
                        vaultOutDataPipe.Send(message);
                    }
                }
                // All available outlook mail stores are sent to the queue. they shouldn't be sent again. So clear existing list.
                if (isIncludeOutlookMailStoreDataEntityIfAvailable && m_OutlookMailStoreDataEntities != null
                    && m_OutlookMailStoreDataEntities.Count > 0)
                {
                    var message = new PipeMessageEnvelope()
                   {
                       Body = new EDocsDocumentCollection
                       {
                           OutlookMailStoreDataEntity = m_OutlookMailStoreDataEntities
                       }
                   };
                    Pipe eDocsOutlookEmailGeneratorOutputDataPipe = GetOutputDataPipe("EDocsOutlookEmailGenerator");
                    if (eDocsOutlookEmailGeneratorOutputDataPipe != null)
                    {
                        eDocsOutlookEmailGeneratorOutputDataPipe.Send(message);
                    }
                    m_OutlookMailStoreDataEntities.Clear();

                }
                IncreaseProcessedDocumentsCount(documentCollection.documents.Count);
            }
            catch (Exception ex)
            {
                ex.Trace().Swallow();
            }
        }
 /// <summary>
 /// Send data to Data pipe
 /// </summary>
 /// <param name="docDetails"></param>
 private void Send(List<DocumentDetail> docDetails)
 {
     OutputDataPipe.ShouldNotBe(null);
     var documentList = new DocumentCollection() { documents = docDetails, dataset = _dataset };
     var message = new PipeMessageEnvelope()
     {
         Body = documentList
     };
     OutputDataPipe.Send(message);
     IncreaseProcessedDocumentsCount(docDetails.Count);
 }
 /// <summary>
 /// Send data to Data pipe
 /// </summary>
 /// <param name="docDetails"></param>
 private void Send(List<DocumentDetail> docDetails)
 {
     var documentList = new DocumentCollection
         {
             documents = docDetails,
             dataset = _dataset,
             IsDeleteTagsForOverlay = false,
             IsIncludeNativeFile = _isIncludeNativeFile
         };
     var message = new PipeMessageEnvelope
         {
             Body = documentList
         };
     OutputDataPipe.Send(message);
     IncreaseProcessedDocumentsCount(docDetails.Count);
 }
 private void Send(List<DocumentDetail> docDetails)
 {
     var documentList = new DocumentCollection() { documents = docDetails, dataset = _dataset };
     var message = new PipeMessageEnvelope()
     {
         Body = documentList
     };
     OutputDataPipe.Send(message);
     //IncreaseProcessedDocumentsCount(docDetails.Count(DocumentDetail => DocumentDetail.docType == DocumentsetType.NativeSet));
     IncreaseProcessedDocumentsCount(docDetails.Count);
 }
 /// <summary>
 /// To send the collection to the pipeline director.
 /// </summary>
 /// <param name="collection"></param>
 private void Send(DocumentCollection collection)
 {
     var message = new PipeMessageEnvelope
     {
         Body = collection
     };
     OutputDataPipe.Send(message);
 }
        /// <summary>
        /// Sends the log.
        /// </summary>
        /// <param name="documentCollection">The document collection.</param>
        /// <param name="isSentForIndexing">if set to <c>true</c> [is sent for indexing].</param>
        /// <param name="documentErrorCollection"></param>
        private  void SendLog(DocumentCollection documentCollection, bool isSentForIndexing,DocumentErrorCollection documentErrorCollection=null)
        {
            if (documentCollection == null || documentCollection.documents == null) return;
            var message = isSentForIndexing ? "Sent for indexing." : "Failed to send for indexing.";
            var nativeDocumentList =
                documentCollection.documents.FindAll(
                    n => n.docType == DocumentsetType.NativeSet);
            if (!nativeDocumentList.Any()) return;

            var searchIndexLogInfos = new List<JobWorkerLog<SearchIndexLogInfo>>();
            try
            {
                foreach (var documentDetail in nativeDocumentList)
                {
                    if (documentDetail.document == null) continue;
                    var logInfo = new SearchIndexLogInfo
                    {
                        Information =
                            string.Format("DCN:{0}", documentDetail.document.DocumentControlNumber),
                        DocumentId = documentDetail.document.DocumentId,
                        DCNNumber = documentDetail.document.DocumentControlNumber,
                        CrossReferenceField = documentDetail.document.CrossReferenceFieldValue,
                        Message = message
                    };
                    SetDocumentError(documentErrorCollection, documentDetail, logInfo);
                    if (String.IsNullOrEmpty(documentDetail.CorrelationId))
                    {
                        documentDetail.CorrelationId = "0";
                    }
                    var searchIndexLogInfo = new JobWorkerLog<SearchIndexLogInfo>
                    {
                        JobRunId = Convert.ToInt32(PipelineId),
                        CorrelationId = long.Parse(documentDetail.CorrelationId),
                        WorkerInstanceId = WorkerId,
                        WorkerRoleType = "8A65E2DC-753C-E311-82FA-005056850057",
                        Success = isSentForIndexing,
                        LogInfo = logInfo
                    };


                    searchIndexLogInfos.Add(searchIndexLogInfo);
                }
                LogPipe.Open();
                var pipleMessageEnvelope = new PipeMessageEnvelope
                {
                    Body = searchIndexLogInfos
                };
                LogPipe.Send(pipleMessageEnvelope);
            }
            catch (Exception exception)
            {
                exception.AddDbgMsg("Failed to log document details");
                exception.Trace().Swallow();
                ReportToDirector(exception);
            }
        }
        /// <summary>
        /// Constructs the overlay fields
        /// </summary>
        /// <param name="documentCollection">DocumentCollection</param>
        /// <param name="nativeDocumentListUpdate">list of DocumentDetail</param>
        /// <param name="isDeleteTag">true or false</param>
        private static void ConstructOverlayDocuments(DocumentCollection documentCollection,
            List<DocumentDetail> nativeDocumentListUpdate, bool isDeleteTag)
        {
            var imagesetId = string.Empty;
            var imageDocumentList = documentCollection.documents.FindAll(i => i.docType == DocumentsetType.ImageSet);
            if (imageDocumentList.Count > 0)
            {
                imagesetId = imageDocumentList.FirstOrDefault().document.CollectionId.ToLower();
            }
            //b) Set Existing Fields 
            foreach (var docs in nativeDocumentListUpdate)
            {
                foreach (var field in docs.OverlayReImportField)
                {
                    if (!docs.document.FieldList.Exists(f => f.FieldName.ToLower() == field.FieldName.ToLower()))
                    {
                        if (field.FieldName.ToLower() == EVSystemFields.ImageSets.ToLower() &&
                            !string.IsNullOrEmpty(imagesetId))
                        {
                            var imagesetValue = (!string.IsNullOrEmpty(field.FieldValue)
                                ? field.FieldValue
                                : string.Empty);
                            field.FieldValue = (!string.IsNullOrEmpty(imagesetValue)
                                ? (imagesetValue + "," + imagesetId)
                                : imagesetValue);
                        }
                        else if (field.FieldName.ToLower() == EVSystemFields.Tag.ToLower() && isDeleteTag)
                        {
                            field.FieldValue = docs.SystemTags != null && docs.SystemTags.Any()
                                ? string.Format(EVSearchSyntax.TagValueFormat + "{0}", docs.SystemTags.First().Id)
                                : string.Empty;
                        }
                        field.IsRequired = true;
                        docs.document.FieldList.Add(field);
                    }
                }

                var redactionTextMatch = EVSystemFields.RedactionText.ToLower();
                var markupMatch = EVSystemFields.MarkUp.ToLower();

                #region Remove Redactions in NativeSet

                if (documentCollection.IsIncludeNativeFile)
                {
                    docs.document.FieldList.RemoveAll(field => field.FieldName.ToLower().Equals(redactionTextMatch));
                    docs.document.FieldList.RemoveAll(field => field.FieldName.ToLower().Equals(markupMatch));
                }

                #endregion

                #region Remove Redactions in ImageSet

                if (!string.IsNullOrWhiteSpace(imagesetId))
                {
                    redactionTextMatch = string.Format("{0}_{1}", imagesetId.Replace('-', '_'), redactionTextMatch);
                    markupMatch = string.Format("{0}_{1}", imagesetId.Replace('-', '_'), markupMatch);

                    docs.document.FieldList.RemoveAll(field => field.FieldName.ToLower().Equals(redactionTextMatch));
                    docs.document.FieldList.RemoveAll(field => field.FieldName.ToLower().Equals(markupMatch));
                }

                #endregion
            }
        }
        /// <summary>
        /// Method to add removable fields
        /// </summary>
        /// <param name="documentCollection">DocumentCollection</param>
        /// <param name="imagesetId">string</param>
        /// <param name="doc">DocumentDetail</param>
        private void ResetRedactableFields(DocumentCollection documentCollection, string imagesetId, DocumentDetail doc)
        {
            #region Remove Tag

            //Removing the Tag field for overlay updated documents here.        
            if (documentCollection.IsDeleteTagsForOverlay)
            {
                var tagField = new RVWDocumentFieldBEO
                {
                    FieldName = EVSystemFields.Tag.ToLower(),
                    FieldValue =
                        doc.SystemTags != null && doc.SystemTags.Any()
                            ? string.Format(EVSearchSyntax.TagValueFormat + "{0}", doc.SystemTags.First().Id)
                            : string.Empty
                };
                doc.document.FieldList.Add(tagField);
            }

            #endregion

            #region Remove Redactions In nativeSet

            var redactionText = EVSystemFields.RedactionText.ToLower();
            var markup = EVSystemFields.MarkUp.ToLower();

            if (documentCollection.IsIncludeNativeFile)
            {
                if (!doc.document.FieldList.Exists(field => field.FieldName.ToLower().Equals(redactionText)))
                {
                    doc.document.FieldList.Add(new RVWDocumentFieldBEO
                    {
                        IsRequired = true,
                        FieldName = redactionText,
                        FieldValue = string.Empty
                    });
                }
                else
                {
                    doc.document.FieldList.Where(field => field.FieldName.ToLower().Equals(redactionText)).ToList().
                        ForEach(x => x.FieldValue = string.Empty);
                }
                if (!doc.document.FieldList.Exists(field => field.FieldName.ToLower().Equals(markup)))
                {
                    doc.document.FieldList.Add(new RVWDocumentFieldBEO
                    {
                        IsRequired = true,
                        FieldName = markup,
                        FieldValue = string.Empty
                    });
                }
                else
                {
                    doc.document.FieldList.Where(field => field.FieldName.ToLower().Equals(markup)).ToList().
                        ForEach(x => x.FieldValue = string.Empty);
                }
            }

            #endregion

            #region Remove Redactions In ImageSet

            if (!string.IsNullOrWhiteSpace(imagesetId))
            {
                redactionText = string.Format("{0}_{1}", imagesetId.Replace('-', '_'), redactionText);
                markup = string.Format("{0}_{1}", imagesetId.Replace('-', '_'), markup);

                if (!doc.document.FieldList.Exists(field => field.FieldName.ToLower().Equals(redactionText)))
                {
                    doc.document.FieldList.Add(new RVWDocumentFieldBEO
                    {
                        IsRequired = true,
                        FieldName = redactionText,
                        FieldValue = string.Empty
                    });
                }
                else
                {
                    doc.document.FieldList.Where(field => field.FieldName.ToLower().Equals(redactionText))
                        .ToList()
                        .ForEach(x => x.FieldValue = string.Empty);
                }
                if (!doc.document.FieldList.Exists(field => field.FieldName.ToLower().Equals(markup)))
                {
                    doc.document.FieldList.Add(new RVWDocumentFieldBEO
                    {
                        IsRequired = true,
                        FieldName = markup,
                        FieldValue = string.Empty
                    });
                }
                else
                {
                    doc.document.FieldList.Where(field => field.FieldName.ToLower().Equals(markup)).ToList().
                        ForEach(x => x.FieldValue = string.Empty);
                }
            }

            #endregion
        }
 /// <summary>
 /// Overlays new ImagesetId against all documents
 /// </summary>
 /// <param name="documents"></param>
 /// <param name="record"></param>
 private void AssignImagesetIdentifiers(ref List<DocumentDetail> documents, DocumentCollection record)
 {
     // Determine for ImageSetId to update against all documents
     var imageSetDoc =
         record.documents.FirstOrDefault(i => i.docType == DocumentsetType.ImageSet);
     if (imageSetDoc != null)
     {
         documents.ForEach(d =>
             d.document.FieldList.Add(new RVWDocumentFieldBEO
             {
                 FieldName = EVSystemFields.ImageSets.ToLower(),
                 FieldValue = imageSetDoc.document.CollectionId
             }));
     }
     AssignImagesetIdentifiers(ref documents);
 }