/// <summary>
/// Process each document
/// </summary>
private void ProcessDocuments(DocumentCollection recordParserResponse)
{
var documentDetailList = new List<DocumentDetail>();
var overlayLogList = new List<JobWorkerLog<OverlaySearchLogInfo>>();
var docManager = new OverlayDocumentManager(_jobParameter, _dataset, PipelineId, WorkerId);
//Bulk Search for entire batch
documentDetailList = docManager.BulkSearch(recordParserResponse.documents, _userInfo, out overlayLogList);
SendRelationshipsInfo(documentDetailList);
#region Send Message
//Send response
if (documentDetailList != null && documentDetailList.Any())
{
if (documentDetailList.Count > _batchSize)
{
Send(documentDetailList.Take(_batchSize).ToList());
var remainDocumentList = documentDetailList.Skip(_batchSize).ToList();
Send(remainDocumentList); //send remaining list
documentDetailList.Clear();
}
else
{
Send(documentDetailList);
documentDetailList.Clear();
}
}
//Send Log
if (overlayLogList != null && overlayLogList.Any())
{
SendLog(overlayLogList);
overlayLogList.Clear();
}
#endregion
}
/// <summary>
/// Send data to Data pipe
/// </summary>
/// <param name="docDetails"></param>
private void Send(List<DocumentDetail> docDetails)
{
bool overlayIsRemoveTag = _jobParameter != null && _jobParameter != null && !_jobParameter.IsOverlayRetainTags;
var documentList = new DocumentCollection() { documents = docDetails, dataset = _dataset, IsDeleteTagsForOverlay = overlayIsRemoveTag, IsIncludeNativeFile = _isIncludeNativeFile };
var message = new PipeMessageEnvelope()
{
Body = documentList
};
OutputDataPipe.Send(message);
IncreaseProcessedDocumentsCount(docDetails.Count);
}
/// <summary>
/// Sends the specified document batch to next worker in the pipeline.
/// </summary>
/// <param name="documentBatch">The document batch.</param>
/// <param name="isIncludeOutlookMailStoreDataEntityIfAvailable">if set to <c>true</c> [includes outlook mail store data entity in the send list (when available)].</param>
private void Send(IEnumerable<RVWDocumentBEO> documentBatch, bool isIncludeOutlookMailStoreDataEntityIfAvailable)
{
try
{
DocumentCollection documentCollection = null;
List<DocumentDetail> documentDetailList = new List<DocumentDetail>();
if (documentBatch != null)
{
foreach (RVWDocumentBEO document in documentBatch)
{
m_CounterForCorrelationId += 1;
DocumentDetail documentDetail = new DocumentDetail
{
CorrelationId = m_CounterForCorrelationId.ToString(),
docType = DocumentsetType.NativeSet,
document = document,
IsNewDocument = true
};
documentDetailList.Add(documentDetail);
}
documentCollection = new DocumentCollection
{
dataset = m_Dataset,
documents = documentDetailList
};
}
if (documentCollection != null)
{
Pipe vaultOutDataPipe = GetOutputDataPipe("Vault");
var message = new PipeMessageEnvelope()
{
Body = documentCollection
};
if (vaultOutDataPipe != null)
{
vaultOutDataPipe.Send(message);
}
}
// All available outlook mail stores are sent to the queue. they shouldn't be sent again. So clear existing list.
if (isIncludeOutlookMailStoreDataEntityIfAvailable && m_OutlookMailStoreDataEntities != null
&& m_OutlookMailStoreDataEntities.Count > 0)
{
var message = new PipeMessageEnvelope()
{
Body = new EDocsDocumentCollection
{
OutlookMailStoreDataEntity = m_OutlookMailStoreDataEntities
}
};
Pipe eDocsOutlookEmailGeneratorOutputDataPipe = GetOutputDataPipe("EDocsOutlookEmailGenerator");
if (eDocsOutlookEmailGeneratorOutputDataPipe != null)
{
eDocsOutlookEmailGeneratorOutputDataPipe.Send(message);
}
m_OutlookMailStoreDataEntities.Clear();
}
IncreaseProcessedDocumentsCount(documentCollection.documents.Count);
}
catch (Exception ex)
{
ex.Trace().Swallow();
}
}
/// <summary>
/// Send data to Data pipe
/// </summary>
/// <param name="docDetails"></param>
private void Send(List<DocumentDetail> docDetails)
{
OutputDataPipe.ShouldNotBe(null);
var documentList = new DocumentCollection() { documents = docDetails, dataset = _dataset };
var message = new PipeMessageEnvelope()
{
Body = documentList
};
OutputDataPipe.Send(message);
IncreaseProcessedDocumentsCount(docDetails.Count);
}
/// <summary>
/// Send data to Data pipe
/// </summary>
/// <param name="docDetails"></param>
private void Send(List<DocumentDetail> docDetails)
{
var documentList = new DocumentCollection
{
documents = docDetails,
dataset = _dataset,
IsDeleteTagsForOverlay = false,
IsIncludeNativeFile = _isIncludeNativeFile
};
var message = new PipeMessageEnvelope
{
Body = documentList
};
OutputDataPipe.Send(message);
IncreaseProcessedDocumentsCount(docDetails.Count);
}
private void Send(List<DocumentDetail> docDetails)
{
var documentList = new DocumentCollection() { documents = docDetails, dataset = _dataset };
var message = new PipeMessageEnvelope()
{
Body = documentList
};
OutputDataPipe.Send(message);
//IncreaseProcessedDocumentsCount(docDetails.Count(DocumentDetail => DocumentDetail.docType == DocumentsetType.NativeSet));
IncreaseProcessedDocumentsCount(docDetails.Count);
}
/// <summary>
/// To send the collection to the pipeline director.
/// </summary>
/// <param name="collection"></param>
private void Send(DocumentCollection collection)
{
var message = new PipeMessageEnvelope
{
Body = collection
};
OutputDataPipe.Send(message);
}
/// <summary>
/// Sends the log.
/// </summary>
/// <param name="documentCollection">The document collection.</param>
/// <param name="isSentForIndexing">if set to <c>true</c> [is sent for indexing].</param>
/// <param name="documentErrorCollection"></param>
private void SendLog(DocumentCollection documentCollection, bool isSentForIndexing,DocumentErrorCollection documentErrorCollection=null)
{
if (documentCollection == null || documentCollection.documents == null) return;
var message = isSentForIndexing ? "Sent for indexing." : "Failed to send for indexing.";
var nativeDocumentList =
documentCollection.documents.FindAll(
n => n.docType == DocumentsetType.NativeSet);
if (!nativeDocumentList.Any()) return;
var searchIndexLogInfos = new List<JobWorkerLog<SearchIndexLogInfo>>();
try
{
foreach (var documentDetail in nativeDocumentList)
{
if (documentDetail.document == null) continue;
var logInfo = new SearchIndexLogInfo
{
Information =
string.Format("DCN:{0}", documentDetail.document.DocumentControlNumber),
DocumentId = documentDetail.document.DocumentId,
DCNNumber = documentDetail.document.DocumentControlNumber,
CrossReferenceField = documentDetail.document.CrossReferenceFieldValue,
Message = message
};
SetDocumentError(documentErrorCollection, documentDetail, logInfo);
if (String.IsNullOrEmpty(documentDetail.CorrelationId))
{
documentDetail.CorrelationId = "0";
}
var searchIndexLogInfo = new JobWorkerLog<SearchIndexLogInfo>
{
JobRunId = Convert.ToInt32(PipelineId),
CorrelationId = long.Parse(documentDetail.CorrelationId),
WorkerInstanceId = WorkerId,
WorkerRoleType = "8A65E2DC-753C-E311-82FA-005056850057",
Success = isSentForIndexing,
LogInfo = logInfo
};
searchIndexLogInfos.Add(searchIndexLogInfo);
}
LogPipe.Open();
var pipleMessageEnvelope = new PipeMessageEnvelope
{
Body = searchIndexLogInfos
};
LogPipe.Send(pipleMessageEnvelope);
}
catch (Exception exception)
{
exception.AddDbgMsg("Failed to log document details");
exception.Trace().Swallow();
ReportToDirector(exception);
}
}
/// <summary>
/// Constructs the overlay fields
/// </summary>
/// <param name="documentCollection">DocumentCollection</param>
/// <param name="nativeDocumentListUpdate">list of DocumentDetail</param>
/// <param name="isDeleteTag">true or false</param>
private static void ConstructOverlayDocuments(DocumentCollection documentCollection,
List<DocumentDetail> nativeDocumentListUpdate, bool isDeleteTag)
{
var imagesetId = string.Empty;
var imageDocumentList = documentCollection.documents.FindAll(i => i.docType == DocumentsetType.ImageSet);
if (imageDocumentList.Count > 0)
{
imagesetId = imageDocumentList.FirstOrDefault().document.CollectionId.ToLower();
}
//b) Set Existing Fields
foreach (var docs in nativeDocumentListUpdate)
{
foreach (var field in docs.OverlayReImportField)
{
if (!docs.document.FieldList.Exists(f => f.FieldName.ToLower() == field.FieldName.ToLower()))
{
if (field.FieldName.ToLower() == EVSystemFields.ImageSets.ToLower() &&
!string.IsNullOrEmpty(imagesetId))
{
var imagesetValue = (!string.IsNullOrEmpty(field.FieldValue)
? field.FieldValue
: string.Empty);
field.FieldValue = (!string.IsNullOrEmpty(imagesetValue)
? (imagesetValue + "," + imagesetId)
: imagesetValue);
}
else if (field.FieldName.ToLower() == EVSystemFields.Tag.ToLower() && isDeleteTag)
{
field.FieldValue = docs.SystemTags != null && docs.SystemTags.Any()
? string.Format(EVSearchSyntax.TagValueFormat + "{0}", docs.SystemTags.First().Id)
: string.Empty;
}
field.IsRequired = true;
docs.document.FieldList.Add(field);
}
}
var redactionTextMatch = EVSystemFields.RedactionText.ToLower();
var markupMatch = EVSystemFields.MarkUp.ToLower();
#region Remove Redactions in NativeSet
if (documentCollection.IsIncludeNativeFile)
{
docs.document.FieldList.RemoveAll(field => field.FieldName.ToLower().Equals(redactionTextMatch));
docs.document.FieldList.RemoveAll(field => field.FieldName.ToLower().Equals(markupMatch));
}
#endregion
#region Remove Redactions in ImageSet
if (!string.IsNullOrWhiteSpace(imagesetId))
{
redactionTextMatch = string.Format("{0}_{1}", imagesetId.Replace('-', '_'), redactionTextMatch);
markupMatch = string.Format("{0}_{1}", imagesetId.Replace('-', '_'), markupMatch);
docs.document.FieldList.RemoveAll(field => field.FieldName.ToLower().Equals(redactionTextMatch));
docs.document.FieldList.RemoveAll(field => field.FieldName.ToLower().Equals(markupMatch));
}
#endregion
}
}
/// <summary>
/// Method to add removable fields
/// </summary>
/// <param name="documentCollection">DocumentCollection</param>
/// <param name="imagesetId">string</param>
/// <param name="doc">DocumentDetail</param>
private void ResetRedactableFields(DocumentCollection documentCollection, string imagesetId, DocumentDetail doc)
{
#region Remove Tag
//Removing the Tag field for overlay updated documents here.
if (documentCollection.IsDeleteTagsForOverlay)
{
var tagField = new RVWDocumentFieldBEO
{
FieldName = EVSystemFields.Tag.ToLower(),
FieldValue =
doc.SystemTags != null && doc.SystemTags.Any()
? string.Format(EVSearchSyntax.TagValueFormat + "{0}", doc.SystemTags.First().Id)
: string.Empty
};
doc.document.FieldList.Add(tagField);
}
#endregion
#region Remove Redactions In nativeSet
var redactionText = EVSystemFields.RedactionText.ToLower();
var markup = EVSystemFields.MarkUp.ToLower();
if (documentCollection.IsIncludeNativeFile)
{
if (!doc.document.FieldList.Exists(field => field.FieldName.ToLower().Equals(redactionText)))
{
doc.document.FieldList.Add(new RVWDocumentFieldBEO
{
IsRequired = true,
FieldName = redactionText,
FieldValue = string.Empty
});
}
else
{
doc.document.FieldList.Where(field => field.FieldName.ToLower().Equals(redactionText)).ToList().
ForEach(x => x.FieldValue = string.Empty);
}
if (!doc.document.FieldList.Exists(field => field.FieldName.ToLower().Equals(markup)))
{
doc.document.FieldList.Add(new RVWDocumentFieldBEO
{
IsRequired = true,
FieldName = markup,
FieldValue = string.Empty
});
}
else
{
doc.document.FieldList.Where(field => field.FieldName.ToLower().Equals(markup)).ToList().
ForEach(x => x.FieldValue = string.Empty);
}
}
#endregion
#region Remove Redactions In ImageSet
if (!string.IsNullOrWhiteSpace(imagesetId))
{
redactionText = string.Format("{0}_{1}", imagesetId.Replace('-', '_'), redactionText);
markup = string.Format("{0}_{1}", imagesetId.Replace('-', '_'), markup);
if (!doc.document.FieldList.Exists(field => field.FieldName.ToLower().Equals(redactionText)))
{
doc.document.FieldList.Add(new RVWDocumentFieldBEO
{
IsRequired = true,
FieldName = redactionText,
FieldValue = string.Empty
});
}
else
{
doc.document.FieldList.Where(field => field.FieldName.ToLower().Equals(redactionText))
.ToList()
.ForEach(x => x.FieldValue = string.Empty);
}
if (!doc.document.FieldList.Exists(field => field.FieldName.ToLower().Equals(markup)))
{
doc.document.FieldList.Add(new RVWDocumentFieldBEO
{
IsRequired = true,
FieldName = markup,
FieldValue = string.Empty
});
}
else
{
doc.document.FieldList.Where(field => field.FieldName.ToLower().Equals(markup)).ToList().
ForEach(x => x.FieldValue = string.Empty);
}
}
#endregion
}
/// <summary>
/// Overlays new ImagesetId against all documents
/// </summary>
/// <param name="documents"></param>
/// <param name="record"></param>
private void AssignImagesetIdentifiers(ref List<DocumentDetail> documents, DocumentCollection record)
{
// Determine for ImageSetId to update against all documents
var imageSetDoc =
record.documents.FirstOrDefault(i => i.docType == DocumentsetType.ImageSet);
if (imageSetDoc != null)
{
documents.ForEach(d =>
d.document.FieldList.Add(new RVWDocumentFieldBEO
{
FieldName = EVSystemFields.ImageSets.ToLower(),
FieldValue = imageSetDoc.document.CollectionId
}));
}
AssignImagesetIdentifiers(ref documents);
}