protected override void ProcessMessage(PipeMessageEnvelope envelope)
{
DocumentCollection documentCollection = null;
#region Extract document details from incoming message object.
documentCollection = envelope.Body as DocumentCollection;
#region Assertion
documentCollection.ShouldNotBe(null);
#endregion
try
{
//get the documents yet to converted
List<ConversionDocumentBEO> documentsInConversionQueue = _nearNativeConverter.GetDocumentsQueuedForConversion(WorkAssignment.JobId);
if(documentsInConversionQueue != null && documentsInConversionQueue.Any())
{
//this is to avoid tight loop which consumes resources, especially network bandwidth
int waitTime = GetConversionValidationWaitTime(documentsInConversionQueue);
Thread.Sleep(waitTime);
InputDataPipe.Purge(); // We don't need more than one "try again later" message in that pipe
InputDataPipe.Send(envelope);
}
//IncreaseProcessedDocumentsCount(0);
}
catch (Exception ex)
{
ReportToDirector(ex);
ex.Trace().Swallow();
}
}
protected override void ProcessMessage(PipeMessageEnvelope message)
{
try
{
var recordInfo = (ProjectDocumentRecordInfo)message.Body;
recordInfo.ShouldNotBe(null);
_docStart = recordInfo.StartNumber;
_docEnd = recordInfo.EndNumber;
var projectDocumentDataList = GetDocuments();
if (projectDocumentDataList==null) return;
SetDocumentIndexStatusAndContentSize(projectDocumentDataList);
Send(projectDocumentDataList);
IncreaseProcessedDocumentsCount(projectDocumentDataList.Count()); //Progress Status
}
catch (Exception ex)
{
ex.Trace().Swallow();
ReportToDirector(ex);
}
}
/// <summary>
/// overrides base ProcessMessage method
/// </summary>
/// <param name="envelope"></param>
protected override void ProcessMessage(PipeMessageEnvelope envelope)
{
var reviewsetRecord = (DocumentRecordCollection) envelope.Body;
reviewsetRecord.ShouldNotBe(null);
reviewsetRecord.ReviewsetDetails.CreatedBy.ShouldNotBeEmpty();
reviewsetRecord.Documents.ShouldNotBe(null);
try
{
if (reviewsetRecord.DocumentTags == null || !reviewsetRecord.DocumentTags.Any())
{
Send(reviewsetRecord);
}
else
{
UpdateTag(reviewsetRecord);
Send(reviewsetRecord);
}
}
catch (Exception ex)
{
ReportToDirector(ex);
ex.Trace().Swallow();
LogMessage(false, string.Format("Error in IndexTaggerWorker - Exception: {0}", ex.ToUserString()),
reviewsetRecord.ReviewsetDetails.CreatedBy, reviewsetRecord.ReviewsetDetails.ReviewSetName);
}
}
/// <summary>
/// This method processes the pipe message and updates the document in Search Sub System
/// </summary>
/// <param name="envelope"></param>
protected override void ProcessMessage(PipeMessageEnvelope envelope)
{
var documentRecords = envelope.Body as DocumentRecordCollection;
Debug.Assert(documentRecords != null, "documentRecords != null");
documentRecords.Documents.Count.ShouldBeGreaterThan(0);
documentRecords.ReviewsetDetails.ShouldNotBe(null);
documentRecords.ReviewsetDetails.ReviewSetId.ShouldNotBeEmpty();
documentRecords.ReviewsetDetails.BinderId.ShouldNotBeEmpty();
try
{
UpdateDocuments(documentRecords);
if (documentRecords.ReviewsetDetails.Activity == "Split")
{
Send(documentRecords);
}
else
{
Send(documentRecords.ReviewsetDetails);
}
}
catch (Exception ex)
{
ReportToDirector(ex);
ex.Trace().Swallow();
LogMessage(false, string.Format("Error in IndexUpdateWorker - Exception: {0}", ex.ToUserString()),
documentRecords.ReviewsetDetails.CreatedBy, documentRecords.ReviewsetDetails.ReviewSetName);
}
IncreaseProcessedDocumentsCount(documentRecords.Documents.Count);
}
/// <summary>
/// overrides base ProcessMessage method
/// </summary>
/// <param name="envelope"></param>
protected override void ProcessMessage(PipeMessageEnvelope envelope)
{
var reviewsetRecord = (DocumentRecordCollection) envelope.Body;
reviewsetRecord.ShouldNotBe(null);
reviewsetRecord.ReviewsetDetails.CreatedBy.ShouldNotBeEmpty();
reviewsetRecord.Documents.ShouldNotBe(null);
_createdBy = reviewsetRecord.ReviewsetDetails.CreatedBy;
if (EVHttpContext.CurrentContext == null)
{
// Moq the session
MockSession();
}
try
{
//stores converted document list
var tempList = ConvertDocumentRecordtoReviewsetDocument(reviewsetRecord.Documents,
reviewsetRecord.ReviewsetDetails);
TagDocumentsNotReviewed(reviewsetRecord, tempList);
}
catch (Exception ex)
{
ReportToDirector(ex);
ex.Trace().Swallow();
LogMessage(false, string.Format("Error in VaultTaggerWorker - Exception: {0}", ex.ToUserString()),
reviewsetRecord.ReviewsetDetails.CreatedBy, reviewsetRecord.ReviewsetDetails.ReviewSetName);
}
}
/// <summary>
/// overrides base ProcessMessage method
/// </summary>
/// <param name="envelope"></param>
protected override void ProcessMessage(PipeMessageEnvelope envelope)
{
try
{
var reviewsetRecord = (DocumentRecordCollection)envelope.Body;
reviewsetRecord.ShouldNotBe(null);
reviewsetRecord.Documents.ShouldNotBe(null);
reviewsetRecord.ReviewsetDetails.ShouldNotBe(null);
reviewsetRecord.ReviewsetDetails.CreatedBy.ShouldNotBeEmpty();
reviewsetRecord.ReviewsetDetails.DatasetId.ShouldNotBe(0);
reviewsetRecord.ReviewsetDetails.MatterId.ShouldNotBe(0);
reviewsetRecord.ReviewsetDetails.SplitReviewSetId.ShouldNotBeEmpty();
reviewsetRecord.ReviewsetDetails.BinderId.ShouldNotBeEmpty();
_createdBy = reviewsetRecord.ReviewsetDetails.CreatedBy;
_totalDocumentCount = reviewsetRecord.TotalDocumentCount;
var removeTmpList = ConvertDocumentRecordtoReviewsetDocument(reviewsetRecord.Documents,
reviewsetRecord.ReviewsetDetails.CreatedBy, reviewsetRecord.ReviewsetDetails.SplitReviewSetId);
UnAssignReviewsetInVault(removeTmpList, reviewsetRecord.ReviewsetDetails.DatasetId);
UnAssignReviewsetInSearchIndex(removeTmpList, reviewsetRecord.ReviewsetDetails);
////TODO: Search Engine Replacement - Search Sub System - Update ReviewSetId in search engine
Send(reviewsetRecord);
}
catch (Exception ex)
{
ReportToDirector(ex);
ex.Trace().Swallow();
}
}
protected override void ProcessMessage(PipeMessageEnvelope message)
{
try
{
var projectDocumentCollection = (ProjectDocumentCollection)message.Body;
projectDocumentCollection.ShouldNotBe(null);
projectDocumentCollection.Documents.ShouldNotBe(null);
var documents = projectDocumentCollection.Documents.Select(projectDocument => new Business.Analytics.DocumentIdentifier
{
ReferenceId = projectDocument.DocumentReferenceId
}).ToList();
_analyticProject = new AnalyticsProject();
_analyticProject.UpdateProjectFieldForDeleteDocuments(_jobParameter.MatterId, _dataset.CollectionId, _jobParameter, documents,
projectDocumentCollection.ProjectFieldId, WorkAssignment.JobId);
Send(projectDocumentCollection);
}
catch (Exception ex)
{
ex.Trace().Swallow();
ReportToDirector(ex);
}
}
protected override void ProcessMessage(PipeMessageEnvelope message)
{
try
{
if (!((bool)message.Body)) return;
//1) Index & Group documents
PerformNearDuplicateComputation();
//2) Get Result
var nearDuplicationResultDocuments = GetNearDuplicationResult();
//3) Send to Data pipe
if (nearDuplicationResultDocuments.Any())
SendDataPipe(nearDuplicationResultDocuments);
nearDuplicationResultDocuments.Clear();
IncreaseProcessedDocumentsCount(_processedDocumentResult);
if (_processedDocumentResult >= 1) return;
Tracer.Error(
"Near Duplication Processing Worker: No documents return from near duplicate result after perform computation for job run id :{0}",
PipelineId);
ConstructAndSendLog(Constants.NearDuplicationResultNoDocuments);
throw new EVException().AddUsrMsg(Constants.NearDuplicationResultNoDocuments);
}
catch (Exception ex)
{
ReportToDirector(ex);
ex.Trace().Swallow();
}
}
public PipelineSection(string machineName, uint orderNumber, PipelineType pipelineType, string pipelineId, RolePlan rolePlan)
{
OrderNumber = orderNumber;
RoleType = rolePlan.RoleType;
Name = rolePlan.Name;
OutputSectionNames = rolePlan.OutputSectionNames ?? new List <string>();
HiringPipeName = new HiringPipeName(machineName, pipelineId, Name);
using (var hiringPipe = new Pipe(HiringPipeName))
{
hiringPipe.Create();
for (uint slotNumber = 0; slotNumber < rolePlan.Slots; slotNumber++)
{
RoleSlotToken roleSlot = new RoleSlotToken(slotNumber);
var envelope = new PipeMessageEnvelope()
{
Body = roleSlot,
Label = "RoleSlot " + slotNumber,
};
hiringPipe.Send(envelope);
}
}
if (orderNumber > 0) // HACK - first role is not supposed to have data pipe
{
DataPipeName = new DataPipeName(machineName, pipelineType, pipelineId, Name);
using (var dataPipe = new Pipe(DataPipeName))
{
dataPipe.Create();
}
}
}
/// <summary>
/// Processes the work item.
/// </summary>
/// <param name="message">The message.</param>
protected override void ProcessMessage(PipeMessageEnvelope message)
{
try
{
DocumentCollection recordParserResponse = (DocumentCollection)message.Body;
#region Assertion
recordParserResponse.ShouldNotBe(null);
recordParserResponse.documents.ShouldNotBe(null);
#endregion
if (_jobParameter.IsAppend)
{
throw new Exception(Constants.ErrorMessageInvalidPipeLine);
}
if (recordParserResponse == null)
{
return;
}
if (recordParserResponse.dataset != null)
{
_dataset = recordParserResponse.dataset;
}
if (recordParserResponse.documents != null && recordParserResponse.documents.Any())
{
ProcessDocuments(recordParserResponse);
}
}
catch (Exception ex)
{
ReportToDirector(ex);
ex.Trace().Swallow();
}
}
/// <summary>
/// Process the message
/// </summary>
/// <param name="envelope">PipeMessageEnvelope</param>
protected override void ProcessMessage(PipeMessageEnvelope envelope)
{
var printCollection = (PrintDocumentCollection) envelope.Body;
InitializeForProcessing(BootParameters);
_mTotalDocumentCount = printCollection.TotalDocumentCount;
ProcessTheDocument(printCollection.Documents);
}
protected override void ProcessMessage(PipeMessageEnvelope message)
{
try
{
if (message.Body == null)
{
return;
}
var lawDocumentsList = (LawSyncDocumentCollection) message.Body;
if (_jobParameter == null)
{
if (lawDocumentsList.IsLawSyncReprocessJob)
{
_jobParameter = lawDocumentsList.OrginalJobParameter;
}
else
{
_jobParameter = (LawSyncBEO) XmlUtility.DeserializeObject(BootParameters, typeof (LawSyncBEO));
}
}
_redactableSetCollectionId = lawDocumentsList.RedactableSetCollectionId;
_datasetExtractionPath = lawDocumentsList.DatasetExtractionPath;
_datasetCollectionId = lawDocumentsList.DatasetCollectionId;
_lawSyncJobId = lawDocumentsList.LawSynJobId;
_logInfoList = new List<JobWorkerLog<LawSyncLogInfo>>();
_documentProcessStateList = new List<DocumentConversionLogBeo>();
var lawImagingDocuments = lawDocumentsList.Documents.Where(d => d.IsImaging).ToList();
if (_jobParameter.IsProduceImage && lawImagingDocuments.Any())
{
var documentIds = lawImagingDocuments.Select(d => d.DocumentReferenceId).ToList();
_documentsBinaryList = GetNearNativeFileForDocuments(_jobParameter.MatterId,
_redactableSetCollectionId,
documentIds);
foreach (var document in lawImagingDocuments)
{
SendDocumentForImaging(document);
}
}
if (_documentProcessStateList.Any())
{
UpdateDcoumentProcessState(_documentProcessStateList);
}
Send(lawDocumentsList);
if (_logInfoList != null && _logInfoList.Any())
{
SendLogPipe(_logInfoList);
}
}
catch (Exception ex)
{
ReportToDirector(ex);
ex.Trace().Swallow();
LogMessage(Constants.LawSyncFailureinImagingMessage + ex.ToUserString());
}
}
private void ProcessMessageNoTrans(PipeMessageEnvelope message)
{
using (var transactionSuppression = OverdriveTransactionScope.Suppress())
{
ProcessMessageNetTimed(message);
transactionSuppression.Complete();
}
}
/// <summary>
/// Send Worker response to Pipe.
/// </summary>
private void Send(ProjectDocumentCollection data)
{
var message = new PipeMessageEnvelope
{
Body = data
};
if (OutputDataPipe != null)
OutputDataPipe.Send(message);
IncreaseProcessedDocumentsCount(data.Documents.Count);
}
public void SendProblemReport(Pipe ReportPipe)
{
TimeStamp = DateTime.Now;
var envelope = new PipeMessageEnvelope()
{
Body = this,
Label = MessageLabel,
};
ReportPipe.Send(envelope);
}
public void SendProblemReport(Pipe ReportPipe)
{
TimeStamp = DateTime.Now;
var envelope = new PipeMessageEnvelope()
{
Body = this,
Label = MessageLabel,
};
ReportPipe.Send(envelope);
}
protected override void ProcessMessage(PipeMessageEnvelope message)
{
//Indicates the final message to do the final step
if (message.Label == "PleaseBuildProject")
{
BuildProject();
return;
}
var projectDocumentCollection = (ProjectDocumentCollection)message.Body;
IncreaseProcessedDocumentsCount(projectDocumentCollection.Documents.Count);
}
private void ReportStateChange()
{
WorkerStateChangedMessage message = new WorkerStateChangedMessage(WorkAssignment, State);
var envelope = new PipeMessageEnvelope()
{
Body = message,
Label = WorkerStateChangedMessage.MessageLabel,
};
ReportPipe.Send(envelope);
}
/// <summary>
/// Processes the work item. pushes give document files for tagging
/// </summary>
/// <param name="message">The message.</param>
protected override void ProcessMessage(PipeMessageEnvelope message)
{
Send(message);
var documentCollection = message.Body as DocumentCollection;
documentCollection.ShouldNotBe(null);
documentCollection.documents.ShouldNotBe(null);
documentCollection.documents.ShouldNotBeEmpty();
var nativeSet =
documentCollection.documents.FindAll(
d => (d.docType == DocumentsetType.NativeSet));
try
{
IDictionary<int, List<BulkDocumentInfoBEO>> tags = new Dictionary<int, List<BulkDocumentInfoBEO>>();
IDictionary<int, string> tagKeys = new Dictionary<int, string>();
//Converting to RVWDocument from Document detail object
var documents = ToDocumentBeoList(nativeSet);
foreach (var document in documents)
{
var bulkDocumentInfoBEO = new BulkDocumentInfoBEO
{
DocumentId = document.DocumentId,
DuplicateId = document.DuplicateId,
FromOriginalQuery = true,
CreatedBy = _jobParams.CreatedBy,
FamilyId = document.FamilyId
};
foreach (var tag in document.Tags)
{
if (tags.ContainsKey(tag.TagId))
{
var bulkDocumentInfoBeOs = tags.FirstOrDefault(t => t.Key == tag.TagId).Value;
bulkDocumentInfoBeOs.Add(bulkDocumentInfoBEO);
}
else
{
var bulkDocumentInfoBeOs = new List<BulkDocumentInfoBEO> {bulkDocumentInfoBEO};
tags.Add(tag.TagId, bulkDocumentInfoBeOs);
tagKeys.Add(tag.TagId, tag.TagName);
}
}
}
BulkTagging(tags, tagKeys);
LogTaggingMessage(nativeSet, true, TagSuccesMessage);
}
catch (Exception ex)
{
LogTaggingMessage(nativeSet, false, TagFailureMessage);
ReportToDirector(ex.ToUserString());
ex.Trace().Swallow();
}
}
public void Send(PipeMessageEnvelope pipeMessage)
{
try
{
if (null != Before) Before();
concretePipe.Send(pipeMessage);
}
finally
{
if (null != After) After();
}
}
/// <summary>
/// Reports the errors.
/// </summary>
/// <param name="logPipe"></param>
/// <param name="errorDocumentDetails">The error document details.</param>
/// <param name="pipelineId">The pipeline identifier.</param>
/// <param name="workerId">The worker identifier.</param>
/// <param name="workerRoleTypeId"></param>
/// <returns></returns>
internal static IEnumerable<JobWorkerLog<ProductionParserLogInfo>> SendProductionLogs(Pipe logPipe, IEnumerable<ProductionDocumentDetail> errorDocumentDetails, string pipelineId, string workerId, string workerRoleTypeId)
{
if(errorDocumentDetails==null) return null;
var productionParserLogInfos = new List<JobWorkerLog<ProductionParserLogInfo>>();
try
{
var jobRunId = Convert.ToInt64(pipelineId);
foreach (var errorDocument in errorDocumentDetails)
{
var parserLog = new JobWorkerLog<ProductionParserLogInfo>
{
JobRunId =jobRunId,
CorrelationId = errorDocument.CorrelationId,
WorkerRoleType = workerRoleTypeId,
WorkerInstanceId = workerId,
IsMessage = false,
Success = false,
CreatedBy = errorDocument.CreatedBy,
LogInfo =
new ProductionParserLogInfo
{
Information = errorDocument.ErrorMessage,
BatesNumber = errorDocument.AllBates,
DatasetName = errorDocument.OriginalDatasetName,
DCN = errorDocument.DCNNumber,
ProductionDocumentNumber = errorDocument.DocumentProductionNumber,
ProductionName = errorDocument.Profile.ProfileName
}
};
productionParserLogInfos.Add(parserLog);
if (logPipe == null)
Trace.TraceError("log pipe is empty");
logPipe.Open();
var message = new PipeMessageEnvelope()
{
Body = productionParserLogInfos
};
logPipe.Send(message);
}
}
catch (Exception exception)
{
var dcns = String.Join(",", errorDocumentDetails.Select(errorDoc => errorDoc.DCNNumber));
exception.AddDbgMsg("Failed to send documents to Redact-It and failed in reporting errors");
exception.AddDbgMsg(string.Format("Document Control Numbers:{0}", dcns));
exception.Trace().Swallow();
}
return productionParserLogInfos;
}
protected override void ProcessMessage(PipeMessageEnvelope message)
{
try
{
if (message.Body == null)
{
return;
}
var lawDocumentsList = (LawSyncDocumentCollection) message.Body;
if (_jobParameter == null)
{
if (lawDocumentsList.IsLawSyncReprocessJob)
{
_jobParameter = lawDocumentsList.OrginalJobParameter;
}
else
{
_jobParameter = (LawSyncBEO) XmlUtility.DeserializeObject(BootParameters, typeof (LawSyncBEO));
}
}
_datasetCollectionId = lawDocumentsList.DatasetCollectionId;
_lawSyncJobId = lawDocumentsList.LawSynJobId;
_logInfoList = new List<JobWorkerLog<LawSyncLogInfo>>();
_documentProcessStateList = new List<DocumentConversionLogBeo>();
_dataset = DataSetBO.GetDataSetDetailForDataSetId(_jobParameter.DatasetId);
if (lawDocumentsList.Documents.Any())
{
foreach (var lawDocument in lawDocumentsList.Documents)
{
SetMetadataForDcoument(lawDocument);
}
}
if (_documentProcessStateList.Any())
{
UpdateDcoumentProcessState(_documentProcessStateList);
}
Send(lawDocumentsList);
if (_logInfoList.Any())
{
SendLogPipe(_logInfoList);
}
}
catch (Exception ex)
{
ReportToDirector(ex);
ex.Trace().Swallow();
LogMessage(Constants.LawSyncFailureinGetMetadataMessage + ex.ToUserString());
}
}
private void SlowDownPostbacks(PipeMessageEnvelope message)
{
if (message.IsPostback)
{
TimeSpan timeSinceTheMessageWasSent = DateTime.UtcNow - message.SentTime.ToUniversalTime();
if (timeSinceTheMessageWasSent < postbackProcessingDelay)
{
int remainingTimeToWait =
(int)((postbackProcessingDelay - timeSinceTheMessageWasSent).TotalMilliseconds);
Thread.Sleep(remainingTimeToWait);
}
}
}
/// <summary>
/// overrides base ProcessMessage method
/// </summary>
/// <param name="envelope"></param>
protected override void ProcessMessage(PipeMessageEnvelope envelope)
{
try
{
var reviewsetRecord = (DocumentRecordCollection) envelope.Body;
WriteDocumentstoVault(reviewsetRecord);
Send(reviewsetRecord);
}
catch (Exception ex)
{
ReportToDirector(ex);
ex.Trace().Swallow();
}
}
public void ReportToDirector(string message)
{
WorkerMessage workerMessage = new WorkerMessage(WorkerId, message);
var envelope = new PipeMessageEnvelope()
{
Body = workerMessage,
Label = WorkerMessage.MessageLabel,
};
if (null != ReportPipe)
{
ReportPipe.Send(envelope);
}
}
/// <summary>
/// Send message
/// </summary>
private void SendMessage()
{
// We get here if Completed() is called for the first time.
// Need to send special message to DeleteProjectCleanup worker.
var pipelineSection = FindPipelineSection("IncludeSubSystemsFinal");
var dataPipeName = pipelineSection.DataPipeName;
using (var dataPipe = new Pipe(dataPipeName))
{
dataPipe.Open();
var envelope = new PipeMessageEnvelope() { Label = "PleaseBuildProject" };
dataPipe.Send(envelope);
}
_finalizationRequested = true;
}
protected override void ProcessMessage(PipeMessageEnvelope message)
{
var bulkTagRecord = (BulkTagRecord)message.Body;
try
{
bulkTagRecord.ShouldNotBe(null);
// Mock http context
bulkTagRecord.CreatedByUserGuid.ShouldNotBeEmpty();
MockSession(bulkTagRecord.CreatedByUserGuid);
if (_processedBatchCount == 0)
{
bulkTagRecord.NumberOfBatches.ShouldBeGreaterThan(0);
_totalBatchCount = bulkTagRecord.NumberOfBatches;
}
_processedBatchCount += 1;
Tracer.Info("Total # of batches processed so far : {0}", _processedBatchCount);
if (bulkTagRecord.Notification.DocumentsTagged != null)
{
_documentsTagged += bulkTagRecord.Notification.DocumentsTagged.Count;
}
if (bulkTagRecord.Notification.DocumentsFailed != null)
{
_documentsFailed += bulkTagRecord.Notification.DocumentsFailed.Count;
}
if (bulkTagRecord.Notification.DocumentsAlreadyTagged != null)
{
_documentsAlreadyTagged += bulkTagRecord.Notification.DocumentsAlreadyTagged.Count;
}
//Send out the completed notification, only all the batches are processed
if (_processedBatchCount != _totalBatchCount) return;
Tracer.Info("Total # of batches processed : {0}", _processedBatchCount);
Tracer.Info("Total # of documents tagged : {0}", _documentsTagged);
Tracer.Info("Total # of documents failed : {0}", _documentsFailed);
Tracer.Info("Total # of documents already tagged : {0}", _documentsAlreadyTagged);
UpdateTagStatistics(bulkTagRecord);
}
catch (Exception ex)
{
ex.Trace().Swallow();
ReportToDirector(ex);
}
}
public void ForceSend()
{
WorkTimeNet = _workTimeNet.Elapsed;
WorkTimeGross = _workTimeGross.Elapsed - _idleTime;
var envelope = new PipeMessageEnvelope()
{
Body = this,
Label = MessageLabel,
};
ReportPipe.Send(envelope);
PreviousHeartBeatSentTime = DateTime.Now;
_dirty = false;
}
protected override void ProcessMessage(PipeMessageEnvelope envelope)
{
List<ProductionDocumentDetail> productionDocuments = null;
List<ProductionDocumentDetail> errorDocuments = null;
try
{
productionDocuments = envelope.Body as List<ProductionDocumentDetail>;
Debug.Assert(productionDocuments != null, "productionDocuments != null");
if (productionDocuments == null || !productionDocuments.Any())
{
Tracer.Warning("Empty batch received! Skipped.");
return;
}
productionDocuments.ShouldNotBeEmpty();
using (new EVTransactionScope(TransactionScopeOption.Suppress))
{
if (productionDocuments != null && productionDocuments.Any())
{
var productionDocument = productionDocuments.First();
productionDocument.MatterId.ShouldNotBeEmpty();
_mMatterId = productionDocument.MatterId;
productionDocument.dataSetBeo.ShouldNotBe(null);
productionDocument.dataSetBeo.CollectionId.ShouldNotBeEmpty();
_mCollectionId = productionDocument.dataSetBeo.CollectionId;
var docs = productionDocuments.Select(doc => new DocumentDataBEO { DocumentId = doc.DocumentId, CollectionId = _mCollectionId }).ToList();
var docFields = MatterBO.GetCollectionDocumentFields(Int64.Parse(_mMatterId), docs);
docFields.ShouldNotBe(null);
_documentDetails = ConstructDocumentFieldsForSearchSubSystem(docFields);
ProcessDocumentFields(productionDocuments);
}
}
}
catch (Exception ex)
{
errorDocuments = productionDocuments;
ReportToDirector(ex);
ex.Trace().Swallow();
}
ProductionLogHelper.SendProductionLogs(LogPipe, errorDocuments, PipelineId, WorkerId,
Constants.ProductionVaultIndexingUpdateWokerRoleId);
}
public RoleSlotToken GetFreeSlot()
{
try
{
PipeMessageEnvelope envelope = HiringPipe.Receive(hiringPipeTimeout);
if (envelope != null)
{
object message = envelope.Body;
RoleSlotToken roleSlot = message as RoleSlotToken;
Debug.Assert(null != roleSlot);
return(roleSlot);
}
// Debugging
//if (SectionName == "S1")
//{
// throw new EVException().AddDbgMsg("Test");
//}
}
catch (Exception ex)
{
MessageQueueException messageQueueException = ex as MessageQueueException;
if (messageQueueException != null && (uint)messageQueueException.ErrorCode == 0x80004005)
{
Tracer.Debug("Cannot find hiring pipe {0}.{1}, so skip processing it.",
WorkRequest.HiringPipeName.SectionName, WorkRequest.PipelineId);
}
else
{
ProblemReport problemReport = new ProblemReport(WorkRequest.SectionName, WorkerStage.Unknown, ex);
using (Pipe reportPipe = new Pipe(WorkRequest.ReportPipeName))
{
try
{
reportPipe.Open();
problemReport.SendProblemReport(reportPipe);
}
catch (Exception innerEx)
{
innerEx.Trace().Swallow();
}
}
}
throw;
}
return(null);
}
protected override void ProcessMessage(PipeMessageEnvelope message)
{
try
{
if (message.Body == null)
{
return;
}
var lawDocumentsList = (LawSyncDocumentCollection) message.Body;
if (_jobParameter == null)
{
if (lawDocumentsList.IsLawSyncReprocessJob)
_jobParameter = lawDocumentsList.OrginalJobParameter;
else
_jobParameter = (LawSyncBEO) XmlUtility.DeserializeObject(BootParameters, typeof (LawSyncBEO));
}
_datasetCollectionId = lawDocumentsList.DatasetCollectionId;
_lawSyncJobId = lawDocumentsList.LawSynJobId;
_logInfoList = new List<JobWorkerLog<LawSyncLogInfo>>();
_documentProcessStateList = new List<DocumentConversionLogBeo>();
_lawEvAdapter = LawBO.GetLawAdapter(_jobParameter.LawCaseId);
_imageArchiveDirectory = _lawEvAdapter.GetImageArchiveDirectory();
var lawImagingDocuments = lawDocumentsList.Documents.Where(d => d.IsImaging).ToList();
if (_jobParameter.IsProduceImage && lawImagingDocuments.Any())
{
_logInfoList = new List<JobWorkerLog<LawSyncLogInfo>>();
ProcessDocumentImages(lawDocumentsList);
}
if (_logInfoList.Any())
{
SendLogPipe(_logInfoList);
}
if (_documentProcessStateList.Any())
{
UpdateDcoumentProcessState(_documentProcessStateList);
}
}
catch (Exception ex)
{
ReportToDirector(ex);
ex.Trace().Swallow();
LogMessage(Constants.LawSyncFailureinSyncImageMessage + ex.ToUserString());
}
}
public void Send(PipeMessageEnvelope pipeMessage)
{
try
{
if (null != Before)
{
Before();
}
concretePipe.Send(pipeMessage);
}
finally
{
if (null != After)
{
After();
}
}
}
/// <summary>
/// Send message
/// </summary>
private void SendMessage()
{
// We get here if Completed() is called for the first time.
// Need to send special message to DeleteProjectCleanup worker.
var pipelineSection = FindPipelineSection("IncludeSubSystemsFinal");
var dataPipeName = pipelineSection.DataPipeName;
using (var dataPipe = new Pipe(dataPipeName))
{
dataPipe.Open();
var envelope = new PipeMessageEnvelope()
{
Label = "PleaseBuildProject"
};
dataPipe.Send(envelope);
}
_finalizationRequested = true;
}
private void ProcessMessageNetTimed(PipeMessageEnvelope message)
{
//long envelopeSize = Utils.BinSizeOf(message);
//long bodySize = Utils.BinSizeOf(message.Body);
//Tracer.Trace("Worker {0} received message of the size {1}/{2}", WorkerId, envelopeSize, bodySize);
try
{
//Tracer.Trace("Worker {0} processing message", WorkerId);
WorkerStatistics.PunchInNet();
ProcessMessage(message);
//Tracer.Trace("Worker {0} processed message", WorkerId);
}
finally
{
WorkerStatistics.PunchOutNet();
WorkerStatistics.RecordInputTraffic(message);
}
}
private void UpdatePipelineState(PipeMessageEnvelope envelope, out bool continueMessagePumping)
{
continueMessagePumping = true;
if (envelope != null)
{
ProcessReport(envelope);
if (PipelineStatus.GetMostRecentState() == PipelineState.AllWorkersQuit)
{
continueMessagePumping = false;
}
}
else
{
// No reports to process, so we have a little idle time
CheckPipelineForCompletion();
}
ReportPipelineStatus("PIPELINE STATUS: ", false);
}
internal static PipeMessageEnvelope GetPipeMessage(this Message msmqMessage)
{
msmqMessage.Formatter = MSMQPipe.OverdriveMessageFormatter;
var pipeMessage = new PipeMessageEnvelope();
pipeMessage.Id = msmqMessage.Id;
pipeMessage.CorrelationId = msmqMessage.CorrelationId;
pipeMessage.Label = msmqMessage.Label;
if (msmqMessage.BodyStream.Length > 0)
{
pipeMessage.Body = msmqMessage.Body;
}
CompressFormatter compressFormatter = msmqMessage.Formatter as CompressFormatter;
if (null != compressFormatter)
{
pipeMessage.BodyLength = compressFormatter.OriginalBodyLength; // Thread local property
pipeMessage.CompressedBodyLength = compressFormatter.CompressedBodyLength; // Thread local property
}
else
{
pipeMessage.CompressedBodyLength = 0; // This only works with compressing formatter
try
{
pipeMessage.BodyLength = msmqMessage.BodyStream.Length;
}
catch (Exception)
{
pipeMessage.BodyLength = 0;
}
}
pipeMessage.SentTime = msmqMessage.SentTime;
MessageFlags appSpecific = (MessageFlags)msmqMessage.AppSpecific;
if ((appSpecific & MessageFlags.Postback) != 0)
{
pipeMessage.IsPostback = true;
}
return pipeMessage;
}
protected override void ProcessMessage(PipeMessageEnvelope message)
{
message.ShouldNotBe(null);
var bulkTagRecord = (BulkTagRecord)message.Body;
try
{
bulkTagRecord.ShouldNotBe(null);
bulkTagRecord.CreatedByUserGuid.ShouldNotBeEmpty();
MockSession(bulkTagRecord.CreatedByUserGuid);
var failedDocs = new List<KeyValuePair<string, string>>();
if (MatterBO.IsCrawlComplete(bulkTagRecord.MatterId.ToString(CultureInfo.InvariantCulture), bulkTagRecord.Originator, failedDocs))
{
try
{
SendLog(CreateLogs(bulkTagRecord.Documents, failedDocs, bulkTagRecord));
}
catch (Exception excep)
{
var errMsg = string.Format("Unable to create / send logs to be sent to log pipe. Exception details: {0}",
excep.ToUserString());
excep.Trace().Swallow();
}
Send(bulkTagRecord);
}
else
{
InputDataPipe.Send(new PipeMessageEnvelope() { Body = bulkTagRecord, IsPostback = true });
}
}
catch (Exception ex)
{
if (bulkTagRecord != null)
{
ex.Data["Originator"] = bulkTagRecord.Originator;
}
ex.Trace().Swallow();
ReportToDirector(ex);
}
}
protected override void ProcessMessage(PipeMessageEnvelope message)
{
try
{
var analyticProject = new AnalyticsProject();
var projectDocumentCollection = (ProjectDocumentCollection) message.Body;
projectDocumentCollection.ShouldNotBe(null);
projectDocumentCollection.Documents.ShouldNotBe(null);
var documentIds = projectDocumentCollection.Documents.Select(p => p.DocId).ToList();
var documentResult = analyticProject.BulkGetDocumentsByDocIds(_jobParameter.MatterId,
_dataset.CollectionId, documentIds);
var projectDocuments = new List<AnalysisSetDocumentInfo>();
foreach (var document in projectDocumentCollection.Documents)
{
var projectDocument = new AnalysisSetDocumentInfo
{
DocumentId = (int) document.DocId,
PredictedCategory = document.PredictedCategory,
DocumentScore = (decimal) document.DocumentScore
};
var result = documentResult.FirstOrDefault(d => d.Id == document.DocId);
if (result == null) continue;
projectDocument.DocumentReferenceId = result.DocumentID;
projectDocuments.Add(projectDocument);
}
analyticProject.UpdateFieldsForCategorizeDocuments(_jobParameter.MatterId, _projectInfo,
_dataset.CollectionId, WorkAssignment.JobId, projectDocuments);
IncreaseProcessedDocumentsCount(projectDocumentCollection.Documents.Count()); //Progress Status
}
catch (Exception ex)
{
ex.Trace().Swallow();
ReportToDirector(ex);
}
}
/// <summary>
/// Processes the message.
/// </summary>
/// <param name="envelope">The envelope.</param>
protected override void ProcessMessage(PipeMessageEnvelope envelope)
{
base.ProcessMessage(envelope);
try
{
EDocsDocumentCollection eDocsDocumentCollection = envelope.Body as EDocsDocumentCollection;
if (eDocsDocumentCollection != null
&& eDocsDocumentCollection.OutlookMailStoreDataEntity != null
&& eDocsDocumentCollection.OutlookMailStoreDataEntity.Any())
{
foreach (OutlookMailStoreEntity outlookMailStoreEntity in eDocsDocumentCollection.OutlookMailStoreDataEntity)
{
if (outlookMailStoreEntity.EntryIdAndEmailMessagePairs != null && outlookMailStoreEntity.EntryIdAndEmailMessagePairs.Any())
{
string pstFilePath = outlookMailStoreEntity.PSTFile.FullName;
OutlookMessageExtractor outlookMessageExtractor = new OutlookMessageExtractor();
// extract .msg files
outlookMailStoreEntity.EntryIdAndEmailMessagePairs
.ToList().SafeForEach(keyValues => GenerateMsgFile(outlookMessageExtractor, pstFilePath, keyValues));
// framework call for increasing count of documents processed.
IncreaseProcessedDocumentsCount(outlookMailStoreEntity.EntryIdAndEmailMessagePairs.Count());
}
}
}
else
{
throw new EVException().AddResMsg(ErrorCodes.EDocsCollectionEntityEmpty);
}
}
catch (Exception ex)
{
ReportToDirector(ex);
ex.Trace().Swallow();
}
}
/// <summary>
/// overrides base ProcessMessage method
/// </summary>
/// <param name="envelope"></param>
protected override void ProcessMessage(PipeMessageEnvelope envelope)
{
//get the review set record and add the count of batches processed
var reviewsetRecord = (ReviewsetRecord)envelope.Body;
reviewsetRecord.ShouldNotBe(null);
try
{
if (!reviewsets.ContainsKey(reviewsetRecord.ReviewSetId))
{
reviewsets.Add(reviewsetRecord.ReviewSetId,
new ProcessedCount
{
TotalNumberofBatches = reviewsetRecord.NumberOfBatches,
NumberofBatchesProcessed = 1
});
}
else
{
reviewsets[reviewsetRecord.ReviewSetId].NumberofBatchesProcessed += 1;
}
if (reviewsets.Count <= 0) return;
foreach (
var reviewset in
reviewsets.Where(
reviewset =>
(reviewset.Value.NumberofBatchesProcessed.Equals(reviewset.Value.TotalNumberofBatches)) &&
reviewsetRecord.ReviewSetId.Equals(reviewset.Key)))
{
UpdateReviewSetStatus(reviewsetRecord);
}
}
catch (Exception ex)
{
ReportToDirector(ex);
ex.Trace().Swallow();
LogMessage(false, "Reviewset status was not updated.", reviewsetRecord.CreatedBy,
reviewsetRecord.ReviewSetId, reviewsetRecord.ReviewSetName);
}
}
protected override void ProcessMessage(PipeMessageEnvelope message)
{
try
{
var projectDocumentCollection = (ProjectDocumentCollection) message.Body;
projectDocumentCollection.ShouldNotBe(null);
projectDocumentCollection.Documents.ShouldNotBe(null);
_analyticProject = new AnalyticsProject();
if (_jobParameter.IsRerunJob) //---Rerun
{
ProcessDocumentsForRerunJob(projectDocumentCollection);
}
else
{
var documents =
projectDocumentCollection.Documents.Select(
projectDocument => new Business.Analytics.DocumentIdentifier
{
ReferenceId = projectDocument.DocumentReferenceId,
Url = projectDocument.TextFilePath,
DocumentIndexingStatus = projectDocument.DocumentIndexStatus,
DocumentTextSize = projectDocument.DocumentTextSize
}).ToList();
_analyticProject.AddDocumentsIntoProject(_jobParameter.MatterId, _dataset, documents, _jobParameter,
projectDocumentCollection.ProjectFieldId, WorkAssignment.JobId);
}
IncreaseProcessedDocumentsCount(projectDocumentCollection.Documents.Count()); //Progress Status
}
catch (Exception ex)
{
ex.Trace().Swallow();
ReportToDirector(ex);
}
}
/// <summary>
/// Initiates the threads and families calculation.
/// </summary>
private void InitiateThreadsAndFamiliesCalculation()
{
if (ThreadsLinkingRequested || IsOverlay)
{
// Sending special message to ThreadsLinker
var threadsSection = FindPipelineSection("ThreadsLinker");
var threadsSectionDataPipeName = threadsSection.DataPipeName;
using (var dataPipe = new Pipe(threadsSectionDataPipeName))
{
dataPipe.Open();
var envelope = new PipeMessageEnvelope {
Label = "EndOfDataMarker"
};
dataPipe.Send(envelope);
}
Tracer.Info("Threads finalization initiated.");
}
if (FamiliesLinkingRequested && !ThreadsLinkingRequested && !IsOverlay)
{
// Sending special message to FamiliesLinker
var familiesSection = FindPipelineSection("FamiliesLinker");
var familiesSectionDataPipeName = familiesSection.DataPipeName;
using (var dataPipe = new Pipe(familiesSectionDataPipeName))
{
dataPipe.Open();
var envelope = new PipeMessageEnvelope {
Label = "EndOfDataMarker"
};
dataPipe.Send(envelope);
}
Tracer.Info("Families calculation initiated.");
}
// If both threads and families linking are requested, then we expect threads linker to send EndOfDataMarker to families linker
FamiliesAndThreadsCalculationsInitiated = true;
}
// This method never throws - possible exception already handled inside.
private bool ProcessMessageSafe()
{
bool quitLifeCycle = false;
try
{
#if RECEIVE_WAITS
PipeMessageEnvelope message = InputDataPipe.Receive(InputDataPipeReceiveTimeout);
#else
PipeMessageEnvelope message = InputDataPipe.Receive(_zeroWait);
#endif
if (null != message)
{
State = WorkerState.Busy;
SlowDownPostbacks(message);
ProcessMessageNoTrans(message);
}
else
{
State = WorkerState.Idle;
#if RECEIVE_WAITS
WorkerStatistics.RecordIdleTime(InputDataPipeReceiveTimeout);
#else
Thread.Sleep(InputDataPipeReceiveTimeout);
#endif
}
WorkerStatistics.Send();
}
catch (Exception ex)
{
ex.Trace().Swallow();
ProblemReport problemReport = new ProblemReport(WorkerId, WorkerStage.WorkerStep, ex);
problemReport.SendProblemReport(ReportPipe);
quitLifeCycle = true; // Worker which caused unhandled exception should quit rather than try to process more messages
}
return(quitLifeCycle);
}
private void ProcessReport(PipeMessageEnvelope envelope)
{
if (_reportPipeMessageCounter++ == 0)
{
_timeSinceTheFirstReportReceived.Start();
}
object message = envelope.Body;
WorkerStateChangedMessage workerStateChangedMessage = message as WorkerStateChangedMessage;
if (workerStateChangedMessage != null)
{
ProcessWorkerStateChangeReports(workerStateChangedMessage);
}
WorkerStatistics workerStatistics = message as WorkerStatistics;
if (workerStatistics != null)
{
ProcessWorkerStatistics(workerStatistics);
}
WorkerMessage workerMessage = message as WorkerMessage;
if (workerMessage != null)
{
ProcessWorkerMessage(workerMessage);
}
ProblemReport problemReport = message as ProblemReport;
if (problemReport != null)
{
ProcessProblemReport(problemReport);
}
}
protected override void ProcessMessage(PipeMessageEnvelope message)
{
var documentresultCollection = (NearDuplicationResultInfoCollection) message.Body;
_documentsFields = new List<DocumentFieldsBEO>();
_documentsFieldsForSearchEngineUpdate = new Dictionary<string, List<KeyValuePair<string, string>>>();
foreach (var resultDocument in documentresultCollection.ResultDocuments)
{
ConstructDocumentFieldsForVault(resultDocument);
ConstructDocumentFieldsForSearchEngineUpdate(resultDocument.DocumentId,
_documentsFields.Where(f => f.DocumentReferenceId == resultDocument.DocumentId).ToList());
}
//Add documents fields into Vault
var databaseUpdateStaus = UpdateNearDuplicationFieldsInDatabase();
//Add documents fields into search engine
var searchEngineUpdateStaus = UpdateNearDuplicationFieldsInSearchEngine();
if (!databaseUpdateStaus || !searchEngineUpdateStaus)
{
ConstructAndSendLog(documentresultCollection.ResultDocuments, databaseUpdateStaus,
true);
}
IncreaseProcessedDocumentsCount(documentresultCollection.ResultDocuments.Count());
}
// This is separate thread!
public void Run()
{
try
{
PipelineStatus.PipelineState = PipelineState.Running;
_runTime.Start();
_timeSinceTheLastProgressReport.Start();
bool continueMessagePumping;
do
{
PipeMessageEnvelope envelope = ReportPipe.Receive(_reportPipeTimeout);
lock (this)
{
UpdatePipelineState(envelope, out continueMessagePumping);
}
} while (continueMessagePumping);
ForceReportPipelineStatus("FINAL PIPELINE STATUS: ", true);
}
catch (Exception ex)
{
ex.Trace();
}
}
protected virtual void ProcessMessage(PipeMessageEnvelope envelope)
{
}
public void RecordInputTraffic(PipeMessageEnvelope envelope)
{
_inputTraffic += envelope.BodyLength;
_compressedInputTraffic += envelope.CompressedBodyLength;
}
