private void SendRelationshipsInfo(IEnumerable <FamilyInfo> familyInfoRecords)
{
FamiliesInfo familiesInfo = new FamiliesInfo();
foreach (FamilyInfo familyInfoRecord in familyInfoRecords)
{
// We don't skip standalone documents for Families, because they always can appear to be topmost parents
//FamilyInfo familyInfoRecord = new FamilyInfo(docReferenceId);
//familyInfoRecord.OriginalDocumentId = doc.document.EVLoadFileDocumentId;
//familyInfoRecord.OriginalParentId = String.IsNullOrEmpty(doc.document.EVLoadFileParentId) ? null : doc.document.EVLoadFileParentId;
//Tracer.Warning("SendRelationshipsInfo: OriginalDocumentId = {0}, OriginalParentId = {1}",
// familyInfoRecord.OriginalDocumentId, familyInfoRecord.OriginalParentId);
if (String.Equals(familyInfoRecord.OriginalDocumentId, familyInfoRecord.OriginalParentId, StringComparison.InvariantCulture))
{
//Tracer.Warning("SendRelationshipsInfo: OriginalDocumentId = {0}, OriginalParentId reset to null", familyInfoRecord.OriginalDocumentId);
familyInfoRecord.OriginalParentId = null; // Document must not be its own parent
}
// Family has priority over thread, so if the document is part of the family we ignore its thread
//if (familyInfoRecord.OriginalParentId != null)
//{
// //Tracer.Warning("SendRelationshipsInfo: OriginalDocumentId = {0}, ConversationIndex reset to null", familyInfoRecord.OriginalDocumentId);
// doc.ConversationIndex = null;
//}
familiesInfo.FamilyInfoList.Add(familyInfoRecord);
}
if (familiesInfo.FamilyInfoList.Any())
{
SendFamilies(familiesInfo);
}
}
private void SendFamilies(FamiliesInfo familiesInfo)
{
Pipe familiesAndThreadsPipe = GetOutputDataPipe("FamiliesLinker");
familiesAndThreadsPipe.ShouldNotBe(null);
var message = new PipeMessageEnvelope()
{
Body = familiesInfo
};
familiesAndThreadsPipe.Send(message);
}
private void SendFamilies(FamiliesInfo familiesInfo)
{
if (familiesInfo == null || familiesInfo.FamilyInfoList == null || !familiesInfo.FamilyInfoList.Any())
{
return;
}
Pipe familiesAndThreadsPipe = GetOutputDataPipe("FamiliesLinker");
familiesAndThreadsPipe.ShouldNotBe(null);
var message = new PipeMessageEnvelope()
{
Body = familiesInfo
};
familiesAndThreadsPipe.Send(message);
}
private void SendFamilies(IEnumerable <DocumentDetail> documentDetailList)
{
if (!_jobParams.CreateFamilyGroups)
{
return;
}
FamiliesInfo familiesInfo = new FamiliesInfo();
foreach (DocumentDetail doc in documentDetailList)
{
if (doc.docType != DocumentsetType.NativeSet)
{
continue; // Only original documents may participate in relationships
}
string docReferenceId = doc.document.DocumentId;
if (String.IsNullOrEmpty(docReferenceId))
{
continue;
}
if (!String.IsNullOrEmpty(doc.document.EVLoadFileDocumentId))
{
// We don't skip standalone documents for Families, because they always can appear to be topmost parents
FamilyInfo familyInfoRecord = new FamilyInfo(docReferenceId);
familyInfoRecord.OriginalDocumentId = doc.document.EVLoadFileDocumentId;
familyInfoRecord.OriginalParentId = (String.IsNullOrEmpty(doc.document.EVLoadFileParentId) || doc.document.EVLoadFileParentId == "0") ? null : doc.document.EVLoadFileParentId;
// Debug
//Tracer.Warning("LawProcessingWorker.SendFamilies: DocId = {0}, OriginalDocumentId = {1}, OriginalParentId = {2}",
// docReferenceId, familyInfoRecord.OriginalDocumentId, familyInfoRecord.OriginalParentId);
if (String.Equals(familyInfoRecord.OriginalDocumentId, familyInfoRecord.OriginalParentId, StringComparison.InvariantCulture))
{
//Tracer.Warning("SendRelationshipsInfo: OriginalDocumentId = {0}, OriginalParentId reset to null", familyInfoRecord.OriginalDocumentId);
familyInfoRecord.OriginalParentId = null; // Document must not be its own parent
}
familiesInfo.FamilyInfoList.Add(familyInfoRecord);
}
}
SendFamilies(familiesInfo);
}
private void SendRelationshipsInfo(IEnumerable <DocumentDetail> documentDetailList)
{
bool familiesLinkingRequested = _jobParameter.IsImportFamilyRelations;
bool threadsLinkingRequested = _jobParameter.IsMapEmailThread;
FamiliesInfo familiesInfo = familiesLinkingRequested ? new FamiliesInfo() : null;
ThreadsInfo threadsInfo = threadsLinkingRequested ? new ThreadsInfo() : null;
foreach (DocumentDetail doc in documentDetailList)
{
if (doc.docType != DocumentsetType.NativeSet)
{
continue; // Only original documents may participate in relationships
}
string docReferenceId = doc.document.DocumentId;
if (String.IsNullOrEmpty(docReferenceId))
{
continue;
}
// Debug
//Tracer.Warning("DOCID {0} corresponds to the document {1}", doc.document.FieldList[0].FieldValue, docReferenceId.Hint(4));
//if (docReferenceId.Hint(4) == "AFE1")
//{
// Tracer.Warning("STOP!");
//}
if (familiesLinkingRequested && !String.IsNullOrEmpty(doc.document.EVLoadFileDocumentId))
{
// We don't skip standalone documents for Families, because they always can appear to be topmost parents
FamilyInfo familyInfoRecord = new FamilyInfo(docReferenceId);
familyInfoRecord.OriginalDocumentId = doc.document.EVLoadFileDocumentId;
familyInfoRecord.OriginalParentId = String.IsNullOrEmpty(doc.document.EVLoadFileParentId) ? null : doc.document.EVLoadFileParentId;
//Tracer.Warning("SendRelationshipsInfo: OriginalDocumentId = {0}, OriginalParentId = {1}",
// familyInfoRecord.OriginalDocumentId, familyInfoRecord.OriginalParentId);
if (String.Equals(familyInfoRecord.OriginalDocumentId, familyInfoRecord.OriginalParentId, StringComparison.InvariantCulture))
{
//Tracer.Warning("SendRelationshipsInfo: OriginalDocumentId = {0}, OriginalParentId reset to null", familyInfoRecord.OriginalDocumentId);
familyInfoRecord.OriginalParentId = null; // Document must not be its own parent
}
// Family has priority over thread, so if the document is part of the family we ignore its thread
//if (familyInfoRecord.OriginalParentId != null)
//{
// //Tracer.Warning("SendRelationshipsInfo: OriginalDocumentId = {0}, ConversationIndex reset to null", familyInfoRecord.OriginalDocumentId);
// doc.ConversationIndex = null;
//}
familiesInfo.FamilyInfoList.Add(familyInfoRecord);
}
// BEWARE: doc.document.ConversationIndex is not the right thing!!
if (threadsLinkingRequested)
{
// Sanitize the value
doc.ConversationIndex = String.IsNullOrEmpty(doc.ConversationIndex) ? null : doc.ConversationIndex;
// Debug
//Tracer.Warning("SendRelationshipsInfo: CollectionId = {0}", doc.document.CollectionId);
var threadInfo = new ThreadInfo(docReferenceId, doc.ConversationIndex);
threadsInfo.ThreadInfoList.Add(threadInfo);
}
}
if (threadsLinkingRequested && threadsInfo.ThreadInfoList.Any())
{
SendThreads(threadsInfo);
}
if (familiesLinkingRequested && familiesInfo.FamilyInfoList.Any())
{
SendFamilies(familiesInfo);
}
}
private void SendRelationshipsInfo(IEnumerable <DocumentDetail> documentDetailList)
{
bool familiesLinkingRequested = _jobParameter.IsImportFamilyRelations;
bool threadsLinkingRequested = _jobParameter.IsMapEmailThread;
FamiliesInfo familiesInfo = familiesLinkingRequested ? new FamiliesInfo() : null;
ThreadsInfo threadsInfo = threadsLinkingRequested ? new ThreadsInfo() : null;
foreach (DocumentDetail doc in documentDetailList)
{
if (doc.docType != DocumentsetType.NativeSet)
{
continue; // Only original documents may participate in relationships
}
string docReferenceId = doc.document.DocumentId;
if (String.IsNullOrEmpty(docReferenceId))
{
continue;
}
if (familiesLinkingRequested && !String.IsNullOrEmpty(doc.document.EVLoadFileDocumentId))
{
// We don't skip standalone documents for Families, because they always can appear to be topmost parents.
// And also we need all of them to provide Original to Real Id translation.
FamilyInfo familyInfoRecord = new FamilyInfo(docReferenceId);
familyInfoRecord.OriginalDocumentId = doc.document.EVLoadFileDocumentId;
familyInfoRecord.OriginalParentId = String.IsNullOrEmpty(doc.document.EVLoadFileParentId) ? null : doc.document.EVLoadFileParentId;
//Tracer.Warning("SendRelationshipsInfo: OriginalDocumentId = {0}, OriginalParentId = {1}",
// familyInfoRecord.OriginalDocumentId, familyInfoRecord.OriginalParentId);
if (String.Equals(familyInfoRecord.OriginalDocumentId, familyInfoRecord.OriginalParentId, StringComparison.InvariantCulture))
{
//Tracer.Warning("SendRelationshipsInfo: OriginalDocumentId = {0}, OriginalParentId reset to null", familyInfoRecord.OriginalDocumentId);
familyInfoRecord.OriginalParentId = null; // Document must not be its own parent
}
familiesInfo.FamilyInfoList.Add(familyInfoRecord);
}
// BEWARE: doc.document.ConversationIndex is not the right thing!!
if (threadsLinkingRequested)
{
// Sanitize the value
doc.ConversationIndex = String.IsNullOrEmpty(doc.ConversationIndex) ? null : doc.ConversationIndex;
// On Append we only calculate relationships between new documents,
// therefore we don't even send standalone documents to threads linker
if (doc.ConversationIndex == null)
{
continue;
}
var threadInfo = new ThreadInfo(docReferenceId, doc.ConversationIndex);
threadsInfo.ThreadInfoList.Add(threadInfo);
}
}
if (threadsLinkingRequested && threadsInfo.ThreadInfoList.Any())
{
SendThreads(threadsInfo);
}
if (familiesLinkingRequested && familiesInfo.FamilyInfoList.Any())
{
SendFamilies(familiesInfo);
}
}
private void ImportDocumentRelationship(string documentId, Document currentDcbDocument, FamiliesInfo familiesInfo)
{
// Debug
//Tracer.Warning("DCB documentId = {0}", documentId);
if (DcbOpticonJobBEO.FamilyRelations.IsEmailDCB)
{
if (null != currentDcbDocument.children && currentDcbDocument.children.Any())
{
foreach (int physicalChildDocumentNumber in currentDcbDocument.children)
{
FamilyInfo familyInfoRecord1 = new FamilyInfo(null);
familyInfoRecord1.OriginalDocumentId = GenerateDocumentIdForEV(physicalChildDocumentNumber.ToString(CultureInfo.InvariantCulture));
familyInfoRecord1.OriginalParentId = GenerateDocumentIdForEV(currentDcbDocument);
familiesInfo.FamilyInfoList.Add(familyInfoRecord1);
}
}
// We don't skip standalone documents for Families, because they always can appear to be topmost parents.
// And also we need all of them to provide Original to Real Id translation.
FamilyInfo familyInfoRecord2 = new FamilyInfo(documentId);
familyInfoRecord2.OriginalDocumentId = GenerateDocumentIdForEV(currentDcbDocument.PhysicalNumber.ToString(CultureInfo.InvariantCulture));
familyInfoRecord2.OriginalParentId = null;
familiesInfo.FamilyInfoList.Add(familyInfoRecord2);
return;
}
/*One way of maintaining parent-child relationship in DCB is through PARENT_DOCID & DOCID fields
Most of the case it is assumed that PARENT_DOCID & DOCID will have UUID, but in CNClassic there is a
way to map DOCID and PARENT_DOCID field during import process
For Document realtion the first priority is given to PAREN_DOCID & DOCID field */
Field dcbDocIdField = currentDcbDocument.FieldItems.Find(o => o.Code.Equals(DcbOpticonJobBEO.FamilyRelations.DocId));
string dcbDocId = null;
if (null != dcbDocIdField)
{
dcbDocId = ExtractDcbDocumentId(dcbDocIdField.Value, DcbOpticonJobBEO.FamilyRelations.IsDocIDFormatRange);
}
if (dcbDocId == null)
{
Tracer.Warning("Potentially bad DCB source data: document {0} has no DocId field", documentId);
return; // No point in sending family record with the original document Id missing
}
Field dcbParentDocIdField = currentDcbDocument.FieldItems.Find(o => o.Code.Equals(DcbOpticonJobBEO.FamilyRelations.ParentDocId));
string dcbParentDocId = null;
if (null != dcbParentDocIdField)
{
dcbParentDocId = ExtractDcbDocumentId(dcbParentDocIdField.Value, DcbOpticonJobBEO.FamilyRelations.IsParentDocIDFormatRange);
}
FamilyInfo familyInfo = new FamilyInfo(documentId);
familyInfo.OriginalDocumentId = GenerateDocumentIdForEV(dcbDocId);
familyInfo.OriginalParentId = GenerateDocumentIdForEV(dcbParentDocId);
familiesInfo.FamilyInfoList.Add(familyInfo);
//Tracer.Trace("Adding Relationship record. Parent = {0}, Child = {1}", threadingEntity.ParentDocumentID, threadingEntity.ChildDocumentID);
}
protected void FetchDocumentFromDCB(int documentNumber,
List<DocumentDetail> documentDetailList, FamiliesInfo familiesInfo, JobWorkerLog<DcbParserLogInfo> dcbParserLogEntry)
{
#region Precondition asserts
documentDetailList.ShouldNotBe(null);
dcbParserLogEntry.ShouldNotBe(null);
#endregion
RVWDocumentBEO evDocument = new RVWDocumentBEO();
try
{
//Get the document from DcbFacade
Document currentDcbDocument = DcbFacade.GetDocument(documentNumber);
//Throw exception if GetDocument fails
currentDcbDocument.ShouldNotBe(null);
//Create the target EV document
evDocument.DocumentId = Guid.NewGuid().ToString().Replace("-", "").ToUpper();
dcbParserLogEntry.LogInfo.DocumentId = evDocument.DocumentId;
evDocument.CollectionId = DcbOpticonJobBEO.TargetDatasetId;
evDocument.MatterId = DcbOpticonJobBEO.MatterId;
//Add the fields required for casemap
RVWDocumentFieldBEO evDocumentAccessionNumField = new RVWDocumentFieldBEO
{
FieldId = Convert.ToInt32(DcbOpticonJobBEO.SysDocId),
FieldName = EVSystemFields.DcbId,
IsSystemField = true,
IsRequired = true,
FieldValue = Convert.ToString(currentDcbDocument.UUID, CultureInfo.InvariantCulture)
};
evDocument.FieldList.Add(evDocumentAccessionNumField);
evDocument.FieldList.Add(_evDocumentSysImportTypeField);
//Set the fields from field mapping except content field
foreach (FieldMapBEO fieldMap in DcbOpticonJobBEO.FieldMappings)
{
Field dcbField = currentDcbDocument.FieldItems.Find(o => (o.Code == fieldMap.SourceFieldID));
//Profile fieldmapping has duplicates
RVWDocumentFieldBEO evDocumentFieldBEO = evDocument.FieldList.Find(o => o.FieldId.Equals(fieldMap.DatasetFieldID));
if ((null != dcbField) && (evDocumentFieldBEO == null) && (fieldMap.DatasetFieldID != _contentFieldId))
{
RVWDocumentFieldBEO evDocuemtnField = new RVWDocumentFieldBEO
{
FieldId = fieldMap.DatasetFieldID,
FieldName = fieldMap.DatasetFieldName
};
FieldBEO evfieldDef = _dataset.DatasetFieldList.Find(o => o.ID == evDocuemtnField.FieldId);
evDocuemtnField.FieldValue = evfieldDef.FieldType.DataTypeId == Constants.DateDataType
? GetDateFiedlValue(dcbField, dcbParserLogEntry)
: Regex.Replace(dcbField.Value, "\r\n", "\n");
evDocument.FieldList.Add(evDocuemtnField);
}
}
//Separate logic for content
StringBuilder sbContent = new StringBuilder();
if (DcbOpticonJobBEO.ContentFields != null)
{
foreach (string contentfield in DcbOpticonJobBEO.ContentFields.Field)
{
Field dcbContentField = currentDcbDocument.FieldItems.Find(o => (o.Name.Equals(contentfield)));
if (null != dcbContentField)
{
sbContent.Append(dcbContentField.Value);
}
}
}
string text = sbContent.ToString().Replace("\r\n", "\n");
//evDocument.DocumentBinary.Content = Regex.Replace(sbContent.ToString(), "\r\n", "\n");
if (!DumpTextToFile(evDocument, text, dcbParserLogEntry))
{
return;
}
//Set the native file path if selected
evDocument.NativeFilePath = GetNativeFilePath(currentDcbDocument);
if (!String.IsNullOrEmpty(evDocument.NativeFilePath) && File.Exists(evDocument.NativeFilePath))
{
FileInfo fileInfo = new FileInfo(evDocument.NativeFilePath);
//Tracer.Trace("DcbParcer located native document {0} for DocumentId = {1} and the file length is {2}",
// evDocument.NativeFilePath, evDocument.DocumentId, fileInfo.Length);
if (fileInfo.Length > 0)
{
evDocument.FileSize = (int)Math.Ceiling(fileInfo.Length / 1024.0);
}
else
{
evDocument.FileSize = 0;
}
evDocument.MD5HashValue = DocumentHashHelper.GetMD5HashValue(evDocument.NativeFilePath);
evDocument.SHAHashValue = DocumentHashHelper.GetSHAHashValue(evDocument.NativeFilePath);
}
//Set the MIME type
string extn = string.Empty;
string newExtn = string.Empty;
extn = Path.GetExtension(evDocument.NativeFilePath);
if (!String.IsNullOrEmpty(extn))
newExtn = extn.Remove(0, 1);
evDocument.MimeType = GetMimeType(newExtn);
evDocument.FileExtension = extn;
string createdByGuid = String.Empty;
if (null != ProfileBEO && null != ProfileBEO.CreatedBy)
{
createdByGuid = ProfileBEO.CreatedBy;
}
evDocument.CreatedBy = createdByGuid;
evDocument.ModifiedBy = createdByGuid;
if (File.Exists(evDocument.NativeFilePath))
{
//Calculating size of file in KB
FileInfo fileInfo = new FileInfo(evDocument.NativeFilePath);
evDocument.FileSize = (int)Math.Ceiling(fileInfo.Length / Constants.KBConversionConstant);
if (evDocument.DocumentBinary == null) { evDocument.DocumentBinary = new RVWDocumentBinaryBEO(); }
RVWExternalFileBEO nativeFile = new RVWExternalFileBEO
{
Type = NATIVE_FILE_TYPE,
Path = evDocument.NativeFilePath
};
evDocument.DocumentBinary.FileList.Add(nativeFile);
}
DocumentDetail documentDetail = new DocumentDetail
{
// CorrId is the same as TaskId and it is 1 based.
CorrelationId = checked(documentNumber + 1).ToString(CultureInfo.InvariantCulture),
IsNewDocument = true,
docType = DocumentsetType.NativeSet,
document = evDocument
};
documentDetailList.Add(documentDetail);
//Add Tags
if (DcbOpticonJobBEO.IncludeTags && null != currentDcbDocument.TagItems && currentDcbDocument.TagItems.Count > 0)
{
if (null == documentDetail.DcbTags)
{
documentDetail.DcbTags = new List<DcbTags>();
}
DcbDocumentTags dcbDocumentTags = new DcbDocumentTags
{
compositeTagNames = currentDcbDocument.TagItems,
DatasetId = DcbOpticonJobBEO.TargetDatasetId,
MatterId = DcbOpticonJobBEO.MatterId,
DocumentId = evDocument.DocumentId
};
documentDetail.DcbTags.Add(dcbDocumentTags);
}
// Add notes
AddComments(documentDetail, evDocument, currentDcbDocument);
//Add Images
if (DcbOpticonJobBEO.ImportImages)
{
RVWDocumentBEO images = ImportDocumentImages(evDocument.DocumentId, currentDcbDocument);
if (null != images)
{
DocumentDetail imageDocumentDetail = new DocumentDetail
{
// CorrId is the same as TaskId and it is 1 based.
CorrelationId = checked(documentNumber + 1).ToString(CultureInfo.InvariantCulture),
IsNewDocument = true,
docType = DocumentsetType.ImageSet,
document = images
};
documentDetailList.Add(imageDocumentDetail);
dcbParserLogEntry.LogInfo.AddedImages = images.DocumentBinary.FileList.Count;
}
//Add Redlines
//ImportDocumentRedlines();
}
//Add Document Relation
if (DcbOpticonJobBEO.IsImportFamilies)
{
ImportDocumentRelationship(evDocument.DocumentId, currentDcbDocument, familiesInfo);
}
#region Postcondition asserts
documentDetailList.ShouldNotBe(null);
#endregion
}
catch (Exception ex)
{
//TaskLogInfo.AddParameters(Constants.ErrorDoAtomicWork + "<br/>" + ex.Message);
//TaskLogInfo.StackTrace = ex.Source + "<br/>" + ex.Message + "<br/>" + ex.StackTrace;
//TaskLogInfo.IsError = true;
ex.Trace().Swallow();
dcbParserLogEntry.Success = false;
if (ex.ToUserString().Contains(Constants.DiskFullErrorMessage))
{
dcbParserLogEntry.LogInfo.Message = "There is not enough space on the disk";
throw;
}
else
{
dcbParserLogEntry.LogInfo.Message = ex.ToUserString();
}
}
}
private void SendFamilies(FamiliesInfo familiesInfo)
{
Pipe familiesAndThreadsPipe = GetOutputDataPipe("FamiliesLinker");
familiesAndThreadsPipe.ShouldNotBe(null);
var message = new PipeMessageEnvelope()
{
Body = familiesInfo
};
familiesAndThreadsPipe.Send(message);
}
private void SendFamilies(FamiliesInfo familiesInfo)
{
if (familiesInfo == null || familiesInfo.FamilyInfoList == null || !familiesInfo.FamilyInfoList.Any())
{
return;
}
Pipe familiesAndThreadsPipe = GetOutputDataPipe("FamiliesLinker");
familiesAndThreadsPipe.ShouldNotBe(null);
var message = new PipeMessageEnvelope()
{
Body = familiesInfo
};
familiesAndThreadsPipe.Send(message);
}
private void SendRelationshipsInfo(IEnumerable<EmailThreadingEntity> rawDocumentRelationships)
{
// For eDocs we ALWAYS send relationships info
//if (!m_Parameters.IsImportFamilyRelations)
//{
// return;
//}
FamiliesInfo familiesInfo = new FamiliesInfo();
ThreadsInfo threadsInfo = new ThreadsInfo();
foreach (EmailThreadingEntity emailThreadingEntity in rawDocumentRelationships)
{
string docReferenceId = emailThreadingEntity.ChildDocumentID;
if (String.IsNullOrEmpty(docReferenceId))
{
continue;
}
if (emailThreadingEntity.RelationshipType == ThreadRelationshipEntity.RelationshipType.OutlookEmailThread)
{
// Sanitize the value
emailThreadingEntity.ConversationIndex = String.IsNullOrEmpty(emailThreadingEntity.ConversationIndex) ? null : emailThreadingEntity.ConversationIndex;
// On Append we only calculate relationships between new documents,
// therefore we don't even send standalone documents to threads linker
if (emailThreadingEntity.ConversationIndex == null)
{
continue;
}
var threadInfo = new ThreadInfo(docReferenceId, emailThreadingEntity.ConversationIndex);
threadsInfo.ThreadInfoList.Add(threadInfo);
}
else
{
// We don't skip standalone documents for Families, because they always can appear to be topmost parents
FamilyInfo familyInfoRecord = new FamilyInfo(docReferenceId);
familyInfoRecord.OriginalDocumentId = docReferenceId;
familyInfoRecord.OriginalParentId = String.IsNullOrEmpty(emailThreadingEntity.ParentDocumentID) ? null : emailThreadingEntity.ParentDocumentID;
//Tracer.Warning("SendRelationshipsInfo: OriginalDocumentId = {0}, OriginalParentId = {1}",
// familyInfoRecord.OriginalDocumentId, familyInfoRecord.OriginalParentId);
if (String.Equals(familyInfoRecord.OriginalDocumentId, familyInfoRecord.OriginalParentId, StringComparison.InvariantCulture))
{
//Tracer.Warning("SendRelationshipsInfo: OriginalDocumentId = {0}, OriginalParentId reset to null", familyInfoRecord.OriginalDocumentId);
familyInfoRecord.OriginalParentId = null; // Document must not be its own parent
}
familiesInfo.FamilyInfoList.Add(familyInfoRecord);
}
const int BatchSize = 500;
if (threadsInfo.ThreadInfoList.Count >= BatchSize)
{
SendThreads(threadsInfo);
threadsInfo.ThreadInfoList.Clear();
}
if (familiesInfo.FamilyInfoList.Count >= BatchSize)
{
SendFamilies(familiesInfo);
familiesInfo.FamilyInfoList.Clear();
}
}
if (threadsInfo.ThreadInfoList.Any())
{
SendThreads(threadsInfo);
}
if (familiesInfo.FamilyInfoList.Any())
{
SendFamilies(familiesInfo);
}
}
protected override void ProcessMessage(PipeMessageEnvelope envelope)
{
if (null == envelope || null == envelope.Body)
{
return;
}
try
{
DcbSlice dcbSlice = (DcbSlice)envelope.Body;
OpenDCB(dcbSlice.DcbCredentials);
ImageSetId = dcbSlice.ImageSetId;
FamiliesInfo familiesInfo = DcbOpticonJobBEO.IsImportFamilies ? new FamiliesInfo() : null;
var documentDetailList = new List <DocumentDetail>();
var dcbParserLogEntries = new List <JobWorkerLog <DcbParserLogInfo> >();
int lastDocumentInTheBatch = dcbSlice.FirstDocument + dcbSlice.NumberOfDocuments - 1;
for (int currentDocumentNumber = dcbSlice.FirstDocument;
currentDocumentNumber <= lastDocumentInTheBatch;
currentDocumentNumber++)
{
JobWorkerLog <DcbParserLogInfo> dcbParserLogEntry =
new JobWorkerLog <DcbParserLogInfo>()
{
JobRunId = long.Parse(PipelineId),
WorkerInstanceId = WorkerId,
CorrelationId = currentDocumentNumber + 1, // CorrId is the same as TaskId and it is 1 based.
// This magic GUID is set in [EVMaster].[dbo].[EV_JOB_WorkerRoleType] to identify DcbParcer for Log Worker
WorkerRoleType = "e754adb7-23c8-44cc-8d4c-12f33aef41b6",
Success = true,
CreatedBy = ProfileBEO.CreatedBy,
IsMessage = false,
LogInfo = new DcbParserLogInfo()
};
FetchDocumentFromDCB(currentDocumentNumber, documentDetailList, familiesInfo, dcbParserLogEntry);
dcbParserLogEntries.Add(dcbParserLogEntry);
}
if (0 == dcbSlice.FirstDocument)
{
DcbDatabaseTags dcbDatabaseTags = FetchDatabaseLevelTags();
if (null != dcbDatabaseTags)
{
if (null == documentDetailList[0].DcbTags)
{
documentDetailList[0].DcbTags = new List <DcbTags>();
}
documentDetailList[0].DcbTags.Add(dcbDatabaseTags);
}
}
if (DcbOpticonJobBEO.IsImportFamilies)
{
SendRelationshipsInfo(familiesInfo.FamilyInfoList);
}
Send(documentDetailList);
SendLog(dcbParserLogEntries);
}
catch (ArgumentOutOfRangeException exRange)
{
exRange.AddUsrMsg(Constants.ExportPathFull);
ReportToDirector(exRange);
exRange.Trace();
LogMessage(false, Constants.ExportPathFull);
throw;
}
catch (Exception ex)
{
ReportToDirector(ex);
ex.Trace().Swallow();
LogMessage(false, "DcbParser failed to process document. Error: " + ex.ToUserString());
}
}
private void SendRelationshipsInfo(IEnumerable<FamilyInfo> familyInfoRecords)
{
FamiliesInfo familiesInfo = new FamiliesInfo();
foreach (FamilyInfo familyInfoRecord in familyInfoRecords)
{
// We don't skip standalone documents for Families, because they always can appear to be topmost parents
//FamilyInfo familyInfoRecord = new FamilyInfo(docReferenceId);
//familyInfoRecord.OriginalDocumentId = doc.document.EVLoadFileDocumentId;
//familyInfoRecord.OriginalParentId = String.IsNullOrEmpty(doc.document.EVLoadFileParentId) ? null : doc.document.EVLoadFileParentId;
//Tracer.Warning("SendRelationshipsInfo: OriginalDocumentId = {0}, OriginalParentId = {1}",
// familyInfoRecord.OriginalDocumentId, familyInfoRecord.OriginalParentId);
if (String.Equals(familyInfoRecord.OriginalDocumentId, familyInfoRecord.OriginalParentId, StringComparison.InvariantCulture))
{
//Tracer.Warning("SendRelationshipsInfo: OriginalDocumentId = {0}, OriginalParentId reset to null", familyInfoRecord.OriginalDocumentId);
familyInfoRecord.OriginalParentId = null; // Document must not be its own parent
}
// Family has priority over thread, so if the document is part of the family we ignore its thread
//if (familyInfoRecord.OriginalParentId != null)
//{
// //Tracer.Warning("SendRelationshipsInfo: OriginalDocumentId = {0}, ConversationIndex reset to null", familyInfoRecord.OriginalDocumentId);
// doc.ConversationIndex = null;
//}
familiesInfo.FamilyInfoList.Add(familyInfoRecord);
}
if (familiesInfo.FamilyInfoList.Any())
{
SendFamilies(familiesInfo);
}
}
protected void FetchDocumentFromDCB(int documentNumber,
List <DocumentDetail> documentDetailList, FamiliesInfo familiesInfo, JobWorkerLog <DcbParserLogInfo> dcbParserLogEntry)
{
#region Precondition asserts
documentDetailList.ShouldNotBe(null);
dcbParserLogEntry.ShouldNotBe(null);
#endregion
RVWDocumentBEO evDocument = new RVWDocumentBEO();
try
{
//Get the document from DcbFacade
Document currentDcbDocument = DcbFacade.GetDocument(documentNumber);
//Throw exception if GetDocument fails
currentDcbDocument.ShouldNotBe(null);
//Create the target EV document
evDocument.DocumentId = Guid.NewGuid().ToString().Replace("-", "").ToUpper();
dcbParserLogEntry.LogInfo.DocumentId = evDocument.DocumentId;
evDocument.CollectionId = DcbOpticonJobBEO.TargetDatasetId;
evDocument.MatterId = DcbOpticonJobBEO.MatterId;
//Add the fields required for casemap
RVWDocumentFieldBEO evDocumentAccessionNumField = new RVWDocumentFieldBEO
{
FieldId = Convert.ToInt32(DcbOpticonJobBEO.SysDocId),
FieldName = EVSystemFields.DcbId,
IsSystemField = true,
IsRequired = true,
FieldValue = Convert.ToString(currentDcbDocument.UUID, CultureInfo.InvariantCulture)
};
evDocument.FieldList.Add(evDocumentAccessionNumField);
evDocument.FieldList.Add(_evDocumentSysImportTypeField);
//Set the fields from field mapping except content field
foreach (FieldMapBEO fieldMap in DcbOpticonJobBEO.FieldMappings)
{
Field dcbField = currentDcbDocument.FieldItems.Find(o => (o.Code == fieldMap.SourceFieldID));
//Profile fieldmapping has duplicates
RVWDocumentFieldBEO evDocumentFieldBEO = evDocument.FieldList.Find(o => o.FieldId.Equals(fieldMap.DatasetFieldID));
if ((null != dcbField) && (evDocumentFieldBEO == null) && (fieldMap.DatasetFieldID != _contentFieldId))
{
RVWDocumentFieldBEO evDocuemtnField = new RVWDocumentFieldBEO
{
FieldId = fieldMap.DatasetFieldID,
FieldName = fieldMap.DatasetFieldName
};
FieldBEO evfieldDef = _dataset.DatasetFieldList.Find(o => o.ID == evDocuemtnField.FieldId);
evDocuemtnField.FieldValue = evfieldDef.FieldType.DataTypeId == Constants.DateDataType
? GetDateFiedlValue(dcbField, dcbParserLogEntry)
: Regex.Replace(dcbField.Value, "\r\n", "\n");
evDocument.FieldList.Add(evDocuemtnField);
}
}
//Separate logic for content
StringBuilder sbContent = new StringBuilder();
if (DcbOpticonJobBEO.ContentFields != null)
{
foreach (string contentfield in DcbOpticonJobBEO.ContentFields.Field)
{
Field dcbContentField = currentDcbDocument.FieldItems.Find(o => (o.Name.Equals(contentfield)));
if (null != dcbContentField)
{
sbContent.Append(dcbContentField.Value);
}
}
}
string text = sbContent.ToString().Replace("\r\n", "\n");
//evDocument.DocumentBinary.Content = Regex.Replace(sbContent.ToString(), "\r\n", "\n");
if (!DumpTextToFile(evDocument, text, dcbParserLogEntry))
{
return;
}
//Set the native file path if selected
evDocument.NativeFilePath = GetNativeFilePath(currentDcbDocument);
if (!String.IsNullOrEmpty(evDocument.NativeFilePath) && File.Exists(evDocument.NativeFilePath))
{
FileInfo fileInfo = new FileInfo(evDocument.NativeFilePath);
//Tracer.Trace("DcbParcer located native document {0} for DocumentId = {1} and the file length is {2}",
// evDocument.NativeFilePath, evDocument.DocumentId, fileInfo.Length);
if (fileInfo.Length > 0)
{
evDocument.FileSize = (int)Math.Ceiling(fileInfo.Length / 1024.0);
}
else
{
evDocument.FileSize = 0;
}
evDocument.MD5HashValue = DocumentHashHelper.GetMD5HashValue(evDocument.NativeFilePath);
evDocument.SHAHashValue = DocumentHashHelper.GetSHAHashValue(evDocument.NativeFilePath);
}
//Set the MIME type
string extn = string.Empty;
string newExtn = string.Empty;
extn = Path.GetExtension(evDocument.NativeFilePath);
if (!String.IsNullOrEmpty(extn))
{
newExtn = extn.Remove(0, 1);
}
evDocument.MimeType = GetMimeType(newExtn);
evDocument.FileExtension = extn;
string createdByGuid = String.Empty;
if (null != ProfileBEO && null != ProfileBEO.CreatedBy)
{
createdByGuid = ProfileBEO.CreatedBy;
}
evDocument.CreatedBy = createdByGuid;
evDocument.ModifiedBy = createdByGuid;
if (File.Exists(evDocument.NativeFilePath))
{
//Calculating size of file in KB
FileInfo fileInfo = new FileInfo(evDocument.NativeFilePath);
evDocument.FileSize = (int)Math.Ceiling(fileInfo.Length / Constants.KBConversionConstant);
if (evDocument.DocumentBinary == null)
{
evDocument.DocumentBinary = new RVWDocumentBinaryBEO();
}
RVWExternalFileBEO nativeFile = new RVWExternalFileBEO
{
Type = NATIVE_FILE_TYPE,
Path = evDocument.NativeFilePath
};
evDocument.DocumentBinary.FileList.Add(nativeFile);
}
DocumentDetail documentDetail = new DocumentDetail
{
// CorrId is the same as TaskId and it is 1 based.
CorrelationId = checked (documentNumber + 1).ToString(CultureInfo.InvariantCulture),
IsNewDocument = true,
docType = DocumentsetType.NativeSet,
document = evDocument
};
documentDetailList.Add(documentDetail);
//Add Tags
if (DcbOpticonJobBEO.IncludeTags && null != currentDcbDocument.TagItems && currentDcbDocument.TagItems.Count > 0)
{
if (null == documentDetail.DcbTags)
{
documentDetail.DcbTags = new List <DcbTags>();
}
DcbDocumentTags dcbDocumentTags = new DcbDocumentTags
{
compositeTagNames = currentDcbDocument.TagItems,
DatasetId = DcbOpticonJobBEO.TargetDatasetId,
MatterId = DcbOpticonJobBEO.MatterId,
DocumentId = evDocument.DocumentId
};
documentDetail.DcbTags.Add(dcbDocumentTags);
}
// Add notes
AddComments(documentDetail, evDocument, currentDcbDocument);
//Add Images
if (DcbOpticonJobBEO.ImportImages)
{
RVWDocumentBEO images = ImportDocumentImages(evDocument.DocumentId, currentDcbDocument);
if (null != images)
{
DocumentDetail imageDocumentDetail = new DocumentDetail
{
// CorrId is the same as TaskId and it is 1 based.
CorrelationId = checked (documentNumber + 1).ToString(CultureInfo.InvariantCulture),
IsNewDocument = true,
docType = DocumentsetType.ImageSet,
document = images
};
documentDetailList.Add(imageDocumentDetail);
dcbParserLogEntry.LogInfo.AddedImages = images.DocumentBinary.FileList.Count;
}
//Add Redlines
//ImportDocumentRedlines();
}
//Add Document Relation
if (DcbOpticonJobBEO.IsImportFamilies)
{
ImportDocumentRelationship(evDocument.DocumentId, currentDcbDocument, familiesInfo);
}
#region Postcondition asserts
documentDetailList.ShouldNotBe(null);
#endregion
}
catch (Exception ex)
{
//TaskLogInfo.AddParameters(Constants.ErrorDoAtomicWork + "<br/>" + ex.Message);
//TaskLogInfo.StackTrace = ex.Source + "<br/>" + ex.Message + "<br/>" + ex.StackTrace;
//TaskLogInfo.IsError = true;
ex.Trace().Swallow();
dcbParserLogEntry.Success = false;
if (ex.ToUserString().Contains(Constants.DiskFullErrorMessage))
{
dcbParserLogEntry.LogInfo.Message = "There is not enough space on the disk";
throw;
}
else
{
dcbParserLogEntry.LogInfo.Message = ex.ToUserString();
}
}
}
private void SendRelationshipsInfo(IEnumerable <EmailThreadingEntity> rawDocumentRelationships)
{
// For eDocs we ALWAYS send relationships info
//if (!m_Parameters.IsImportFamilyRelations)
//{
// return;
//}
FamiliesInfo familiesInfo = new FamiliesInfo();
ThreadsInfo threadsInfo = new ThreadsInfo();
foreach (EmailThreadingEntity emailThreadingEntity in rawDocumentRelationships)
{
string docReferenceId = emailThreadingEntity.ChildDocumentID;
if (String.IsNullOrEmpty(docReferenceId))
{
continue;
}
if (emailThreadingEntity.RelationshipType == ThreadRelationshipEntity.RelationshipType.OutlookEmailThread)
{
// Sanitize the value
emailThreadingEntity.ConversationIndex = String.IsNullOrEmpty(emailThreadingEntity.ConversationIndex) ? null : emailThreadingEntity.ConversationIndex;
// On Append we only calculate relationships between new documents,
// therefore we don't even send standalone documents to threads linker
if (emailThreadingEntity.ConversationIndex == null)
{
continue;
}
var threadInfo = new ThreadInfo(docReferenceId, emailThreadingEntity.ConversationIndex);
threadsInfo.ThreadInfoList.Add(threadInfo);
}
else
{
// We don't skip standalone documents for Families, because they always can appear to be topmost parents
FamilyInfo familyInfoRecord = new FamilyInfo(docReferenceId);
familyInfoRecord.OriginalDocumentId = docReferenceId;
familyInfoRecord.OriginalParentId = String.IsNullOrEmpty(emailThreadingEntity.ParentDocumentID) ? null : emailThreadingEntity.ParentDocumentID;
//Tracer.Warning("SendRelationshipsInfo: OriginalDocumentId = {0}, OriginalParentId = {1}",
// familyInfoRecord.OriginalDocumentId, familyInfoRecord.OriginalParentId);
if (String.Equals(familyInfoRecord.OriginalDocumentId, familyInfoRecord.OriginalParentId, StringComparison.InvariantCulture))
{
//Tracer.Warning("SendRelationshipsInfo: OriginalDocumentId = {0}, OriginalParentId reset to null", familyInfoRecord.OriginalDocumentId);
familyInfoRecord.OriginalParentId = null; // Document must not be its own parent
}
familiesInfo.FamilyInfoList.Add(familyInfoRecord);
}
const int BatchSize = 500;
if (threadsInfo.ThreadInfoList.Count >= BatchSize)
{
SendThreads(threadsInfo);
threadsInfo.ThreadInfoList.Clear();
}
if (familiesInfo.FamilyInfoList.Count >= BatchSize)
{
SendFamilies(familiesInfo);
familiesInfo.FamilyInfoList.Clear();
}
}
if (threadsInfo.ThreadInfoList.Any())
{
SendThreads(threadsInfo);
}
if (familiesInfo.FamilyInfoList.Any())
{
SendFamilies(familiesInfo);
}
}
private void ImportDocumentRelationship(string documentId, Document currentDcbDocument, FamiliesInfo familiesInfo)
{
// Debug
//Tracer.Warning("DCB documentId = {0}", documentId);
if (DcbOpticonJobBEO.FamilyRelations.IsEmailDCB)
{
if (null != currentDcbDocument.children && currentDcbDocument.children.Any())
{
foreach (int physicalChildDocumentNumber in currentDcbDocument.children)
{
FamilyInfo familyInfoRecord1 = new FamilyInfo(null);
familyInfoRecord1.OriginalDocumentId = GenerateDocumentIdForEV(physicalChildDocumentNumber.ToString(CultureInfo.InvariantCulture));
familyInfoRecord1.OriginalParentId = GenerateDocumentIdForEV(currentDcbDocument);
familiesInfo.FamilyInfoList.Add(familyInfoRecord1);
}
}
// We don't skip standalone documents for Families, because they always can appear to be topmost parents.
// And also we need all of them to provide Original to Real Id translation.
FamilyInfo familyInfoRecord2 = new FamilyInfo(documentId);
familyInfoRecord2.OriginalDocumentId = GenerateDocumentIdForEV(currentDcbDocument.PhysicalNumber.ToString(CultureInfo.InvariantCulture));
familyInfoRecord2.OriginalParentId = null;
familiesInfo.FamilyInfoList.Add(familyInfoRecord2);
return;
}
/*One way of maintaining parent-child relationship in DCB is through PARENT_DOCID & DOCID fields
* Most of the case it is assumed that PARENT_DOCID & DOCID will have UUID, but in CNClassic there is a
* way to map DOCID and PARENT_DOCID field during import process
* For Document realtion the first priority is given to PAREN_DOCID & DOCID field */
Field dcbDocIdField = currentDcbDocument.FieldItems.Find(o => o.Code.Equals(DcbOpticonJobBEO.FamilyRelations.DocId));
string dcbDocId = null;
if (null != dcbDocIdField)
{
dcbDocId = ExtractDcbDocumentId(dcbDocIdField.Value, DcbOpticonJobBEO.FamilyRelations.IsDocIDFormatRange);
}
if (dcbDocId == null)
{
Tracer.Warning("Potentially bad DCB source data: document {0} has no DocId field", documentId);
return; // No point in sending family record with the original document Id missing
}
Field dcbParentDocIdField = currentDcbDocument.FieldItems.Find(o => o.Code.Equals(DcbOpticonJobBEO.FamilyRelations.ParentDocId));
string dcbParentDocId = null;
if (null != dcbParentDocIdField)
{
dcbParentDocId = ExtractDcbDocumentId(dcbParentDocIdField.Value, DcbOpticonJobBEO.FamilyRelations.IsParentDocIDFormatRange);
}
FamilyInfo familyInfo = new FamilyInfo(documentId);
familyInfo.OriginalDocumentId = GenerateDocumentIdForEV(dcbDocId);
familyInfo.OriginalParentId = GenerateDocumentIdForEV(dcbParentDocId);
familiesInfo.FamilyInfoList.Add(familyInfo);
//Tracer.Trace("Adding Relationship record. Parent = {0}, Child = {1}", threadingEntity.ParentDocumentID, threadingEntity.ChildDocumentID);
}
private void SendFamilies(IEnumerable<DocumentDetail> documentDetailList)
{
if (!_jobParams.CreateFamilyGroups)
{
return;
}
FamiliesInfo familiesInfo = new FamiliesInfo();
foreach (DocumentDetail doc in documentDetailList)
{
if (doc.docType != DocumentsetType.NativeSet)
{
continue; // Only original documents may participate in relationships
}
string docReferenceId = doc.document.DocumentId;
if (String.IsNullOrEmpty(docReferenceId))
{
continue;
}
if (!String.IsNullOrEmpty(doc.document.EVLoadFileDocumentId))
{
// We don't skip standalone documents for Families, because they always can appear to be topmost parents
FamilyInfo familyInfoRecord = new FamilyInfo(docReferenceId);
familyInfoRecord.OriginalDocumentId = doc.document.EVLoadFileDocumentId;
familyInfoRecord.OriginalParentId = (String.IsNullOrEmpty(doc.document.EVLoadFileParentId) || doc.document.EVLoadFileParentId == "0") ? null : doc.document.EVLoadFileParentId;
// Debug
//Tracer.Warning("LawProcessingWorker.SendFamilies: DocId = {0}, OriginalDocumentId = {1}, OriginalParentId = {2}",
// docReferenceId, familyInfoRecord.OriginalDocumentId, familyInfoRecord.OriginalParentId);
if (String.Equals(familyInfoRecord.OriginalDocumentId, familyInfoRecord.OriginalParentId, StringComparison.InvariantCulture))
{
//Tracer.Warning("SendRelationshipsInfo: OriginalDocumentId = {0}, OriginalParentId reset to null", familyInfoRecord.OriginalDocumentId);
familyInfoRecord.OriginalParentId = null; // Document must not be its own parent
}
familiesInfo.FamilyInfoList.Add(familyInfoRecord);
}
}
SendFamilies(familiesInfo);
}
